From c089f8b102fe9b4c3ca47a5b807415aea949276b Mon Sep 17 00:00:00 2001 From: Guangming Luo Date: Mon, 1 Jul 2024 19:03:32 +0800 Subject: [PATCH] chore: update Copyright header to CloudWeGo (#51) --- .licenserc.yaml | 1 - _typos.toml | 9 + debug/debug.go | 56 +- frugal.go | 14 +- fuzz/builder/dynamic_struct.go | 2 +- fuzz/builder/dynamic_struct_test.go | 2 +- fuzz/builder/main.go | 2 +- fuzz/check.go | 2 +- fuzz/fuzz_test.go | 2 +- fuzz/struct_fuzz.go | 2 +- fuzz/type.go | 2 +- internal/atm/abi/abi.go | 16 +- internal/atm/abi/abi_amd64.go | 302 ++- internal/atm/abi/abi_amd64_test.go | 8 +- internal/atm/abi/abi_arm64.go | 10 +- internal/atm/abi/abi_legacy_amd64.go | 130 +- internal/atm/abi/abi_regabi_amd64.go | 246 +- internal/atm/emu/asm.s | 2 +- internal/atm/emu/emu.go | 360 +-- internal/atm/emu/emu_test.go | 132 +- internal/atm/emu/stubs.go | 4 +- internal/atm/hir/builder.go | 362 +-- internal/atm/hir/call.go | 267 +- internal/atm/hir/ir.go | 443 ++-- internal/atm/hir/pools.go | 64 +- internal/atm/hir/prog.go | 86 +- internal/atm/hir/regs.go | 96 +- internal/atm/pgen/asm.s | 2 +- internal/atm/pgen/iasm_amd64.go | 98 +- internal/atm/pgen/pgen_abi_amd64.go | 704 +++--- internal/atm/pgen/pgen_amd64.go | 1888 +++++++------- internal/atm/pgen/pgen_amd64_test.go | 344 +-- .../atm/pgen/pgen_gcwb_go116_120_amd64.go | 94 +- .../atm/pgen/pgen_gcwb_go121_121_amd64.go | 100 +- internal/atm/pgen/pgen_legacy_amd64.go | 84 +- internal/atm/pgen/pgen_regabi_amd64.go | 99 +- internal/atm/rtx/asm.s | 2 +- internal/atm/rtx/clobber_amd64.go | 362 +-- internal/atm/rtx/clobber_amd64_test.go | 8 +- internal/atm/rtx/gcwb_go116_120.go | 10 +- internal/atm/rtx/gcwb_go121_121.go | 16 +- internal/atm/rtx/memcpy.go | 16 +- internal/atm/rtx/memzero.go | 12 +- internal/atm/rtx/memzero_amd64.go | 80 +- internal/atm/rtx/memzero_test.go | 54 +- internal/atm/rtx/stack.go | 8 +- internal/atm/ssa/block.go | 989 ++++---- internal/atm/ssa/blockiter.go | 96 +- internal/atm/ssa/builder.go | 280 ++- internal/atm/ssa/cfg.go | 66 +- internal/atm/ssa/cfg_test.go | 214 +- internal/atm/ssa/compile.go | 88 +- internal/atm/ssa/constdata.go | 42 +- internal/atm/ssa/dominator.go | 426 ++-- internal/atm/ssa/funcdata.go | 58 +- internal/atm/ssa/int65.go | 80 +- internal/atm/ssa/int65_test.go | 42 +- internal/atm/ssa/ir.go | 1391 +++++----- internal/atm/ssa/ir_amd64.go | 2239 +++++++++-------- internal/atm/ssa/pass_abispec.go | 163 +- internal/atm/ssa/pass_abispec_amd64.go | 472 ++-- internal/atm/ssa/pass_blockmerge.go | 103 +- internal/atm/ssa/pass_branchelim.go | 1012 ++++---- internal/atm/ssa/pass_compact_amd64.go | 473 ++-- internal/atm/ssa/pass_comsubexpr.go | 233 +- internal/atm/ssa/pass_constprop.go | 542 ++-- internal/atm/ssa/pass_copyelim.go | 245 +- internal/atm/ssa/pass_fusion_amd64.go | 736 +++--- internal/atm/ssa/pass_layout.go | 186 +- internal/atm/ssa/pass_lowering_amd64.go | 311 ++- internal/atm/ssa/pass_mbarrier_amd64.go | 256 +- internal/atm/ssa/pass_operandalloc_amd64.go | 147 +- internal/atm/ssa/pass_phielim.go | 166 +- internal/atm/ssa/pass_phiprop.go | 354 +-- internal/atm/ssa/pass_reduce.go | 19 +- internal/atm/ssa/pass_regalloc.go | 1580 ++++++------ internal/atm/ssa/pass_rematerialize.go | 56 +- internal/atm/ssa/pass_reorder.go | 617 +++-- internal/atm/ssa/pass_return_spread.go | 234 +- internal/atm/ssa/pass_splitcritical.go | 118 +- internal/atm/ssa/pass_stack_liveness.go | 205 +- internal/atm/ssa/pass_tdce.go | 272 +- internal/atm/ssa/pass_zeroreg.go | 134 +- internal/atm/ssa/phi.go | 242 +- internal/atm/ssa/pos.go | 26 +- internal/atm/ssa/rename.go | 280 +-- internal/atm/ssa/slotset.go | 82 +- internal/atm/ssa/utils.go | 156 +- internal/binary/decoder/alloc.go | 12 +- internal/binary/decoder/alloc_emu.go | 28 +- internal/binary/decoder/asm.s | 2 +- internal/binary/decoder/bitmap.go | 42 +- internal/binary/decoder/bitmap_emu.go | 26 +- internal/binary/decoder/compiler.go | 894 ++++--- internal/binary/decoder/compiler_test.go | 74 +- internal/binary/decoder/decoder.go | 220 +- internal/binary/decoder/decoder_test.go | 252 +- internal/binary/decoder/errors.go | 40 +- internal/binary/decoder/errors_emu.go | 54 +- internal/binary/decoder/initfn.go | 76 +- internal/binary/decoder/linker.go | 26 +- internal/binary/decoder/linker_amd64.go | 22 +- internal/binary/decoder/linker_arm64.go | 2 +- internal/binary/decoder/linker_emu.go | 80 +- internal/binary/decoder/mapassign.go | 18 +- internal/binary/decoder/mapassign_emu.go | 64 +- internal/binary/decoder/native_amd64.go | 2 +- internal/binary/decoder/opcode.go | 184 +- internal/binary/decoder/optimizer.go | 454 ++-- internal/binary/decoder/pools.go | 144 +- internal/binary/decoder/skipping.go | 12 +- internal/binary/decoder/skipping_amd64.go | 6 +- internal/binary/decoder/skipping_arm64.go | 4 +- internal/binary/decoder/skipping_emu.go | 486 ++-- internal/binary/decoder/skipping_emu_test.go | 270 +- internal/binary/decoder/state.go | 50 +- internal/binary/decoder/strconv.go | 8 +- internal/binary/decoder/strconv_emu.go | 22 +- internal/binary/decoder/translator.go | 1256 ++++----- internal/binary/decoder/translator_test.go | 50 +- internal/binary/defs/defaults.go | 50 +- internal/binary/defs/defaults_test.go | 42 +- internal/binary/defs/resolver.go | 386 +-- internal/binary/defs/resolver_test.go | 32 +- internal/binary/defs/sizes.go | 85 +- internal/binary/defs/types.go | 786 +++--- internal/binary/defs/types_test.go | 26 +- internal/binary/encoder/asm.s | 2 +- internal/binary/encoder/bucket.go | 132 +- internal/binary/encoder/compiler.go | 286 ++- internal/binary/encoder/compiler_encode.go | 621 ++--- internal/binary/encoder/compiler_measure.go | 612 ++--- internal/binary/encoder/compiler_test.go | 48 +- internal/binary/encoder/encoder.go | 178 +- internal/binary/encoder/encoder_test.go | 148 +- internal/binary/encoder/hash.go | 22 +- internal/binary/encoder/linker.go | 26 +- internal/binary/encoder/linker_amd64.go | 22 +- internal/binary/encoder/linker_arm64.go | 2 +- internal/binary/encoder/linker_emu.go | 94 +- internal/binary/encoder/mapiter.go | 16 +- internal/binary/encoder/mapiter_emu.go | 26 +- internal/binary/encoder/opcode.go | 176 +- internal/binary/encoder/optimizer.go | 740 +++--- internal/binary/encoder/pools.go | 140 +- internal/binary/encoder/state.go | 42 +- internal/binary/encoder/translator.go | 903 +++---- internal/binary/encoder/translator_test.go | 52 +- internal/binary/encoder/unique.go | 80 +- internal/binary/encoder/unique_emu.go | 44 +- internal/binary/encoder/utils.go | 24 +- internal/cpu/features.go | 6 +- internal/loader/funcdata.go | 92 +- internal/loader/funcdata_go116_117.go | 357 +-- internal/loader/funcdata_go118_121.go | 295 ++- internal/loader/funcdata_invalid.go | 8 +- internal/loader/loader.go | 78 +- internal/loader/loader_amd64_test.go | 260 +- internal/opts/limits.go | 32 +- internal/opts/options.go | 22 +- internal/rt/asm.s | 2 +- internal/rt/escape.go | 9 +- internal/rt/exports.go | 8 +- internal/rt/frame.go | 14 +- internal/rt/funcname.go | 22 +- internal/rt/iface.go | 38 +- internal/rt/iface_test.go | 24 +- internal/rt/runtime.go | 373 +-- internal/rt/stackmap.go | 168 +- internal/rt/stackmap_test.go | 98 +- internal/utils/buffer.go | 12 +- internal/utils/buffer_emu.go | 34 +- internal/utils/errors.go | 48 +- internal/utils/flags.go | 4 +- internal/utils/ints.go | 12 +- internal/utils/pcache.go | 218 +- internal/utils/types.go | 8 +- iov/buffer.go | 8 +- licenses/{LISENSE-go-spew => LICENSE-go-spew} | 0 options.go | 46 +- pretouch.go | 138 +- tests/allsize_test.go | 2 +- tests/baseline_test.go | 2 +- tests/debughook_amd64.go | 48 +- tests/deep_nested_test.go | 16 + tests/frugal_test.go | 2 +- 186 files changed, 18267 insertions(+), 16994 deletions(-) create mode 100644 _typos.toml rename licenses/{LISENSE-go-spew => LICENSE-go-spew} (100%) diff --git a/.licenserc.yaml b/.licenserc.yaml index 51084a5..726b267 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -5,7 +5,6 @@ header: paths: - "**/*.go" - - "**/*.s" paths-ignore: - internal/binary/decoder/native_subr_amd64.go diff --git a/_typos.toml b/_typos.toml new file mode 100644 index 0000000..d8412f3 --- /dev/null +++ b/_typos.toml @@ -0,0 +1,9 @@ +# Typo check: https://github.com/crate-ci/typos + +[files] +extend-exclude = ["go.mod", "go.sum", "fuzz/go.mod", "fuzz/go.sum"] + +[default.extend-words] +Pn = "Pn" +worl = "worl" +rcall = "rcall" \ No newline at end of file diff --git a/debug/debug.go b/debug/debug.go index 07bc2a9..e30816c 100644 --- a/debug/debug.go +++ b/debug/debug.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,47 +17,47 @@ package debug import ( - `github.com/cloudwego/frugal/internal/binary/decoder` - `github.com/cloudwego/frugal/internal/binary/encoder` - `github.com/cloudwego/frugal/internal/loader` + "github.com/cloudwego/frugal/internal/binary/decoder" + "github.com/cloudwego/frugal/internal/binary/encoder" + "github.com/cloudwego/frugal/internal/loader" ) // A Stats records statistics about the JIT compiler. type Stats struct { - Memory MemStats - Encoder CacheStats - Decoder CacheStats + Memory MemStats + Encoder CacheStats + Decoder CacheStats } // A MemStats records statistics about the memory allocator used in the JIT compiler. type MemStats struct { - Alloc int - Count int + Alloc int + Count int } // A CacheStats records statistics about the type cache. type CacheStats struct { - Hit int - Miss int - Size int + Hit int + Miss int + Size int } // GetStats returns statistics of the JIT compiler. func GetStats() Stats { - return Stats { - Memory: MemStats { - Count: int(loader.FnCount), - Alloc: int(loader.LoadSize), - }, - Encoder: CacheStats { - Hit : int(encoder.HitCount), - Miss : int(encoder.MissCount), - Size : int(encoder.TypeCount), - }, - Decoder: CacheStats { - Hit : int(decoder.HitCount), - Miss : int(decoder.MissCount), - Size : int(decoder.TypeCount), - }, - } + return Stats{ + Memory: MemStats{ + Count: int(loader.FnCount), + Alloc: int(loader.LoadSize), + }, + Encoder: CacheStats{ + Hit: int(encoder.HitCount), + Miss: int(encoder.MissCount), + Size: int(encoder.TypeCount), + }, + Decoder: CacheStats{ + Hit: int(decoder.HitCount), + Miss: int(decoder.MissCount), + Size: int(decoder.TypeCount), + }, + } } diff --git a/frugal.go b/frugal.go index 8322118..dcc985c 100644 --- a/frugal.go +++ b/frugal.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,23 +17,23 @@ package frugal import ( - `github.com/cloudwego/frugal/internal/binary/decoder` - `github.com/cloudwego/frugal/internal/binary/encoder` - `github.com/cloudwego/frugal/iov` + "github.com/cloudwego/frugal/internal/binary/decoder" + "github.com/cloudwego/frugal/internal/binary/encoder" + "github.com/cloudwego/frugal/iov" ) // EncodedSize measures the encoded size of val. func EncodedSize(val interface{}) int { - return encoder.EncodedSize(val) + return encoder.EncodedSize(val) } // EncodeObject serializes val into buf with Thrift Binary Protocol, with optional Zero-Copy iov.BufferWriter. // buf must be large enough to contain the entire serialization result. func EncodeObject(buf []byte, mem iov.BufferWriter, val interface{}) (int, error) { - return encoder.EncodeObject(buf, mem, val) + return encoder.EncodeObject(buf, mem, val) } // DecodeObject deserializes buf into val with Thrift Binary Protocol. func DecodeObject(buf []byte, val interface{}) (int, error) { - return decoder.DecodeObject(buf, val) + return decoder.DecodeObject(buf, val) } diff --git a/fuzz/builder/dynamic_struct.go b/fuzz/builder/dynamic_struct.go index ad0d808..ac6e068 100644 --- a/fuzz/builder/dynamic_struct.go +++ b/fuzz/builder/dynamic_struct.go @@ -1,4 +1,4 @@ -// Copyright 2022 ByteDance Inc. +// Copyright 2022 CloudWeGo Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/fuzz/builder/dynamic_struct_test.go b/fuzz/builder/dynamic_struct_test.go index 7856d57..1fe328a 100644 --- a/fuzz/builder/dynamic_struct_test.go +++ b/fuzz/builder/dynamic_struct_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 ByteDance Inc. +// Copyright 2022 CloudWeGo Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/fuzz/builder/main.go b/fuzz/builder/main.go index 95fcdf2..bc6fddb 100644 --- a/fuzz/builder/main.go +++ b/fuzz/builder/main.go @@ -1,4 +1,4 @@ -// Copyright 2022 ByteDance Inc. +// Copyright 2022 CloudWeGo Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/fuzz/check.go b/fuzz/check.go index 6b18497..61cafd5 100644 --- a/fuzz/check.go +++ b/fuzz/check.go @@ -1,4 +1,4 @@ -// Copyright 2022 ByteDance Inc. +// Copyright 2022 CloudWeGo Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/fuzz/fuzz_test.go b/fuzz/fuzz_test.go index f1670e9..364ef31 100644 --- a/fuzz/fuzz_test.go +++ b/fuzz/fuzz_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 ByteDance Inc. +// Copyright 2022 CloudWeGo Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/fuzz/struct_fuzz.go b/fuzz/struct_fuzz.go index 0ea1241..8f3bc17 100644 --- a/fuzz/struct_fuzz.go +++ b/fuzz/struct_fuzz.go @@ -1,4 +1,4 @@ -// Copyright 2022 ByteDance Inc. +// Copyright 2022 CloudWeGo Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/fuzz/type.go b/fuzz/type.go index 6438621..a3a8238 100644 --- a/fuzz/type.go +++ b/fuzz/type.go @@ -1,4 +1,4 @@ -// Copyright 2022 ByteDance Inc. +// Copyright 2022 CloudWeGo Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/internal/atm/abi/abi.go b/internal/atm/abi/abi.go index 279abd3..c0b03a7 100644 --- a/internal/atm/abi/abi.go +++ b/internal/atm/abi/abi.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,24 +17,24 @@ package abi import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) type AbstractABI interface { - RegisterMethod(id int, mt rt.Method) int - RegisterFunction(id int, fn interface{}) unsafe.Pointer + RegisterMethod(id int, mt rt.Method) int + RegisterFunction(id int, fn interface{}) unsafe.Pointer } var ( - ABI = ArchCreateABI() + ABI = ArchCreateABI() ) const ( - PtrSize = 8 // pointer size + PtrSize = 8 // pointer size ) func alignUp(n uintptr, a int) uintptr { - return (n + uintptr(a) - 1) &^ (uintptr(a) - 1) + return (n + uintptr(a) - 1) &^ (uintptr(a) - 1) } diff --git a/internal/atm/abi/abi_amd64.go b/internal/atm/abi/abi_amd64.go index 86101be..945a8c7 100644 --- a/internal/atm/abi/abi_amd64.go +++ b/internal/atm/abi/abi_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,206 +17,230 @@ package abi import ( - `fmt` - `reflect` - `sort` - `strings` - `sync` - `unsafe` - - `github.com/cloudwego/iasm/x86_64` - `github.com/cloudwego/frugal/internal/rt` + "fmt" + "reflect" + "sort" + "strings" + "sync" + "unsafe" + + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/iasm/x86_64" ) const ( - PtrAlign = 8 // pointer alignment + PtrAlign = 8 // pointer alignment ) type Parameter struct { - Mem uintptr - Reg x86_64.Register64 - Type reflect.Type - InRegister bool + Mem uintptr + Reg x86_64.Register64 + Type reflect.Type + InRegister bool } var ( - intType = reflect.TypeOf(0) - ptrType = reflect.TypeOf(unsafe.Pointer(nil)) + intType = reflect.TypeOf(0) + ptrType = reflect.TypeOf(unsafe.Pointer(nil)) ) func mkReg(vt reflect.Type, reg x86_64.Register64) (p Parameter) { - p.Reg = reg - p.Type = vt - p.InRegister = true - return + p.Reg = reg + p.Type = vt + p.InRegister = true + return } func mkStack(vt reflect.Type, mem uintptr) (p Parameter) { - p.Mem = mem - p.Type = vt - p.InRegister = false - return + p.Mem = mem + p.Type = vt + p.InRegister = false + return } func (self Parameter) String() string { - if self.InRegister { - return fmt.Sprintf("%%%s", self.Reg) - } else { - return fmt.Sprintf("%d(%%rsp)", self.Mem) - } + if self.InRegister { + return fmt.Sprintf("%%%s", self.Reg) + } else { + return fmt.Sprintf("%d(%%rsp)", self.Mem) + } } func (self Parameter) IsPointer() bool { - switch self.Type.Kind() { - case reflect.Bool : fallthrough - case reflect.Int : fallthrough - case reflect.Int8 : fallthrough - case reflect.Int16 : fallthrough - case reflect.Int32 : fallthrough - case reflect.Int64 : fallthrough - case reflect.Uint : fallthrough - case reflect.Uint8 : fallthrough - case reflect.Uint16 : fallthrough - case reflect.Uint32 : fallthrough - case reflect.Uint64 : fallthrough - case reflect.Uintptr : return false - case reflect.Chan : fallthrough - case reflect.Func : fallthrough - case reflect.Map : fallthrough - case reflect.Ptr : fallthrough - case reflect.UnsafePointer : return true - case reflect.Float32 : fallthrough - case reflect.Float64 : fallthrough - case reflect.Complex64 : fallthrough - case reflect.Complex128 : fallthrough - case reflect.Array : fallthrough - case reflect.Struct : panic("abi: unsupported types") - default : panic("abi: invalid value type") - } + switch self.Type.Kind() { + case reflect.Bool: + fallthrough + case reflect.Int: + fallthrough + case reflect.Int8: + fallthrough + case reflect.Int16: + fallthrough + case reflect.Int32: + fallthrough + case reflect.Int64: + fallthrough + case reflect.Uint: + fallthrough + case reflect.Uint8: + fallthrough + case reflect.Uint16: + fallthrough + case reflect.Uint32: + fallthrough + case reflect.Uint64: + fallthrough + case reflect.Uintptr: + return false + case reflect.Chan: + fallthrough + case reflect.Func: + fallthrough + case reflect.Map: + fallthrough + case reflect.Ptr: + fallthrough + case reflect.UnsafePointer: + return true + case reflect.Float32: + fallthrough + case reflect.Float64: + fallthrough + case reflect.Complex64: + fallthrough + case reflect.Complex128: + fallthrough + case reflect.Array: + fallthrough + case reflect.Struct: + panic("abi: unsupported types") + default: + panic("abi: invalid value type") + } } type _StackSlot struct { - p bool - m uintptr + p bool + m uintptr } type FunctionLayout struct { - Id int - Sp uintptr - Args []Parameter - Rets []Parameter + Id int + Sp uintptr + Args []Parameter + Rets []Parameter } func (self *FunctionLayout) String() string { - if self.Id < 0 { - return fmt.Sprintf("{func,%s}", self.formatFn()) - } else { - return fmt.Sprintf("{meth/%d,%s}", self.Id, self.formatFn()) - } + if self.Id < 0 { + return fmt.Sprintf("{func,%s}", self.formatFn()) + } else { + return fmt.Sprintf("{meth/%d,%s}", self.Id, self.formatFn()) + } } func (self *FunctionLayout) StackMap() *rt.StackMap { - var st []_StackSlot - var mb rt.StackMapBuilder - - /* add arguments */ - for _, v := range self.Args { - st = append(st, _StackSlot { - m: v.Mem, - p: v.IsPointer(), - }) - } - - /* add stack-passed return values */ - for _, v := range self.Rets { - if !v.InRegister { - st = append(st, _StackSlot { - m: v.Mem, - p: v.IsPointer(), - }) - } - } - - /* sort by memory offset */ - sort.Slice(st, func(i int, j int) bool { - return st[i].m < st[j].m - }) - - /* add the bits */ - for _, v := range st { - mb.AddField(v.p) - } - - /* build the stack map */ - return mb.Build() + var st []_StackSlot + var mb rt.StackMapBuilder + + /* add arguments */ + for _, v := range self.Args { + st = append(st, _StackSlot{ + m: v.Mem, + p: v.IsPointer(), + }) + } + + /* add stack-passed return values */ + for _, v := range self.Rets { + if !v.InRegister { + st = append(st, _StackSlot{ + m: v.Mem, + p: v.IsPointer(), + }) + } + } + + /* sort by memory offset */ + sort.Slice(st, func(i int, j int) bool { + return st[i].m < st[j].m + }) + + /* add the bits */ + for _, v := range st { + mb.AddField(v.p) + } + + /* build the stack map */ + return mb.Build() } func (self *FunctionLayout) formatFn() string { - return fmt.Sprintf("$%#x,(%s),(%s)", self.Sp, self.formatSeq(self.Args), self.formatSeq(self.Rets)) + return fmt.Sprintf("$%#x,(%s),(%s)", self.Sp, self.formatSeq(self.Args), self.formatSeq(self.Rets)) } func (self *FunctionLayout) formatSeq(v []Parameter) string { - nb := len(v) - mm := make([]string, len(v)) + nb := len(v) + mm := make([]string, len(v)) - /* convert each part */ - for i := 0; i < nb; i++ { - mm[i] = v[i].String() - } + /* convert each part */ + for i := 0; i < nb; i++ { + mm[i] = v[i].String() + } - /* join them together */ - return strings.Join(mm, ",") + /* join them together */ + return strings.Join(mm, ",") } type AMD64ABI struct { - m sync.RWMutex - fnTab map[int]*FunctionLayout + m sync.RWMutex + fnTab map[int]*FunctionLayout } func ArchCreateABI() *AMD64ABI { - return &AMD64ABI { - fnTab: make(map[int]*FunctionLayout), - } + return &AMD64ABI{ + fnTab: make(map[int]*FunctionLayout), + } } func (self *AMD64ABI) GetLayout(id int) (layout *FunctionLayout) { - self.m.RLock() - layout = self.fnTab[id] - self.m.RUnlock() - return + self.m.RLock() + layout = self.fnTab[id] + self.m.RUnlock() + return } func (self *AMD64ABI) SetLayout(id int, layout *FunctionLayout) { - self.m.Lock() - self.fnTab[id] = layout - self.m.Unlock() + self.m.Lock() + self.fnTab[id] = layout + self.m.Unlock() } func (self *AMD64ABI) DumpLayouts() map[int]*FunctionLayout { - self.m.RLock() - result := make(map[int]*FunctionLayout, len(self.fnTab)) - for k, v := range self.fnTab { - result[k] = v - } - self.m.RUnlock() - return result + self.m.RLock() + result := make(map[int]*FunctionLayout, len(self.fnTab)) + for k, v := range self.fnTab { + result[k] = v + } + self.m.RUnlock() + return result } func (self *AMD64ABI) RegisterMethod(id int, mt rt.Method) int { - self.SetLayout(id, self.LayoutFunc(mt.Id, mt.Vt.Pack().Method(mt.Id).Type)) - return mt.Id + self.SetLayout(id, self.LayoutFunc(mt.Id, mt.Vt.Pack().Method(mt.Id).Type)) + return mt.Id } func (self *AMD64ABI) RegisterFunction(id int, fn interface{}) (fp unsafe.Pointer) { - vv := rt.UnpackEface(fn) - vt := vv.Type.Pack() + vv := rt.UnpackEface(fn) + vt := vv.Type.Pack() - /* must be a function */ - if vt.Kind() != reflect.Func { - panic("fn is not a function") - } + /* must be a function */ + if vt.Kind() != reflect.Func { + panic("fn is not a function") + } - /* layout the function, and get the real function address */ - self.SetLayout(id, self.LayoutFunc(-1, vt)) - return *(*unsafe.Pointer)(vv.Value) + /* layout the function, and get the real function address */ + self.SetLayout(id, self.LayoutFunc(-1, vt)) + return *(*unsafe.Pointer)(vv.Value) } diff --git a/internal/atm/abi/abi_amd64_test.go b/internal/atm/abi/abi_amd64_test.go index 52a5c22..bcf1cbb 100644 --- a/internal/atm/abi/abi_amd64_test.go +++ b/internal/atm/abi/abi_amd64_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,11 +17,11 @@ package abi import ( - `testing` + "testing" - `github.com/davecgh/go-spew/spew` + "github.com/davecgh/go-spew/spew" ) func TestABI_FunctionLayout(t *testing.T) { - spew.Dump(ABI.DumpLayouts()) + spew.Dump(ABI.DumpLayouts()) } diff --git a/internal/atm/abi/abi_arm64.go b/internal/atm/abi/abi_arm64.go index 8d16df6..b3476e3 100644 --- a/internal/atm/abi/abi_arm64.go +++ b/internal/atm/abi/abi_arm64.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 ByteDance Inc. + * Copyright 2024 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,16 +17,16 @@ package abi import ( - `reflect` - `unsafe` + "reflect" + "unsafe" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) // NOTE: this file is temporary and is used only for emu on arm // TODO: delete this file after frugal supports arm -type ARM64ABI struct {} +type ARM64ABI struct{} func ArchCreateABI() *ARM64ABI { return &ARM64ABI{} diff --git a/internal/atm/abi/abi_legacy_amd64.go b/internal/atm/abi/abi_legacy_amd64.go index 7651642..0e73d40 100644 --- a/internal/atm/abi/abi_legacy_amd64.go +++ b/internal/atm/abi/abi_legacy_amd64.go @@ -1,7 +1,8 @@ +//go:build !go1.17 // +build !go1.17 /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,68 +20,95 @@ package abi import ( - `reflect` + "reflect" - `github.com/cloudwego/iasm/x86_64` + "github.com/cloudwego/iasm/x86_64" ) func salloc(p []Parameter, sp uintptr, vt reflect.Type) (uintptr, []Parameter) { - switch vt.Kind() { - case reflect.Bool : return sp + 8, append(p, mkStack(reflect.TypeOf(false), sp)) - case reflect.Int : return sp + 8, append(p, mkStack(intType, sp)) - case reflect.Int8 : return sp + 8, append(p, mkStack(reflect.TypeOf(int8(0)), sp)) - case reflect.Int16 : return sp + 8, append(p, mkStack(reflect.TypeOf(int16(0)), sp)) - case reflect.Int32 : return sp + 8, append(p, mkStack(reflect.TypeOf(int32(0)), sp)) - case reflect.Int64 : return sp + 8, append(p, mkStack(reflect.TypeOf(int64(0)), sp)) - case reflect.Uint : return sp + 8, append(p, mkStack(reflect.TypeOf(uint(0)), sp)) - case reflect.Uint8 : return sp + 8, append(p, mkStack(reflect.TypeOf(uint8(0)), sp)) - case reflect.Uint16 : return sp + 8, append(p, mkStack(reflect.TypeOf(uint16(0)), sp)) - case reflect.Uint32 : return sp + 8, append(p, mkStack(reflect.TypeOf(uint32(0)), sp)) - case reflect.Uint64 : return sp + 8, append(p, mkStack(reflect.TypeOf(uint64(0)), sp)) - case reflect.Uintptr : return sp + 8, append(p, mkStack(reflect.TypeOf(uintptr(0)), sp)) - case reflect.Float32 : panic("abi: go116: not implemented: float32") - case reflect.Float64 : panic("abi: go116: not implemented: float64") - case reflect.Complex64 : panic("abi: go116: not implemented: complex64") - case reflect.Complex128 : panic("abi: go116: not implemented: complex128") - case reflect.Array : panic("abi: go116: not implemented: arrays") - case reflect.Chan : return sp + 8, append(p, mkStack(reflect.TypeOf((chan int)(nil)), sp)) - case reflect.Func : return sp + 8, append(p, mkStack(reflect.TypeOf((func())(nil)), sp)) - case reflect.Map : return sp + 8, append(p, mkStack(reflect.TypeOf((map[int]int)(nil)), sp)) - case reflect.Ptr : return sp + 8, append(p, mkStack(reflect.TypeOf((*int)(nil)), sp)) - case reflect.UnsafePointer : return sp + 8, append(p, mkStack(ptrType, sp)) - case reflect.Interface : return sp + 16, append(p, mkStack(ptrType, sp), mkStack(ptrType, sp + 8)) - case reflect.Slice : return sp + 24, append(p, mkStack(ptrType, sp), mkStack(intType, sp + 8), mkStack(intType, sp + 16)) - case reflect.String : return sp + 16, append(p, mkStack(ptrType, sp), mkStack(intType, sp + 8)) - case reflect.Struct : panic("abi: go116: not implemented: structs") - default : panic("abi: invalid value type") - } + switch vt.Kind() { + case reflect.Bool: + return sp + 8, append(p, mkStack(reflect.TypeOf(false), sp)) + case reflect.Int: + return sp + 8, append(p, mkStack(intType, sp)) + case reflect.Int8: + return sp + 8, append(p, mkStack(reflect.TypeOf(int8(0)), sp)) + case reflect.Int16: + return sp + 8, append(p, mkStack(reflect.TypeOf(int16(0)), sp)) + case reflect.Int32: + return sp + 8, append(p, mkStack(reflect.TypeOf(int32(0)), sp)) + case reflect.Int64: + return sp + 8, append(p, mkStack(reflect.TypeOf(int64(0)), sp)) + case reflect.Uint: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint(0)), sp)) + case reflect.Uint8: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint8(0)), sp)) + case reflect.Uint16: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint16(0)), sp)) + case reflect.Uint32: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint32(0)), sp)) + case reflect.Uint64: + return sp + 8, append(p, mkStack(reflect.TypeOf(uint64(0)), sp)) + case reflect.Uintptr: + return sp + 8, append(p, mkStack(reflect.TypeOf(uintptr(0)), sp)) + case reflect.Float32: + panic("abi: go116: not implemented: float32") + case reflect.Float64: + panic("abi: go116: not implemented: float64") + case reflect.Complex64: + panic("abi: go116: not implemented: complex64") + case reflect.Complex128: + panic("abi: go116: not implemented: complex128") + case reflect.Array: + panic("abi: go116: not implemented: arrays") + case reflect.Chan: + return sp + 8, append(p, mkStack(reflect.TypeOf((chan int)(nil)), sp)) + case reflect.Func: + return sp + 8, append(p, mkStack(reflect.TypeOf((func())(nil)), sp)) + case reflect.Map: + return sp + 8, append(p, mkStack(reflect.TypeOf((map[int]int)(nil)), sp)) + case reflect.Ptr: + return sp + 8, append(p, mkStack(reflect.TypeOf((*int)(nil)), sp)) + case reflect.UnsafePointer: + return sp + 8, append(p, mkStack(ptrType, sp)) + case reflect.Interface: + return sp + 16, append(p, mkStack(ptrType, sp), mkStack(ptrType, sp+8)) + case reflect.Slice: + return sp + 24, append(p, mkStack(ptrType, sp), mkStack(intType, sp+8), mkStack(intType, sp+16)) + case reflect.String: + return sp + 16, append(p, mkStack(ptrType, sp), mkStack(intType, sp+8)) + case reflect.Struct: + panic("abi: go116: not implemented: structs") + default: + panic("abi: invalid value type") + } } func (self *AMD64ABI) Reserved() map[x86_64.Register64]int32 { - return nil + return nil } func (self *AMD64ABI) LayoutFunc(id int, ft reflect.Type) *FunctionLayout { - var sp uintptr - var fn FunctionLayout + var sp uintptr + var fn FunctionLayout - /* allocate the receiver if any (interface call always uses pointer) */ - if id >= 0 { - sp, fn.Args = salloc(fn.Args, sp, ptrType) - } + /* allocate the receiver if any (interface call always uses pointer) */ + if id >= 0 { + sp, fn.Args = salloc(fn.Args, sp, ptrType) + } - /* assign every arguments */ - for i := 0; i < ft.NumIn(); i++ { - sp, fn.Args = salloc(fn.Args, sp, ft.In(i)) - } + /* assign every arguments */ + for i := 0; i < ft.NumIn(); i++ { + sp, fn.Args = salloc(fn.Args, sp, ft.In(i)) + } - /* assign every return value */ - for i := 0; i < ft.NumOut(); i++ { - sp, fn.Rets = salloc(fn.Rets, sp, ft.Out(i)) - } + /* assign every return value */ + for i := 0; i < ft.NumOut(); i++ { + sp, fn.Rets = salloc(fn.Rets, sp, ft.Out(i)) + } - /* update function ID and stack pointer */ - fn.Id = id - fn.Sp = sp - return &fn + /* update function ID and stack pointer */ + fn.Id = id + fn.Sp = sp + return &fn } diff --git a/internal/atm/abi/abi_regabi_amd64.go b/internal/atm/abi/abi_regabi_amd64.go index 2ccf5a8..2070c85 100644 --- a/internal/atm/abi/abi_regabi_amd64.go +++ b/internal/atm/abi/abi_regabi_amd64.go @@ -1,7 +1,8 @@ +//go:build go1.17 // +build go1.17 /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,143 +26,174 @@ package abi import ( - `reflect` + "reflect" - `github.com/cloudwego/iasm/x86_64` + "github.com/cloudwego/iasm/x86_64" ) -var regOrder = [...]x86_64.Register64 { - x86_64.RAX, - x86_64.RBX, - x86_64.RCX, - x86_64.RDI, - x86_64.RSI, - x86_64.R8, - x86_64.R9, - x86_64.R10, - x86_64.R11, +var regOrder = [...]x86_64.Register64{ + x86_64.RAX, + x86_64.RBX, + x86_64.RCX, + x86_64.RDI, + x86_64.RSI, + x86_64.R8, + x86_64.R9, + x86_64.R10, + x86_64.R11, } type _StackAlloc struct { - i int - s uintptr + i int + s uintptr } func (self *_StackAlloc) reg(vt reflect.Type) (p Parameter) { - p = mkReg(vt, regOrder[self.i]) - self.i++ - return + p = mkReg(vt, regOrder[self.i]) + self.i++ + return } func (self *_StackAlloc) stack(vt reflect.Type) (p Parameter) { - p = mkStack(vt, self.s) - self.s += vt.Size() - return + p = mkStack(vt, self.s) + self.s += vt.Size() + return } func (self *_StackAlloc) spill(n uintptr, a int) uintptr { - self.s = alignUp(self.s, a) + n - return self.s + self.s = alignUp(self.s, a) + n + return self.s } func (self *_StackAlloc) alloc(p []Parameter, vt reflect.Type) []Parameter { - nb := vt.Size() - vk := vt.Kind() - - /* zero-sized objects are allocated on stack */ - if nb == 0 { - return append(p, mkStack(intType, self.s)) - } - - /* check for value type */ - switch vk { - case reflect.Bool : return self.valloc(p, reflect.TypeOf(false)) - case reflect.Int : return self.valloc(p, intType) - case reflect.Int8 : return self.valloc(p, reflect.TypeOf(int8(0))) - case reflect.Int16 : return self.valloc(p, reflect.TypeOf(int16(0))) - case reflect.Int32 : return self.valloc(p, reflect.TypeOf(int32(0))) - case reflect.Int64 : return self.valloc(p, reflect.TypeOf(int64(0))) - case reflect.Uint : return self.valloc(p, reflect.TypeOf(uint(0))) - case reflect.Uint8 : return self.valloc(p, reflect.TypeOf(uint8(0))) - case reflect.Uint16 : return self.valloc(p, reflect.TypeOf(uint16(0))) - case reflect.Uint32 : return self.valloc(p, reflect.TypeOf(uint32(0))) - case reflect.Uint64 : return self.valloc(p, reflect.TypeOf(uint64(0))) - case reflect.Uintptr : return self.valloc(p, reflect.TypeOf(uintptr(0))) - case reflect.Float32 : panic("abi: go117: not implemented: float32") - case reflect.Float64 : panic("abi: go117: not implemented: float64") - case reflect.Complex64 : panic("abi: go117: not implemented: complex64") - case reflect.Complex128 : panic("abi: go117: not implemented: complex128") - case reflect.Array : panic("abi: go117: not implemented: arrays") - case reflect.Chan : return self.valloc(p, reflect.TypeOf((chan int)(nil))) - case reflect.Func : return self.valloc(p, reflect.TypeOf((func())(nil))) - case reflect.Map : return self.valloc(p, reflect.TypeOf((map[int]int)(nil))) - case reflect.Ptr : return self.valloc(p, reflect.TypeOf((*int)(nil))) - case reflect.UnsafePointer : return self.valloc(p, ptrType) - case reflect.Interface : return self.valloc(p, ptrType, ptrType) - case reflect.Slice : return self.valloc(p, ptrType, intType, intType) - case reflect.String : return self.valloc(p, ptrType, intType) - case reflect.Struct : panic("abi: go117: not implemented: structs") - default : panic("abi: invalid value type") - } + nb := vt.Size() + vk := vt.Kind() + + /* zero-sized objects are allocated on stack */ + if nb == 0 { + return append(p, mkStack(intType, self.s)) + } + + /* check for value type */ + switch vk { + case reflect.Bool: + return self.valloc(p, reflect.TypeOf(false)) + case reflect.Int: + return self.valloc(p, intType) + case reflect.Int8: + return self.valloc(p, reflect.TypeOf(int8(0))) + case reflect.Int16: + return self.valloc(p, reflect.TypeOf(int16(0))) + case reflect.Int32: + return self.valloc(p, reflect.TypeOf(int32(0))) + case reflect.Int64: + return self.valloc(p, reflect.TypeOf(int64(0))) + case reflect.Uint: + return self.valloc(p, reflect.TypeOf(uint(0))) + case reflect.Uint8: + return self.valloc(p, reflect.TypeOf(uint8(0))) + case reflect.Uint16: + return self.valloc(p, reflect.TypeOf(uint16(0))) + case reflect.Uint32: + return self.valloc(p, reflect.TypeOf(uint32(0))) + case reflect.Uint64: + return self.valloc(p, reflect.TypeOf(uint64(0))) + case reflect.Uintptr: + return self.valloc(p, reflect.TypeOf(uintptr(0))) + case reflect.Float32: + panic("abi: go117: not implemented: float32") + case reflect.Float64: + panic("abi: go117: not implemented: float64") + case reflect.Complex64: + panic("abi: go117: not implemented: complex64") + case reflect.Complex128: + panic("abi: go117: not implemented: complex128") + case reflect.Array: + panic("abi: go117: not implemented: arrays") + case reflect.Chan: + return self.valloc(p, reflect.TypeOf((chan int)(nil))) + case reflect.Func: + return self.valloc(p, reflect.TypeOf((func())(nil))) + case reflect.Map: + return self.valloc(p, reflect.TypeOf((map[int]int)(nil))) + case reflect.Ptr: + return self.valloc(p, reflect.TypeOf((*int)(nil))) + case reflect.UnsafePointer: + return self.valloc(p, ptrType) + case reflect.Interface: + return self.valloc(p, ptrType, ptrType) + case reflect.Slice: + return self.valloc(p, ptrType, intType, intType) + case reflect.String: + return self.valloc(p, ptrType, intType) + case reflect.Struct: + panic("abi: go117: not implemented: structs") + default: + panic("abi: invalid value type") + } } func (self *_StackAlloc) ralloc(p []Parameter, vts ...reflect.Type) []Parameter { - for _, vt := range vts { p = append(p, self.reg(vt)) } - return p + for _, vt := range vts { + p = append(p, self.reg(vt)) + } + return p } func (self *_StackAlloc) salloc(p []Parameter, vts ...reflect.Type) []Parameter { - for _, vt := range vts { p = append(p, self.stack(vt)) } - return p + for _, vt := range vts { + p = append(p, self.stack(vt)) + } + return p } func (self *_StackAlloc) valloc(p []Parameter, vts ...reflect.Type) []Parameter { - if self.i + len(vts) <= len(regOrder) { - return self.ralloc(p, vts...) - } else { - return self.salloc(p, vts...) - } + if self.i+len(vts) <= len(regOrder) { + return self.ralloc(p, vts...) + } else { + return self.salloc(p, vts...) + } } func (self *AMD64ABI) Reserved() map[x86_64.Register64]int32 { - return map[x86_64.Register64]int32 { - x86_64.R14: 0, // current goroutine - x86_64.R15: 1, // GOT reference - } + return map[x86_64.Register64]int32{ + x86_64.R14: 0, // current goroutine + x86_64.R15: 1, // GOT reference + } } func (self *AMD64ABI) LayoutFunc(id int, ft reflect.Type) *FunctionLayout { - var sa _StackAlloc - var fn FunctionLayout - - /* allocate the receiver if any (interface call always uses pointer) */ - if id >= 0 { - fn.Args = sa.alloc(fn.Args, ptrType) - } - - /* assign every arguments */ - for i := 0; i < ft.NumIn(); i++ { - fn.Args = sa.alloc(fn.Args, ft.In(i)) - } - - /* reset the register counter, and add a pointer alignment field */ - sa.i = 0 - sa.spill(0, PtrAlign) - - /* assign every return value */ - for i := 0; i < ft.NumOut(); i++ { - fn.Rets = sa.alloc(fn.Rets, ft.Out(i)) - } - - /* assign spill slots */ - for i := 0; i < len(fn.Args); i++ { - if fn.Args[i].InRegister { - fn.Args[i].Mem = sa.spill(PtrSize, PtrAlign) - PtrSize - } - } - - /* add the final pointer alignment field */ - fn.Id = id - fn.Sp = sa.spill(0, PtrAlign) - return &fn + var sa _StackAlloc + var fn FunctionLayout + + /* allocate the receiver if any (interface call always uses pointer) */ + if id >= 0 { + fn.Args = sa.alloc(fn.Args, ptrType) + } + + /* assign every arguments */ + for i := 0; i < ft.NumIn(); i++ { + fn.Args = sa.alloc(fn.Args, ft.In(i)) + } + + /* reset the register counter, and add a pointer alignment field */ + sa.i = 0 + sa.spill(0, PtrAlign) + + /* assign every return value */ + for i := 0; i < ft.NumOut(); i++ { + fn.Rets = sa.alloc(fn.Rets, ft.Out(i)) + } + + /* assign spill slots */ + for i := 0; i < len(fn.Args); i++ { + if fn.Args[i].InRegister { + fn.Args[i].Mem = sa.spill(PtrSize, PtrAlign) - PtrSize + } + } + + /* add the final pointer alignment field */ + fn.Id = id + fn.Sp = sa.spill(0, PtrAlign) + return &fn } diff --git a/internal/atm/emu/asm.s b/internal/atm/emu/asm.s index a8c6cc1..9c99547 100644 --- a/internal/atm/emu/asm.s +++ b/internal/atm/emu/asm.s @@ -1,4 +1,4 @@ -// Copyright 2022 ByteDance Inc. +// Copyright 2022 CloudWeGo Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/internal/atm/emu/emu.go b/internal/atm/emu/emu.go index bd225aa..c879e69 100644 --- a/internal/atm/emu/emu.go +++ b/internal/atm/emu/emu.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,186 +17,248 @@ package emu import ( - `fmt` - `math/bits` - `runtime` - `sync` - `unsafe` + "fmt" + "math/bits" + "runtime" + "sync" + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` + "github.com/cloudwego/frugal/internal/atm/hir" ) type Value struct { - U uint64 - P unsafe.Pointer + U uint64 + P unsafe.Pointer } type Emulator struct { - pc *hir.Ir - uv [6]uint64 - pv [7]unsafe.Pointer - ar [8]Value - rv [8]Value + pc *hir.Ir + uv [6]uint64 + pv [7]unsafe.Pointer + ar [8]Value + rv [8]Value } var ( - emulatorPool sync.Pool + emulatorPool sync.Pool ) func LoadProgram(p hir.Program) (e *Emulator) { - if v := emulatorPool.Get(); v == nil { - return &Emulator{pc: p.Head} - } else { - return v.(*Emulator).Reset(p) - } + if v := emulatorPool.Get(); v == nil { + return &Emulator{pc: p.Head} + } else { + return v.(*Emulator).Reset(p) + } } func bool2u64(val bool) uint64 { - if val { - return 1 - } else { - return 0 - } + if val { + return 1 + } else { + return 0 + } } func checkptr(p unsafe.Pointer) unsafe.Pointer { - if p != nil { _ = *(*uint8)(p) } - return p + if p != nil { + _ = *(*uint8)(p) + } + return p } func (self *Emulator) trap() { - println("****** DEBUGGER BREAK ******") - println("Current State:", self.String()) - runtime.Breakpoint() + println("****** DEBUGGER BREAK ******") + println("Current State:", self.String()) + runtime.Breakpoint() } func (self *Emulator) Ru(i int) uint64 { return self.rv[i].U } func (self *Emulator) Rp(i int) unsafe.Pointer { return self.rv[i].P } -func (self *Emulator) Au(i int, v uint64) *Emulator { self.ar[i].U = v; return self } +func (self *Emulator) Au(i int, v uint64) *Emulator { self.ar[i].U = v; return self } func (self *Emulator) Ap(i int, v unsafe.Pointer) *Emulator { self.ar[i].P = v; return self } func (self *Emulator) Run() { - var i uint8 - var v uint64 - var p *hir.Ir - var q *hir.Ir - - /* run until end */ - for self.pc != nil { - p, self.pc = self.pc, self.pc.Ln - self.uv[hir.Rz], self.pv[hir.Pn] = 0, nil - - /* main switch on OpCode */ - switch p.Op { - case hir.OP_nop : break - case hir.OP_ip : self.pv[p.Pd] = checkptr(p.Pr) - case hir.OP_lb : self.uv[p.Rx] = uint64(*(*uint8)(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(p.Iv)))) - case hir.OP_lw : self.uv[p.Rx] = uint64(*(*uint16)(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(p.Iv)))) - case hir.OP_ll : self.uv[p.Rx] = uint64(*(*uint32)(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(p.Iv)))) - case hir.OP_lq : self.uv[p.Rx] = *(*uint64)(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(p.Iv))) - case hir.OP_lp : self.pv[p.Pd] = checkptr(*(*unsafe.Pointer)(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(p.Iv)))) - case hir.OP_sb : *(*uint8)(unsafe.Pointer(uintptr(self.pv[p.Pd]) + uintptr(p.Iv))) = uint8(self.uv[p.Rx]) - case hir.OP_sw : *(*uint16)(unsafe.Pointer(uintptr(self.pv[p.Pd]) + uintptr(p.Iv))) = uint16(self.uv[p.Rx]) - case hir.OP_sl : *(*uint32)(unsafe.Pointer(uintptr(self.pv[p.Pd]) + uintptr(p.Iv))) = uint32(self.uv[p.Rx]) - case hir.OP_sq : *(*uint64)(unsafe.Pointer(uintptr(self.pv[p.Pd]) + uintptr(p.Iv))) = self.uv[p.Rx] - case hir.OP_sp : *(*unsafe.Pointer)(unsafe.Pointer(uintptr(self.pv[p.Pd]) + uintptr(p.Iv))) = self.pv[p.Ps] - case hir.OP_ldaq : self.uv[p.Rx] = self.ar[p.Iv].U - case hir.OP_ldap : self.pv[p.Pd] = checkptr(self.ar[p.Iv].P) - case hir.OP_addp : self.pv[p.Pd] = checkptr(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(self.uv[p.Rx]))) - case hir.OP_subp : self.pv[p.Pd] = checkptr(unsafe.Pointer(uintptr(self.pv[p.Ps]) - uintptr(self.uv[p.Rx]))) - case hir.OP_addpi : self.pv[p.Pd] = checkptr(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(p.Iv))) - case hir.OP_add : self.uv[p.Rz] = self.uv[p.Rx] + self.uv[p.Ry] - case hir.OP_sub : self.uv[p.Rz] = self.uv[p.Rx] - self.uv[p.Ry] - case hir.OP_addi : self.uv[p.Ry] = self.uv[p.Rx] + uint64(p.Iv) - case hir.OP_muli : self.uv[p.Ry] = self.uv[p.Rx] * uint64(p.Iv) - case hir.OP_andi : self.uv[p.Ry] = self.uv[p.Rx] & uint64(p.Iv) - case hir.OP_xori : self.uv[p.Ry] = self.uv[p.Rx] ^ uint64(p.Iv) - case hir.OP_shri : self.uv[p.Ry] = self.uv[p.Rx] >> p.Iv - case hir.OP_bsi : self.uv[p.Ry] = self.uv[p.Rx] | (1 << p.Iv) - case hir.OP_swapw : self.uv[p.Ry] = uint64(bits.ReverseBytes16(uint16(self.uv[p.Rx]))) - case hir.OP_swapl : self.uv[p.Ry] = uint64(bits.ReverseBytes32(uint32(self.uv[p.Rx]))) - case hir.OP_swapq : self.uv[p.Ry] = bits.ReverseBytes64(self.uv[p.Rx]) - case hir.OP_sxlq : self.uv[p.Ry] = uint64(int32(self.uv[p.Rx])) - case hir.OP_beq : if self.uv[p.Rx] == self.uv[p.Ry] { self.pc = p.Br } - case hir.OP_bne : if self.uv[p.Rx] != self.uv[p.Ry] { self.pc = p.Br } - case hir.OP_blt : if int64(self.uv[p.Rx]) < int64(self.uv[p.Ry]) { self.pc = p.Br } - case hir.OP_bltu : if self.uv[p.Rx] < self.uv[p.Ry] { self.pc = p.Br } - case hir.OP_bgeu : if self.uv[p.Rx] >= self.uv[p.Ry] { self.pc = p.Br } - case hir.OP_beqp : if self.pv[p.Ps] == self.pv[p.Pd] { self.pc = p.Br } - case hir.OP_bnep : if self.pv[p.Ps] != self.pv[p.Pd] { self.pc = p.Br } - case hir.OP_jmp : self.pc = p.Br - case hir.OP_bzero : memclrNoHeapPointers(self.pv[p.Pd], uintptr(p.Iv)) - case hir.OP_bcopy : memmove(self.pv[p.Pd], self.pv[p.Ps], uintptr(self.uv[p.Rx])) - case hir.OP_break : self.trap() - - /* call to C / Go / Go interface functions */ - case hir.OP_ccall: fallthrough - case hir.OP_gcall: fallthrough - case hir.OP_icall: hir.LookupCall(p.Iv).Call(self, p) - - /* bit test and set */ - case hir.OP_bts: { - bi := self.uv[p.Rx] - bv := self.uv[p.Ry] - self.uv[p.Ry] = bv | (1 << (bi % 64)) - self.uv[p.Rz] = bool2u64(bv & (1 << (bi % 64)) != 0) - } - - /* table switch */ - case hir.OP_bsw: { - if v = self.uv[p.Rx]; v < uint64(p.Iv) { - if q = *(**hir.Ir)(unsafe.Pointer(uintptr(p.Pr) + uintptr(v) * 8)); q != nil { - self.pc = q - } - } - } - - /* return from function */ - case hir.OP_ret: { - for i, self.pc = 0, nil; i < p.Rn; i++ { - if r := p.Rr[i]; r & hir.ArgPointer == 0 { - self.rv[i].U = self.uv[r & hir.ArgMask] - } else { - self.rv[i].P = self.pv[r & hir.ArgMask] - } - } - } - - /* illegal OpCode */ - default: { - panic(fmt.Sprintf("illegal OpCode: %#02x", p.Op)) - } - } - } -} + var i uint8 + var v uint64 + var p *hir.Ir + var q *hir.Ir + + /* run until end */ + for self.pc != nil { + p, self.pc = self.pc, self.pc.Ln + self.uv[hir.Rz], self.pv[hir.Pn] = 0, nil + + /* main switch on OpCode */ + switch p.Op { + case hir.OP_nop: + break + case hir.OP_ip: + self.pv[p.Pd] = checkptr(p.Pr) + case hir.OP_lb: + self.uv[p.Rx] = uint64(*(*uint8)(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(p.Iv)))) + case hir.OP_lw: + self.uv[p.Rx] = uint64(*(*uint16)(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(p.Iv)))) + case hir.OP_ll: + self.uv[p.Rx] = uint64(*(*uint32)(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(p.Iv)))) + case hir.OP_lq: + self.uv[p.Rx] = *(*uint64)(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(p.Iv))) + case hir.OP_lp: + self.pv[p.Pd] = checkptr(*(*unsafe.Pointer)(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(p.Iv)))) + case hir.OP_sb: + *(*uint8)(unsafe.Pointer(uintptr(self.pv[p.Pd]) + uintptr(p.Iv))) = uint8(self.uv[p.Rx]) + case hir.OP_sw: + *(*uint16)(unsafe.Pointer(uintptr(self.pv[p.Pd]) + uintptr(p.Iv))) = uint16(self.uv[p.Rx]) + case hir.OP_sl: + *(*uint32)(unsafe.Pointer(uintptr(self.pv[p.Pd]) + uintptr(p.Iv))) = uint32(self.uv[p.Rx]) + case hir.OP_sq: + *(*uint64)(unsafe.Pointer(uintptr(self.pv[p.Pd]) + uintptr(p.Iv))) = self.uv[p.Rx] + case hir.OP_sp: + *(*unsafe.Pointer)(unsafe.Pointer(uintptr(self.pv[p.Pd]) + uintptr(p.Iv))) = self.pv[p.Ps] + case hir.OP_ldaq: + self.uv[p.Rx] = self.ar[p.Iv].U + case hir.OP_ldap: + self.pv[p.Pd] = checkptr(self.ar[p.Iv].P) + case hir.OP_addp: + self.pv[p.Pd] = checkptr(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(self.uv[p.Rx]))) + case hir.OP_subp: + self.pv[p.Pd] = checkptr(unsafe.Pointer(uintptr(self.pv[p.Ps]) - uintptr(self.uv[p.Rx]))) + case hir.OP_addpi: + self.pv[p.Pd] = checkptr(unsafe.Pointer(uintptr(self.pv[p.Ps]) + uintptr(p.Iv))) + case hir.OP_add: + self.uv[p.Rz] = self.uv[p.Rx] + self.uv[p.Ry] + case hir.OP_sub: + self.uv[p.Rz] = self.uv[p.Rx] - self.uv[p.Ry] + case hir.OP_addi: + self.uv[p.Ry] = self.uv[p.Rx] + uint64(p.Iv) + case hir.OP_muli: + self.uv[p.Ry] = self.uv[p.Rx] * uint64(p.Iv) + case hir.OP_andi: + self.uv[p.Ry] = self.uv[p.Rx] & uint64(p.Iv) + case hir.OP_xori: + self.uv[p.Ry] = self.uv[p.Rx] ^ uint64(p.Iv) + case hir.OP_shri: + self.uv[p.Ry] = self.uv[p.Rx] >> p.Iv + case hir.OP_bsi: + self.uv[p.Ry] = self.uv[p.Rx] | (1 << p.Iv) + case hir.OP_swapw: + self.uv[p.Ry] = uint64(bits.ReverseBytes16(uint16(self.uv[p.Rx]))) + case hir.OP_swapl: + self.uv[p.Ry] = uint64(bits.ReverseBytes32(uint32(self.uv[p.Rx]))) + case hir.OP_swapq: + self.uv[p.Ry] = bits.ReverseBytes64(self.uv[p.Rx]) + case hir.OP_sxlq: + self.uv[p.Ry] = uint64(int32(self.uv[p.Rx])) + case hir.OP_beq: + if self.uv[p.Rx] == self.uv[p.Ry] { + self.pc = p.Br + } + case hir.OP_bne: + if self.uv[p.Rx] != self.uv[p.Ry] { + self.pc = p.Br + } + case hir.OP_blt: + if int64(self.uv[p.Rx]) < int64(self.uv[p.Ry]) { + self.pc = p.Br + } + case hir.OP_bltu: + if self.uv[p.Rx] < self.uv[p.Ry] { + self.pc = p.Br + } + case hir.OP_bgeu: + if self.uv[p.Rx] >= self.uv[p.Ry] { + self.pc = p.Br + } + case hir.OP_beqp: + if self.pv[p.Ps] == self.pv[p.Pd] { + self.pc = p.Br + } + case hir.OP_bnep: + if self.pv[p.Ps] != self.pv[p.Pd] { + self.pc = p.Br + } + case hir.OP_jmp: + self.pc = p.Br + case hir.OP_bzero: + memclrNoHeapPointers(self.pv[p.Pd], uintptr(p.Iv)) + case hir.OP_bcopy: + memmove(self.pv[p.Pd], self.pv[p.Ps], uintptr(self.uv[p.Rx])) + case hir.OP_break: + self.trap() + + /* call to C / Go / Go interface functions */ + case hir.OP_ccall: + fallthrough + case hir.OP_gcall: + fallthrough + case hir.OP_icall: + hir.LookupCall(p.Iv).Call(self, p) + /* bit test and set */ + case hir.OP_bts: + { + bi := self.uv[p.Rx] + bv := self.uv[p.Ry] + self.uv[p.Ry] = bv | (1 << (bi % 64)) + self.uv[p.Rz] = bool2u64(bv&(1<<(bi%64)) != 0) + } + + /* table switch */ + case hir.OP_bsw: + { + if v = self.uv[p.Rx]; v < uint64(p.Iv) { + if q = *(**hir.Ir)(unsafe.Pointer(uintptr(p.Pr) + uintptr(v)*8)); q != nil { + self.pc = q + } + } + } + + /* return from function */ + case hir.OP_ret: + { + for i, self.pc = 0, nil; i < p.Rn; i++ { + if r := p.Rr[i]; r&hir.ArgPointer == 0 { + self.rv[i].U = self.uv[r&hir.ArgMask] + } else { + self.rv[i].P = self.pv[r&hir.ArgMask] + } + } + } + + /* illegal OpCode */ + default: + { + panic(fmt.Sprintf("illegal OpCode: %#02x", p.Op)) + } + } + } +} func (self *Emulator) Free() { - emulatorPool.Put(self) + emulatorPool.Put(self) } func (self *Emulator) Reset(p hir.Program) *Emulator { - *self = Emulator{pc: p.Head} - return self + *self = Emulator{pc: p.Head} + return self } /** Implementation of ir.CallState **/ func (self *Emulator) Gr(id hir.GenericRegister) uint64 { - return self.uv[id] + return self.uv[id] } func (self *Emulator) Pr(id hir.PointerRegister) unsafe.Pointer { - return self.pv[id] + return self.pv[id] } func (self *Emulator) SetGr(id hir.GenericRegister, val uint64) { - self.uv[id] = val + self.uv[id] = val } func (self *Emulator) SetPr(id hir.PointerRegister, val unsafe.Pointer) { - self.pv[id] = val + self.pv[id] = val } /** State Dumping **/ @@ -220,22 +282,22 @@ const _F_emulator = `Emulator { }` func (self *Emulator) String() string { - return fmt.Sprintf( - _F_emulator, - self.pc, - self.pc.Disassemble(nil), - self.uv[0], - self.uv[1], - self.uv[2], - self.uv[3], - self.uv[4], - self.uv[5], - self.pv[0], - self.pv[1], - self.pv[2], - self.pv[3], - self.pv[4], - self.pv[5], - self.pv[6], - ) + return fmt.Sprintf( + _F_emulator, + self.pc, + self.pc.Disassemble(nil), + self.uv[0], + self.uv[1], + self.uv[2], + self.uv[3], + self.uv[4], + self.uv[5], + self.pv[0], + self.pv[1], + self.pv[2], + self.pv[3], + self.pv[4], + self.pv[5], + self.pv[6], + ) } diff --git a/internal/atm/emu/emu_test.go b/internal/atm/emu/emu_test.go index e2a78bf..ba3ac2b 100644 --- a/internal/atm/emu/emu_test.go +++ b/internal/atm/emu/emu_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,76 +17,88 @@ package emu import ( - `testing` - `unsafe` + "testing" + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/davecgh/go-spew/spew` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/davecgh/go-spew/spew" ) var ( - testfn = hir.RegisterGCall(testemu_pfunc, func(ctx hir.CallContext) { - var v0 struct {P unsafe.Pointer; L uint64} - var v1 struct {P unsafe.Pointer; L uint64} - var v2 struct {P unsafe.Pointer; L uint64} - if !ctx.Verify("*i*i*i", "*i*i") { - panic("invalid testemu_pfunc call") - } - v0.P = ctx.Ap(0) - v0.L = ctx.Au(1) - v1.P = ctx.Ap(2) - v1.L = ctx.Au(3) - v2.P = ctx.Ap(4) - v2.L = ctx.Au(5) - r0, r1 := testemu_pfunc( - *(*string)(unsafe.Pointer(&v0)), - *(*string)(unsafe.Pointer(&v1)), - *(*string)(unsafe.Pointer(&v2)), - ) - ctx.Ru(1, uint64(len(r0))) - ctx.Ru(3, uint64(len(r1))) - ctx.Rp(0, *(*unsafe.Pointer)(unsafe.Pointer(&r0))) - ctx.Rp(2, *(*unsafe.Pointer)(unsafe.Pointer(&r1))) - }) + testfn = hir.RegisterGCall(testemu_pfunc, func(ctx hir.CallContext) { + var v0 struct { + P unsafe.Pointer + L uint64 + } + var v1 struct { + P unsafe.Pointer + L uint64 + } + var v2 struct { + P unsafe.Pointer + L uint64 + } + if !ctx.Verify("*i*i*i", "*i*i") { + panic("invalid testemu_pfunc call") + } + v0.P = ctx.Ap(0) + v0.L = ctx.Au(1) + v1.P = ctx.Ap(2) + v1.L = ctx.Au(3) + v2.P = ctx.Ap(4) + v2.L = ctx.Au(5) + r0, r1 := testemu_pfunc( + *(*string)(unsafe.Pointer(&v0)), + *(*string)(unsafe.Pointer(&v1)), + *(*string)(unsafe.Pointer(&v2)), + ) + ctx.Ru(1, uint64(len(r0))) + ctx.Ru(3, uint64(len(r1))) + ctx.Rp(0, *(*unsafe.Pointer)(unsafe.Pointer(&r0))) + ctx.Rp(2, *(*unsafe.Pointer)(unsafe.Pointer(&r1))) + }) ) func testemu_pfunc(a string, b string, c string) (d string, e string) { - d = a + b - e = b + c - return + d = a + b + e = b + c + return } func runEmulator(init func(emu *Emulator), prog func(p *hir.Builder)) *Emulator { - pb := hir.CreateBuilder() - prog(pb) - emu := LoadProgram(pb.Build()) - if init != nil { - init(emu) - } - emu.Run() - return emu + pb := hir.CreateBuilder() + prog(pb) + emu := LoadProgram(pb.Build()) + if init != nil { + init(emu) + } + emu.Run() + return emu } func TestEmu_OpCode_GCALL(t *testing.T) { - a := "aaa" - b := "bbb" - c := "ccc" - emu := runEmulator(nil, func(p *hir.Builder) { - p.IP(&a, hir.P0) - p.IP(&b, hir.P1) - p.IP(&c, hir.P2) - p.LQ(hir.P0, 8, hir.R0) - p.LP(hir.P0, 0, hir.P0) - p.LQ(hir.P1, 8, hir.R1) - p.LP(hir.P1, 0, hir.P1) - p.LQ(hir.P2, 8, hir.R2) - p.LP(hir.P2, 0, hir.P2) - p.GCALL(testfn).A0(hir.P0).A1(hir.R0).A2(hir.P1).A3(hir.R1).A4(hir.P2).A5(hir.R2).R0(hir.P0).R1(hir.R0).R2(hir.P1).R3(hir.R1) - p.RET().R0(hir.P0).R1(hir.R0).R2(hir.P1).R3(hir.R1) - }) - val := [2]struct{P unsafe.Pointer; L uint64} { - {P: emu.Rp(0), L: emu.Ru(1)}, - {P: emu.Rp(2), L: emu.Ru(3)}, - } - spew.Dump(*(*[2]string)(unsafe.Pointer(&val))) + a := "aaa" + b := "bbb" + c := "ccc" + emu := runEmulator(nil, func(p *hir.Builder) { + p.IP(&a, hir.P0) + p.IP(&b, hir.P1) + p.IP(&c, hir.P2) + p.LQ(hir.P0, 8, hir.R0) + p.LP(hir.P0, 0, hir.P0) + p.LQ(hir.P1, 8, hir.R1) + p.LP(hir.P1, 0, hir.P1) + p.LQ(hir.P2, 8, hir.R2) + p.LP(hir.P2, 0, hir.P2) + p.GCALL(testfn).A0(hir.P0).A1(hir.R0).A2(hir.P1).A3(hir.R1).A4(hir.P2).A5(hir.R2).R0(hir.P0).R1(hir.R0).R2(hir.P1).R3(hir.R1) + p.RET().R0(hir.P0).R1(hir.R0).R2(hir.P1).R3(hir.R1) + }) + val := [2]struct { + P unsafe.Pointer + L uint64 + }{ + {P: emu.Rp(0), L: emu.Ru(1)}, + {P: emu.Rp(2), L: emu.Ru(3)}, + } + spew.Dump(*(*[2]string)(unsafe.Pointer(&val))) } diff --git a/internal/atm/emu/stubs.go b/internal/atm/emu/stubs.go index 5a99098..652458f 100644 --- a/internal/atm/emu/stubs.go +++ b/internal/atm/emu/stubs.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ package emu import ( - `unsafe` + "unsafe" ) //go:noescape diff --git a/internal/atm/hir/builder.go b/internal/atm/hir/builder.go index daf7b5c..d39d6da 100644 --- a/internal/atm/hir/builder.go +++ b/internal/atm/hir/builder.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,396 +17,396 @@ package hir import ( - `strconv` - `strings` - `unsafe` + "strconv" + "strings" + "unsafe" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) const ( - _LB_jump_pc = "_jump_pc_" + _LB_jump_pc = "_jump_pc_" ) type Builder struct { - i int - head *Ir - tail *Ir - refs map[string]*Ir - pends map[string][]**Ir + i int + head *Ir + tail *Ir + refs map[string]*Ir + pends map[string][]**Ir } func CreateBuilder() *Builder { - return newBuilder() + return newBuilder() } func (self *Builder) add(ins *Ir) *Ir { - self.push(ins) - return ins + self.push(ins) + return ins } func (self *Builder) jmp(p *Ir, to string) *Ir { - ok := false - lr := Likely - lb := strings.ReplaceAll(to, "{n}", strconv.Itoa(self.i)) + ok := false + lr := Likely + lb := strings.ReplaceAll(to, "{n}", strconv.Itoa(self.i)) - /* backward jumps are predicted "likely", forward "unlikely" */ - if p.Br, ok = self.refs[lb]; !ok { - lr = Unlikely - self.pends[lb] = append(self.pends[lb], &p.Br) - } + /* backward jumps are predicted "likely", forward "unlikely" */ + if p.Br, ok = self.refs[lb]; !ok { + lr = Unlikely + self.pends[lb] = append(self.pends[lb], &p.Br) + } - /* unconditional jumps are always predicted "likely" */ - if p.Op == OP_jmp { - lr = Likely - } + /* unconditional jumps are always predicted "likely" */ + if p.Op == OP_jmp { + lr = Likely + } - /* add to instruction buffer */ - p.An = uint8(lr) - return self.add(p) + /* add to instruction buffer */ + p.An = uint8(lr) + return self.add(p) } func (self *Builder) tab(p *Ir, sw []string) *Ir { - nb := len(sw) - sb := make([]*Ir, nb) + nb := len(sw) + sb := make([]*Ir, nb) - /* patch each branch */ - for i, to := range sw { - ok := false - lb := strings.ReplaceAll(to, "{n}", strconv.Itoa(self.i)) + /* patch each branch */ + for i, to := range sw { + ok := false + lb := strings.ReplaceAll(to, "{n}", strconv.Itoa(self.i)) - /* check for backward jumps */ - if lb != "" { - if sb[i], ok = self.refs[lb]; !ok { - self.pends[lb] = append(self.pends[lb], &sb[i]) - } - } - } + /* check for backward jumps */ + if lb != "" { + if sb[i], ok = self.refs[lb]; !ok { + self.pends[lb] = append(self.pends[lb], &sb[i]) + } + } + } - /* save the switch table */ - p.Iv = int64(nb) - p.Pr = (*rt.GoSlice)(unsafe.Pointer(&sb)).Ptr - return self.add(p) + /* save the switch table */ + p.Iv = int64(nb) + p.Pr = (*rt.GoSlice)(unsafe.Pointer(&sb)).Ptr + return self.add(p) } func (self *Builder) push(ins *Ir) { - if self.head == nil { - self.head, self.tail = ins, ins - } else { - self.tail.Ln, self.tail = ins, ins - } + if self.head == nil { + self.head, self.tail = ins, ins + } else { + self.tail.Ln, self.tail = ins, ins + } } func (self *Builder) rejmp(br **Ir) { - for *br != nil && (*br).Ln != nil && (*br).Op == OP_nop { - *br = (*br).Ln - } + for *br != nil && (*br).Ln != nil && (*br).Op == OP_nop { + *br = (*br).Ln + } } func (self *Builder) At(pc int) string { - return _LB_jump_pc + strconv.Itoa(pc) + return _LB_jump_pc + strconv.Itoa(pc) } func (self *Builder) Mark(pc int) { - self.i++ - self.Label(self.At(pc)) + self.i++ + self.Label(self.At(pc)) } func (self *Builder) Label(to string) { - ok := false - lb := strings.ReplaceAll(to, "{n}", strconv.Itoa(self.i)) + ok := false + lb := strings.ReplaceAll(to, "{n}", strconv.Itoa(self.i)) - /* check for duplications */ - if _, ok = self.refs[lb]; ok { - panic("label " + lb + " has already been linked") - } + /* check for duplications */ + if _, ok = self.refs[lb]; ok { + panic("label " + lb + " has already been linked") + } - /* get the pending links */ - p := self.NOP() - v := self.pends[lb] + /* get the pending links */ + p := self.NOP() + v := self.pends[lb] - /* patch all the pending jumps */ - for _, q := range v { - *q = p - } + /* patch all the pending jumps */ + for _, q := range v { + *q = p + } - /* mark the label as resolved */ - self.refs[lb] = p - delete(self.pends, lb) + /* mark the label as resolved */ + self.refs[lb] = p + delete(self.pends, lb) } func (self *Builder) Build() (r Program) { - var n int - var p *Ir - var q *Ir - - /* check for unresolved labels */ - for key := range self.pends { - panic("labels are not fully resolved: " + key) - } - - /* adjust jumps to point at actual instructions */ - for p = self.head; p != nil; p = p.Ln { - if p.IsBranch() { - if p.Op != OP_bsw { - self.rejmp(&p.Br) - } else { - for i := int64(0); i < p.Iv * 8; i += 8 { - self.rejmp((**Ir)(unsafe.Pointer(uintptr(p.Pr) + uintptr(i)))) - } - } - } - } - - /* remove NOPs at the front */ - for self.head != nil && self.head.Op == OP_nop { - self.head = self.head.Ln - } - - /* no instructions left, the program was composed entirely by NOPs */ - if self.head == nil { - self.tail = nil - return - } - - /* remove all the NOPs, there should be no jumps pointing to any NOPs */ - for p = self.head; p != nil; p, n = p.Ln, n + 1 { - for p.Ln != nil && p.Ln.Op == OP_nop { - q = p.Ln - p.Ln = q.Ln - q.Free() - } - } - - /* the Builder's life-time ends here */ - r.Head = self.head - freeBuilder(self) - return + var n int + var p *Ir + var q *Ir + + /* check for unresolved labels */ + for key := range self.pends { + panic("labels are not fully resolved: " + key) + } + + /* adjust jumps to point at actual instructions */ + for p = self.head; p != nil; p = p.Ln { + if p.IsBranch() { + if p.Op != OP_bsw { + self.rejmp(&p.Br) + } else { + for i := int64(0); i < p.Iv*8; i += 8 { + self.rejmp((**Ir)(unsafe.Pointer(uintptr(p.Pr) + uintptr(i)))) + } + } + } + } + + /* remove NOPs at the front */ + for self.head != nil && self.head.Op == OP_nop { + self.head = self.head.Ln + } + + /* no instructions left, the program was composed entirely by NOPs */ + if self.head == nil { + self.tail = nil + return + } + + /* remove all the NOPs, there should be no jumps pointing to any NOPs */ + for p = self.head; p != nil; p, n = p.Ln, n+1 { + for p.Ln != nil && p.Ln.Op == OP_nop { + q = p.Ln + p.Ln = q.Ln + q.Free() + } + } + + /* the Builder's life-time ends here */ + r.Head = self.head + freeBuilder(self) + return } func (self *Builder) Append(p *Ir) (r *Ir) { - self.push(p) - r = self.head - freeBuilder(self) - return + self.push(p) + r = self.head + freeBuilder(self) + return } func (self *Builder) NOP() *Ir { - return self.add(newInstr(OP_nop)) + return self.add(newInstr(OP_nop)) } func (self *Builder) IB(v int8, rx GenericRegister) *Ir { - return self.ADDI(Rz, int64(v), rx) + return self.ADDI(Rz, int64(v), rx) } func (self *Builder) IW(v int16, rx GenericRegister) *Ir { - return self.ADDI(Rz, int64(v), rx) + return self.ADDI(Rz, int64(v), rx) } func (self *Builder) IL(v int32, rx GenericRegister) *Ir { - return self.ADDI(Rz, int64(v), rx) + return self.ADDI(Rz, int64(v), rx) } func (self *Builder) IQ(v int64, rx GenericRegister) *Ir { - return self.ADDI(Rz, v, rx) + return self.ADDI(Rz, v, rx) } func (self *Builder) IP(v interface{}, pd PointerRegister) *Ir { - if vv := rt.UnpackEface(v); !rt.IsPtr(vv.Type) { - panic("v is not a pointer") - } else { - return self.add(newInstr(OP_ip).pr(vv.Value).pd(pd)) - } + if vv := rt.UnpackEface(v); !rt.IsPtr(vv.Type) { + panic("v is not a pointer") + } else { + return self.add(newInstr(OP_ip).pr(vv.Value).pd(pd)) + } } func (self *Builder) LB(ps PointerRegister, disp int64, rx GenericRegister) *Ir { - return self.add(newInstr(OP_lb).ps(ps).iv(disp).rx(rx)) + return self.add(newInstr(OP_lb).ps(ps).iv(disp).rx(rx)) } func (self *Builder) LW(ps PointerRegister, disp int64, rx GenericRegister) *Ir { - return self.add(newInstr(OP_lw).ps(ps).iv(disp).rx(rx)) + return self.add(newInstr(OP_lw).ps(ps).iv(disp).rx(rx)) } func (self *Builder) LL(ps PointerRegister, disp int64, rx GenericRegister) *Ir { - return self.add(newInstr(OP_ll).ps(ps).iv(disp).rx(rx)) + return self.add(newInstr(OP_ll).ps(ps).iv(disp).rx(rx)) } func (self *Builder) LQ(ps PointerRegister, disp int64, rx GenericRegister) *Ir { - return self.add(newInstr(OP_lq).ps(ps).iv(disp).rx(rx)) + return self.add(newInstr(OP_lq).ps(ps).iv(disp).rx(rx)) } func (self *Builder) LP(ps PointerRegister, disp int64, pd PointerRegister) *Ir { - return self.add(newInstr(OP_lp).ps(ps).iv(disp).pd(pd)) + return self.add(newInstr(OP_lp).ps(ps).iv(disp).pd(pd)) } func (self *Builder) SB(rx GenericRegister, pd PointerRegister, disp int64) *Ir { - return self.add(newInstr(OP_sb).rx(rx).pd(pd).iv(disp)) + return self.add(newInstr(OP_sb).rx(rx).pd(pd).iv(disp)) } func (self *Builder) SW(rx GenericRegister, pd PointerRegister, disp int64) *Ir { - return self.add(newInstr(OP_sw).rx(rx).pd(pd).iv(disp)) + return self.add(newInstr(OP_sw).rx(rx).pd(pd).iv(disp)) } func (self *Builder) SL(rx GenericRegister, pd PointerRegister, disp int64) *Ir { - return self.add(newInstr(OP_sl).rx(rx).pd(pd).iv(disp)) + return self.add(newInstr(OP_sl).rx(rx).pd(pd).iv(disp)) } func (self *Builder) SQ(rx GenericRegister, pd PointerRegister, disp int64) *Ir { - return self.add(newInstr(OP_sq).rx(rx).pd(pd).iv(disp)) + return self.add(newInstr(OP_sq).rx(rx).pd(pd).iv(disp)) } func (self *Builder) SP(ps PointerRegister, pd PointerRegister, disp int64) *Ir { - return self.add(newInstr(OP_sp).ps(ps).pd(pd).iv(disp)) + return self.add(newInstr(OP_sp).ps(ps).pd(pd).iv(disp)) } func (self *Builder) MOV(rx GenericRegister, ry GenericRegister) *Ir { - return self.ADD(rx, Rz, ry) + return self.ADD(rx, Rz, ry) } func (self *Builder) MOVP(ps PointerRegister, pd PointerRegister) *Ir { - return self.ADDP(ps, Rz, pd) + return self.ADDP(ps, Rz, pd) } func (self *Builder) LDAQ(id int, rx GenericRegister) *Ir { - return self.add(newInstr(OP_ldaq).iv(int64(id)).rx(rx)) + return self.add(newInstr(OP_ldaq).iv(int64(id)).rx(rx)) } func (self *Builder) LDAP(id int, pd PointerRegister) *Ir { - return self.add(newInstr(OP_ldap).iv(int64(id)).pd(pd)) + return self.add(newInstr(OP_ldap).iv(int64(id)).pd(pd)) } func (self *Builder) ADDP(ps PointerRegister, rx GenericRegister, pd PointerRegister) *Ir { - return self.add(newInstr(OP_addp).ps(ps).rx(rx).pd(pd)) + return self.add(newInstr(OP_addp).ps(ps).rx(rx).pd(pd)) } func (self *Builder) SUBP(ps PointerRegister, rx GenericRegister, pd PointerRegister) *Ir { - return self.add(newInstr(OP_subp).ps(ps).rx(rx).pd(pd)) + return self.add(newInstr(OP_subp).ps(ps).rx(rx).pd(pd)) } func (self *Builder) ADDPI(ps PointerRegister, im int64, pd PointerRegister) *Ir { - return self.add(newInstr(OP_addpi).ps(ps).iv(im).pd(pd)) + return self.add(newInstr(OP_addpi).ps(ps).iv(im).pd(pd)) } func (self *Builder) SUBPI(ps PointerRegister, im int64, pd PointerRegister) *Ir { - return self.ADDPI(ps, -im, pd) + return self.ADDPI(ps, -im, pd) } func (self *Builder) ADD(rx GenericRegister, ry GenericRegister, rz GenericRegister) *Ir { - return self.add(newInstr(OP_add).rx(rx).ry(ry).rz(rz)) + return self.add(newInstr(OP_add).rx(rx).ry(ry).rz(rz)) } func (self *Builder) SUB(rx GenericRegister, ry GenericRegister, rz GenericRegister) *Ir { - return self.add(newInstr(OP_sub).rx(rx).ry(ry).rz(rz)) + return self.add(newInstr(OP_sub).rx(rx).ry(ry).rz(rz)) } func (self *Builder) BTS(rx GenericRegister, ry GenericRegister, rz GenericRegister) *Ir { - return self.add(newInstr(OP_bts).rx(rx).ry(ry).rz(rz)) + return self.add(newInstr(OP_bts).rx(rx).ry(ry).rz(rz)) } func (self *Builder) ADDI(rx GenericRegister, im int64, ry GenericRegister) *Ir { - return self.add(newInstr(OP_addi).rx(rx).iv(im).ry(ry)) + return self.add(newInstr(OP_addi).rx(rx).iv(im).ry(ry)) } func (self *Builder) SUBI(rx GenericRegister, im int64, ry GenericRegister) *Ir { - return self.ADDI(rx, -im, ry) + return self.ADDI(rx, -im, ry) } func (self *Builder) MULI(rx GenericRegister, im int64, ry GenericRegister) *Ir { - return self.add(newInstr(OP_muli).rx(rx).iv(im).ry(ry)) + return self.add(newInstr(OP_muli).rx(rx).iv(im).ry(ry)) } func (self *Builder) ANDI(rx GenericRegister, im int64, ry GenericRegister) *Ir { - return self.add(newInstr(OP_andi).rx(rx).iv(im).ry(ry)) + return self.add(newInstr(OP_andi).rx(rx).iv(im).ry(ry)) } func (self *Builder) XORI(rx GenericRegister, im int64, ry GenericRegister) *Ir { - return self.add(newInstr(OP_xori).rx(rx).iv(im).ry(ry)) + return self.add(newInstr(OP_xori).rx(rx).iv(im).ry(ry)) } func (self *Builder) SHRI(rx GenericRegister, im int64, ry GenericRegister) *Ir { - return self.add(newInstr(OP_shri).rx(rx).iv(im).ry(ry)) + return self.add(newInstr(OP_shri).rx(rx).iv(im).ry(ry)) } func (self *Builder) BSI(rx GenericRegister, im int64, ry GenericRegister) *Ir { - return self.add(newInstr(OP_bsi).rx(rx).iv(im).ry(ry)) + return self.add(newInstr(OP_bsi).rx(rx).iv(im).ry(ry)) } func (self *Builder) SWAPW(rx GenericRegister, ry GenericRegister) *Ir { - return self.add(newInstr(OP_swapw).rx(rx).ry(ry)) + return self.add(newInstr(OP_swapw).rx(rx).ry(ry)) } func (self *Builder) SWAPL(rx GenericRegister, ry GenericRegister) *Ir { - return self.add(newInstr(OP_swapl).rx(rx).ry(ry)) + return self.add(newInstr(OP_swapl).rx(rx).ry(ry)) } func (self *Builder) SWAPQ(rx GenericRegister, ry GenericRegister) *Ir { - return self.add(newInstr(OP_swapq).rx(rx).ry(ry)) + return self.add(newInstr(OP_swapq).rx(rx).ry(ry)) } func (self *Builder) SXLQ(rx GenericRegister, ry GenericRegister) *Ir { - return self.add(newInstr(OP_sxlq).rx(rx).ry(ry)) + return self.add(newInstr(OP_sxlq).rx(rx).ry(ry)) } func (self *Builder) BEQ(rx GenericRegister, ry GenericRegister, to string) *Ir { - return self.jmp(newInstr(OP_beq).rx(rx).ry(ry), to) + return self.jmp(newInstr(OP_beq).rx(rx).ry(ry), to) } func (self *Builder) BNE(rx GenericRegister, ry GenericRegister, to string) *Ir { - return self.jmp(newInstr(OP_bne).rx(rx).ry(ry), to) + return self.jmp(newInstr(OP_bne).rx(rx).ry(ry), to) } func (self *Builder) BLT(rx GenericRegister, ry GenericRegister, to string) *Ir { - return self.jmp(newInstr(OP_blt).rx(rx).ry(ry), to) + return self.jmp(newInstr(OP_blt).rx(rx).ry(ry), to) } func (self *Builder) BLTU(rx GenericRegister, ry GenericRegister, to string) *Ir { - return self.jmp(newInstr(OP_bltu).rx(rx).ry(ry), to) + return self.jmp(newInstr(OP_bltu).rx(rx).ry(ry), to) } func (self *Builder) BGEU(rx GenericRegister, ry GenericRegister, to string) *Ir { - return self.jmp(newInstr(OP_bgeu).rx(rx).ry(ry), to) + return self.jmp(newInstr(OP_bgeu).rx(rx).ry(ry), to) } func (self *Builder) BSW(rx GenericRegister, sw []string) *Ir { - return self.tab(newInstr(OP_bsw).rx(rx), sw) + return self.tab(newInstr(OP_bsw).rx(rx), sw) } func (self *Builder) BEQP(ps PointerRegister, pd PointerRegister, to string) *Ir { - return self.jmp(newInstr(OP_beqp).ps(ps).pd(pd), to) + return self.jmp(newInstr(OP_beqp).ps(ps).pd(pd), to) } func (self *Builder) BNEP(ps PointerRegister, pd PointerRegister, to string) *Ir { - return self.jmp(newInstr(OP_bnep).ps(ps).pd(pd), to) + return self.jmp(newInstr(OP_bnep).ps(ps).pd(pd), to) } func (self *Builder) JMP(to string) *Ir { - return self.jmp(newInstr(OP_jmp), to) + return self.jmp(newInstr(OP_jmp), to) } func (self *Builder) BZERO(nb int64, pd PointerRegister) *Ir { - return self.add(newInstr(OP_bzero).iv(nb).pd(pd)) + return self.add(newInstr(OP_bzero).iv(nb).pd(pd)) } func (self *Builder) BCOPY(ps PointerRegister, rx GenericRegister, pd PointerRegister) *Ir { - return self.add(newInstr(OP_bcopy).ps(ps).rx(rx).pd(pd)) + return self.add(newInstr(OP_bcopy).ps(ps).rx(rx).pd(pd)) } func (self *Builder) CCALL(fn *CallHandle) *Ir { - return self.add(newInstr(OP_ccall).iv(int64(fn.Id))) + return self.add(newInstr(OP_ccall).iv(int64(fn.Id))) } func (self *Builder) GCALL(fn *CallHandle) *Ir { - return self.add(newInstr(OP_gcall).iv(int64(fn.Id))) + return self.add(newInstr(OP_gcall).iv(int64(fn.Id))) } func (self *Builder) ICALL(vt PointerRegister, vp PointerRegister, mt *CallHandle) *Ir { - return self.add(newInstr(OP_icall).ps(vt).pd(vp).iv(int64(mt.Id))) + return self.add(newInstr(OP_icall).ps(vt).pd(vp).iv(int64(mt.Id))) } func (self *Builder) RET() *Ir { - return self.add(newInstr(OP_ret)) + return self.add(newInstr(OP_ret)) } func (self *Builder) BREAK() *Ir { - return self.add(newInstr(OP_break)) + return self.add(newInstr(OP_break)) } diff --git a/internal/atm/hir/call.go b/internal/atm/hir/call.go index d8f38b1..a10f701 100644 --- a/internal/atm/hir/call.go +++ b/internal/atm/hir/call.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,206 +17,213 @@ package hir import ( - `fmt` - `runtime` - `sync` - `unsafe` + "fmt" + "runtime" + "sync" + "unsafe" - `github.com/cloudwego/frugal/internal/atm/abi` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/abi" + "github.com/cloudwego/frugal/internal/rt" ) type ( - CallType uint8 + CallType uint8 ) const ( - CCall CallType = iota - GCall - ICall + CCall CallType = iota + GCall + ICall ) type CallState interface { - Gr(id GenericRegister) uint64 - Pr(id PointerRegister) unsafe.Pointer - SetGr(id GenericRegister, val uint64) - SetPr(id PointerRegister, val unsafe.Pointer) + Gr(id GenericRegister) uint64 + Pr(id PointerRegister) unsafe.Pointer + SetGr(id GenericRegister, val uint64) + SetPr(id PointerRegister, val unsafe.Pointer) } type CallHandle struct { - Id int - Slot int - Type CallType - Func unsafe.Pointer - proxy func(CallContext) + Id int + Slot int + Type CallType + Func unsafe.Pointer + proxy func(CallContext) } func (self *CallHandle) Name() string { - return runtime.FuncForPC(uintptr(self.Func)).Name() + return runtime.FuncForPC(uintptr(self.Func)).Name() } func (self *CallHandle) Call(r CallState, p *Ir) { - self.proxy(CallContext { - repo: r, - kind: self.Type, - argc: p.An, - retc: p.Rn, - argv: p.Ar, - retv: p.Rr, - itab: p.Ps, - data: p.Pd, - }) + self.proxy(CallContext{ + repo: r, + kind: self.Type, + argc: p.An, + retc: p.Rn, + argv: p.Ar, + retv: p.Rr, + itab: p.Ps, + data: p.Pd, + }) } func (self *CallHandle) String() string { - return fmt.Sprintf("*%#x[%s]", self.Func, self.Name()) + return fmt.Sprintf("*%#x[%s]", self.Func, self.Name()) } type CallContext struct { - kind CallType - repo CallState - itab PointerRegister - data PointerRegister - argc uint8 - retc uint8 - argv [8]uint8 - retv [8]uint8 + kind CallType + repo CallState + itab PointerRegister + data PointerRegister + argc uint8 + retc uint8 + argv [8]uint8 + retv [8]uint8 } func (self CallContext) Au(i int) uint64 { - if p := self.argv[i]; p &ArgPointer != 0 { - panic("invoke: invalid int argument") - } else { - return self.repo.Gr(GenericRegister(p & ArgMask)) - } + if p := self.argv[i]; p&ArgPointer != 0 { + panic("invoke: invalid int argument") + } else { + return self.repo.Gr(GenericRegister(p & ArgMask)) + } } func (self CallContext) Ap(i int) unsafe.Pointer { - if p := self.argv[i]; p &ArgPointer == 0 { - panic("invoke: invalid pointer argument") - } else { - return self.repo.Pr(PointerRegister(p & ArgMask)) - } + if p := self.argv[i]; p&ArgPointer == 0 { + panic("invoke: invalid pointer argument") + } else { + return self.repo.Pr(PointerRegister(p & ArgMask)) + } } func (self CallContext) Ru(i int, v uint64) { - if p := self.retv[i]; p &ArgPointer != 0 { - panic("invoke: invalid int return value") - } else { - self.repo.SetGr(GenericRegister(p &ArgMask), v) - } + if p := self.retv[i]; p&ArgPointer != 0 { + panic("invoke: invalid int return value") + } else { + self.repo.SetGr(GenericRegister(p&ArgMask), v) + } } func (self CallContext) Rp(i int, v unsafe.Pointer) { - if p := self.retv[i]; p &ArgPointer == 0 { - panic("invoke: invalid pointer return value") - } else { - self.repo.SetPr(PointerRegister(p &ArgMask), v) - } + if p := self.retv[i]; p&ArgPointer == 0 { + panic("invoke: invalid pointer return value") + } else { + self.repo.SetPr(PointerRegister(p&ArgMask), v) + } } func (self CallContext) Itab() *rt.GoItab { - if self.kind != ICall { - panic("invoke: itab is not available") - } else { - return (*rt.GoItab)(self.repo.Pr(self.itab)) - } + if self.kind != ICall { + panic("invoke: itab is not available") + } else { + return (*rt.GoItab)(self.repo.Pr(self.itab)) + } } func (self CallContext) Data() unsafe.Pointer { - if self.kind != ICall { - panic("invoke: data is not available") - } else { - return self.repo.Pr(self.data) - } + if self.kind != ICall { + panic("invoke: data is not available") + } else { + return self.repo.Pr(self.data) + } } func (self CallContext) Verify(args string, rets string) bool { - return self.verifySeq(args, self.argc, self.argv) && self.verifySeq(rets, self.retc, self.retv) + return self.verifySeq(args, self.argc, self.argv) && self.verifySeq(rets, self.retc, self.retv) } func (self CallContext) verifySeq(s string, n uint8, v [8]uint8) bool { - nb := int(n) - ne := len(s) - - /* sanity check */ - if ne > len(v) { - panic("invoke: invalid descriptor") - } - - /* check for value count */ - if nb != ne { - return false - } - - /* check for every argument */ - for i := 0; i < nb; i++ { - switch s[i] { - case 'i' : if v[i] &ArgPointer != 0 { return false } - case '*' : if v[i] &ArgPointer == 0 { return false } - default : panic("invoke: invalid descriptor char: " + s[i:i + 1]) - } - } - - /* all checked ok */ - return true + nb := int(n) + ne := len(s) + + /* sanity check */ + if ne > len(v) { + panic("invoke: invalid descriptor") + } + + /* check for value count */ + if nb != ne { + return false + } + + /* check for every argument */ + for i := 0; i < nb; i++ { + switch s[i] { + case 'i': + if v[i]&ArgPointer != 0 { + return false + } + case '*': + if v[i]&ArgPointer == 0 { + return false + } + default: + panic("invoke: invalid descriptor char: " + s[i:i+1]) + } + } + + /* all checked ok */ + return true } type callHandleManager struct { - m sync.RWMutex - handles []*CallHandle + m sync.RWMutex + handles []*CallHandle } func (self *callHandleManager) NewWithID() *CallHandle { - self.m.Lock() - h := new(CallHandle) - h.Id = len(self.handles) - self.handles = append(self.handles, h) - self.m.Unlock() - return h + self.m.Lock() + h := new(CallHandle) + h.Id = len(self.handles) + self.handles = append(self.handles, h) + self.m.Unlock() + return h } func (self *callHandleManager) Get(id int) (h *CallHandle) { - self.m.RLock() - if id >= 0 || id < len(self.handles) { - h = self.handles[id] - } - self.m.RUnlock() - return + self.m.RLock() + if id >= 0 || id < len(self.handles) { + h = self.handles[id] + } + self.m.RUnlock() + return } var ( - funcTab = &callHandleManager{} + funcTab = &callHandleManager{} ) func LookupCall(id int64) *CallHandle { - h := funcTab.Get(int(id)) - if h == nil { - panic("invalid function ID") - } - return h + h := funcTab.Get(int(id)) + if h == nil { + panic("invalid function ID") + } + return h } func RegisterICall(mt rt.Method, proxy func(CallContext)) (h *CallHandle) { - h = funcTab.NewWithID() - h.Type = ICall - h.Slot = abi.ABI.RegisterMethod(h.Id, mt) - h.proxy = proxy - return + h = funcTab.NewWithID() + h.Type = ICall + h.Slot = abi.ABI.RegisterMethod(h.Id, mt) + h.proxy = proxy + return } func RegisterGCall(fn interface{}, proxy func(CallContext)) (h *CallHandle) { - h = funcTab.NewWithID() - h.Type = GCall - h.Func = abi.ABI.RegisterFunction(h.Id, fn) - h.proxy = proxy - return + h = funcTab.NewWithID() + h.Type = GCall + h.Func = abi.ABI.RegisterFunction(h.Id, fn) + h.proxy = proxy + return } func RegisterCCall(fn unsafe.Pointer, proxy func(CallContext)) (h *CallHandle) { - h = funcTab.NewWithID() - h.Type = CCall - h.Func = fn - h.proxy = proxy - return + h = funcTab.NewWithID() + h.Type = CCall + h.Func = fn + h.proxy = proxy + return } diff --git a/internal/atm/hir/ir.go b/internal/atm/hir/ir.go index e893c14..6038d62 100644 --- a/internal/atm/hir/ir.go +++ b/internal/atm/hir/ir.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,97 +17,97 @@ package hir import ( - `fmt` - `strings` - `unsafe` + "fmt" + "strings" + "unsafe" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) type ( OpCode byte - Constness byte - Likeliness byte + Constness byte + Likeliness byte ) const ( - OP_nop OpCode = iota // no operation - OP_ip // ptr(Pr) -> Pd - OP_lb // u64(*(* u8)(Ps + Iv)) -> Rx - OP_lw // u64(*(*u16)(Ps + Iv)) -> Rx - OP_ll // u64(*(*u32)(Ps + Iv)) -> Rx - OP_lq // *(*u64)(Ps + Iv) -> Rx - OP_lp // *(*ptr)(Ps + Iv) -> Pd - OP_sb // u8(Rx) -> *(*u8)(Pd + Iv) - OP_sw // u16(Rx) -> *(*u16)(Pd + Iv) - OP_sl // u32(Rx) -> *(*u32)(Pd + Iv) - OP_sq // Rx -> *(*u64)(Pd + Iv) - OP_sp // Ps -> *(*ptr)(Pd + Iv) - OP_ldaq // arg[Iv] -> Rx - OP_ldap // arg[Iv] -> Pd - OP_addp // Ps + Rx -> Pd - OP_subp // Ps - Rx -> Pd - OP_addpi // Ps + Iv -> Pd - OP_add // Rx + Ry -> Rz - OP_sub // Rx - Ry -> Rz - OP_bts // Ry & (1 << (Rx % PTR_BITS)) != 0 -> Rz, Ry |= 1 << (Rx % PTR_BITS) - OP_addi // Rx + Iv -> Ry - OP_muli // Rx * Iv -> Ry - OP_andi // Rx & Iv -> Ry - OP_xori // Rx ^ Iv -> Ry - OP_shri // Rx >> Iv -> Ry - OP_bsi // Rx | (1 << Iv) -> Ry - OP_swapw // bswap16(Rx) -> Ry - OP_swapl // bswap32(Rx) -> Ry - OP_swapq // bswap64(Rx) -> Ry - OP_sxlq // sign_extend_32_to_64(Rx) -> Ry - OP_beq // if (Rx == Ry) Br.PC -> PC - OP_bne // if (Rx != Ry) Br.PC -> PC - OP_blt // if (Rx < Ry) Br.PC -> PC - OP_bltu // if (u(Rx) < u(Ry)) Br.PC -> PC - OP_bgeu // if (u(Rx) >= u(Ry)) Br.PC -> PC - OP_beqp // if (Ps == Pd) Br.PC -> PC - OP_bnep // if (Ps == Pd) Br.PC -> PC - OP_bsw // if (u(Rx) < u(An)) Sw[u(Rx)].PC -> PC - OP_jmp // Br.PC -> PC - OP_bzero // memset(Pd, 0, Iv) - OP_bcopy // memcpy(Pd, Ps, Rx) - OP_ccall // call external C functions - OP_gcall // call external Go functions - OP_icall // call external Go iface methods - OP_ret // return from function - OP_break // trigger a debugger breakpoint + OP_nop OpCode = iota // no operation + OP_ip // ptr(Pr) -> Pd + OP_lb // u64(*(* u8)(Ps + Iv)) -> Rx + OP_lw // u64(*(*u16)(Ps + Iv)) -> Rx + OP_ll // u64(*(*u32)(Ps + Iv)) -> Rx + OP_lq // *(*u64)(Ps + Iv) -> Rx + OP_lp // *(*ptr)(Ps + Iv) -> Pd + OP_sb // u8(Rx) -> *(*u8)(Pd + Iv) + OP_sw // u16(Rx) -> *(*u16)(Pd + Iv) + OP_sl // u32(Rx) -> *(*u32)(Pd + Iv) + OP_sq // Rx -> *(*u64)(Pd + Iv) + OP_sp // Ps -> *(*ptr)(Pd + Iv) + OP_ldaq // arg[Iv] -> Rx + OP_ldap // arg[Iv] -> Pd + OP_addp // Ps + Rx -> Pd + OP_subp // Ps - Rx -> Pd + OP_addpi // Ps + Iv -> Pd + OP_add // Rx + Ry -> Rz + OP_sub // Rx - Ry -> Rz + OP_bts // Ry & (1 << (Rx % PTR_BITS)) != 0 -> Rz, Ry |= 1 << (Rx % PTR_BITS) + OP_addi // Rx + Iv -> Ry + OP_muli // Rx * Iv -> Ry + OP_andi // Rx & Iv -> Ry + OP_xori // Rx ^ Iv -> Ry + OP_shri // Rx >> Iv -> Ry + OP_bsi // Rx | (1 << Iv) -> Ry + OP_swapw // bswap16(Rx) -> Ry + OP_swapl // bswap32(Rx) -> Ry + OP_swapq // bswap64(Rx) -> Ry + OP_sxlq // sign_extend_32_to_64(Rx) -> Ry + OP_beq // if (Rx == Ry) Br.PC -> PC + OP_bne // if (Rx != Ry) Br.PC -> PC + OP_blt // if (Rx < Ry) Br.PC -> PC + OP_bltu // if (u(Rx) < u(Ry)) Br.PC -> PC + OP_bgeu // if (u(Rx) >= u(Ry)) Br.PC -> PC + OP_beqp // if (Ps == Pd) Br.PC -> PC + OP_bnep // if (Ps == Pd) Br.PC -> PC + OP_bsw // if (u(Rx) < u(An)) Sw[u(Rx)].PC -> PC + OP_jmp // Br.PC -> PC + OP_bzero // memset(Pd, 0, Iv) + OP_bcopy // memcpy(Pd, Ps, Rx) + OP_ccall // call external C functions + OP_gcall // call external Go functions + OP_icall // call external Go iface methods + OP_ret // return from function + OP_break // trigger a debugger breakpoint ) const ( - Const Constness = iota - Volatile + Const Constness = iota + Volatile ) const ( - Likely Likeliness = iota - Unlikely + Likely Likeliness = iota + Unlikely ) type Ir struct { - Op OpCode - Rx GenericRegister - Ry GenericRegister - Rz GenericRegister - Ps PointerRegister - Pd PointerRegister - An uint8 - Rn uint8 - Ar [8]uint8 - Rr [8]uint8 - Iv int64 - Pr unsafe.Pointer - Br *Ir - Ln *Ir + Op OpCode + Rx GenericRegister + Ry GenericRegister + Rz GenericRegister + Ps PointerRegister + Pd PointerRegister + An uint8 + Rn uint8 + Ar [8]uint8 + Rr [8]uint8 + Iv int64 + Pr unsafe.Pointer + Br *Ir + Ln *Ir } -func (self *Ir) iv(v int64) *Ir { self.Iv = v; return self } -func (self *Ir) pr(v unsafe.Pointer) *Ir { self.Pr = v; return self } +func (self *Ir) iv(v int64) *Ir { self.Iv = v; return self } +func (self *Ir) pr(v unsafe.Pointer) *Ir { self.Pr = v; return self } func (self *Ir) rx(v GenericRegister) *Ir { self.Rx = v; return self } func (self *Ir) ry(v GenericRegister) *Ir { self.Ry = v; return self } func (self *Ir) rz(v GenericRegister) *Ir { self.Rz = v; return self } @@ -115,23 +115,23 @@ func (self *Ir) ps(v PointerRegister) *Ir { self.Ps = v; return self } func (self *Ir) pd(v PointerRegister) *Ir { self.Pd = v; return self } func (self *Ir) constness(c Constness) *Ir { - if self.Op != OP_ip { - panic("constness only applicable to `OP_ip` instruction") - } else { - self.An = uint8(c) - return self - } + if self.Op != OP_ip { + panic("constness only applicable to `OP_ip` instruction") + } else { + self.An = uint8(c) + return self + } } func (self *Ir) likeliness(p Likeliness) *Ir { - if !self.IsBranch() { - panic("likeliness only applicable to branch instructions") - } else if self.Op == OP_bsw { - panic("cannot specify likeliness for `OP_bsw`") - } else { - self.An = uint8(p) - return self - } + if !self.IsBranch() { + panic("likeliness only applicable to branch instructions") + } else if self.Op == OP_bsw { + panic("cannot specify likeliness for `OP_bsw`") + } else { + self.An = uint8(p) + return self + } } func (self *Ir) A0(v Register) *Ir { self.An, self.Ar[0] = 1, v.A(); return self } @@ -152,152 +152,199 @@ func (self *Ir) R5(v Register) *Ir { self.Rn, self.Rr[5] = 6, v.A(); return self func (self *Ir) R6(v Register) *Ir { self.Rn, self.Rr[6] = 7, v.A(); return self } func (self *Ir) R7(v Register) *Ir { self.Rn, self.Rr[7] = 8, v.A(); return self } -func (self *Ir) Const() *Ir { return self.constness(Const) } +func (self *Ir) Const() *Ir { return self.constness(Const) } func (self *Ir) Volatile() *Ir { return self.constness(Volatile) } -func (self *Ir) Likely() *Ir { return self.likeliness(Likely) } +func (self *Ir) Likely() *Ir { return self.likeliness(Likely) } func (self *Ir) Unlikely() *Ir { return self.likeliness(Unlikely) } func (self *Ir) Free() { - freeInstr(self) + freeInstr(self) } func (self *Ir) Switch() (p []*Ir) { - (*rt.GoSlice)(unsafe.Pointer(&p)).Ptr = self.Pr - (*rt.GoSlice)(unsafe.Pointer(&p)).Len = int(self.Iv) - (*rt.GoSlice)(unsafe.Pointer(&p)).Cap = int(self.Iv) - return + (*rt.GoSlice)(unsafe.Pointer(&p)).Ptr = self.Pr + (*rt.GoSlice)(unsafe.Pointer(&p)).Len = int(self.Iv) + (*rt.GoSlice)(unsafe.Pointer(&p)).Cap = int(self.Iv) + return } func (self *Ir) IsBranch() bool { - return self.Op >= OP_beq && self.Op <= OP_jmp + return self.Op >= OP_beq && self.Op <= OP_jmp } func (self *Ir) Constness() Constness { - if self.Op != OP_ip { - panic("constness only applicable to `OP_ip` instruction") - } else { - return Constness(self.An) - } + if self.Op != OP_ip { + panic("constness only applicable to `OP_ip` instruction") + } else { + return Constness(self.An) + } } func (self *Ir) Likeliness() Likeliness { - if !self.IsBranch() { - panic("likeliness only applicable to branch instructions") - } else if self.Op == OP_bsw { - panic("`OP_bsw` does not have likeliness assigned") - } else { - return Likeliness(self.An) - } + if !self.IsBranch() { + panic("likeliness only applicable to branch instructions") + } else if self.Op == OP_bsw { + panic("`OP_bsw` does not have likeliness assigned") + } else { + return Likeliness(self.An) + } } func (self *Ir) formatCall() string { - return fmt.Sprintf( - "%s, %s", - self.formatArgs(&self.Ar, self.An), - self.formatArgs(&self.Rr, self.Rn), - ) + return fmt.Sprintf( + "%s, %s", + self.formatArgs(&self.Ar, self.An), + self.formatArgs(&self.Rr, self.Rn), + ) } func (self *Ir) formatArgs(vv *[8]uint8, nb uint8) string { - i := uint8(0) - ret := make([]string, nb) + i := uint8(0) + ret := make([]string, nb) - /* add each register */ - for i = 0; i < nb; i++ { - if v := vv[i]; (v & ArgPointer) == 0 { - ret[i] = "%" + GenericRegister(v & ArgMask).String() - } else { - ret[i] = "%" + PointerRegister(v & ArgMask).String() - } - } + /* add each register */ + for i = 0; i < nb; i++ { + if v := vv[i]; (v & ArgPointer) == 0 { + ret[i] = "%" + GenericRegister(v&ArgMask).String() + } else { + ret[i] = "%" + PointerRegister(v&ArgMask).String() + } + } - /* compose the result */ - return fmt.Sprintf( - "{%s}", - strings.Join(ret, ", "), - ) + /* compose the result */ + return fmt.Sprintf( + "{%s}", + strings.Join(ret, ", "), + ) } func (self *Ir) formatRefs(refs map[*Ir]string, v *Ir) string { - if vv, ok := refs[v]; ok { - return vv - } else { - return fmt.Sprintf("@%p", v) - } + if vv, ok := refs[v]; ok { + return vv + } else { + return fmt.Sprintf("@%p", v) + } } func (self *Ir) formatTable(refs map[*Ir]string) string { - tab := self.Switch() - ret := make([]string, 0, self.Iv) + tab := self.Switch() + ret := make([]string, 0, self.Iv) - /* empty table */ - if self.Iv == 0 { - return "{}" - } + /* empty table */ + if self.Iv == 0 { + return "{}" + } - /* format every label */ - for i, lb := range tab { - if lb != nil { - ret = append(ret, fmt.Sprintf("%4ccase $%d: %s,\n", ' ', i, self.formatRefs(refs, lb))) - } - } + /* format every label */ + for i, lb := range tab { + if lb != nil { + ret = append(ret, fmt.Sprintf("%4ccase $%d: %s,\n", ' ', i, self.formatRefs(refs, lb))) + } + } - /* join them together */ - return fmt.Sprintf( - "{\n%s}", - strings.Join(ret, ""), - ) + /* join them together */ + return fmt.Sprintf( + "{\n%s}", + strings.Join(ret, ""), + ) } func (self *Ir) Disassemble(refs map[*Ir]string) string { - switch self.Op { - case OP_nop : return "nop" - case OP_ip : return fmt.Sprintf("ip $%p, %%%s", self.Pr, self.Pd) - case OP_lb : return fmt.Sprintf("lb %d(%%%s), %%%s", self.Iv, self.Ps, self.Rx) - case OP_lw : return fmt.Sprintf("lw %d(%%%s), %%%s", self.Iv, self.Ps, self.Rx) - case OP_ll : return fmt.Sprintf("ll %d(%%%s), %%%s", self.Iv, self.Ps, self.Rx) - case OP_lq : return fmt.Sprintf("lq %d(%%%s), %%%s", self.Iv, self.Ps, self.Rx) - case OP_lp : return fmt.Sprintf("lp %d(%%%s), %%%s", self.Iv, self.Ps, self.Pd) - case OP_sb : return fmt.Sprintf("sb %%%s, %d(%%%s)", self.Rx, self.Iv, self.Pd) - case OP_sw : return fmt.Sprintf("sw %%%s, %d(%%%s)", self.Rx, self.Iv, self.Pd) - case OP_sl : return fmt.Sprintf("sl %%%s, %d(%%%s)", self.Rx, self.Iv, self.Pd) - case OP_sq : return fmt.Sprintf("sq %%%s, %d(%%%s)", self.Rx, self.Iv, self.Pd) - case OP_sp : return fmt.Sprintf("sp %%%s, %d(%%%s)", self.Ps, self.Iv, self.Pd) - case OP_ldaq : return fmt.Sprintf("ldaq $%d, %%%s", self.Iv, self.Rx) - case OP_ldap : return fmt.Sprintf("ldap $%d, %%%s", self.Iv, self.Pd) - case OP_addp : return fmt.Sprintf("addp %%%s, %%%s, %%%s", self.Ps, self.Rx, self.Pd) - case OP_subp : return fmt.Sprintf("subp %%%s, %%%s, %%%s", self.Ps, self.Rx, self.Pd) - case OP_addpi : return fmt.Sprintf("addpi %%%s, $%d, %%%s", self.Ps, self.Iv, self.Pd) - case OP_add : return fmt.Sprintf("add %%%s, %%%s, %%%s", self.Rx, self.Ry, self.Rz) - case OP_sub : return fmt.Sprintf("sub %%%s, %%%s, %%%s", self.Rx, self.Ry, self.Rz) - case OP_bts : return fmt.Sprintf("bts %%%s, %%%s, %%%s", self.Rx, self.Ry, self.Rz) - case OP_addi : return fmt.Sprintf("addi %%%s, $%d, %%%s", self.Rx, self.Iv, self.Ry) - case OP_muli : return fmt.Sprintf("muli %%%s, $%d, %%%s", self.Rx, self.Iv, self.Ry) - case OP_andi : return fmt.Sprintf("andi %%%s, $%d, %%%s", self.Rx, self.Iv, self.Ry) - case OP_xori : return fmt.Sprintf("xori %%%s, $%d, %%%s", self.Rx, self.Iv, self.Ry) - case OP_shri : return fmt.Sprintf("shri %%%s, $%d, %%%s", self.Rx, self.Iv, self.Ry) - case OP_bsi : return fmt.Sprintf("bsi %%%s, $%d, %%%s", self.Rx, self.Iv, self.Ry) - case OP_swapw : return fmt.Sprintf("swapw %%%s, %%%s", self.Rx, self.Ry) - case OP_swapl : return fmt.Sprintf("swapl %%%s, %%%s", self.Rx, self.Ry) - case OP_swapq : return fmt.Sprintf("swapq %%%s, %%%s", self.Rx, self.Ry) - case OP_sxlq : return fmt.Sprintf("sxlq %%%s, %%%s", self.Rx, self.Ry) - case OP_beq : return fmt.Sprintf("beq %%%s, %%%s, %s", self.Rx, self.Ry, self.formatRefs(refs, self.Br)) - case OP_bne : return fmt.Sprintf("bne %%%s, %%%s, %s", self.Rx, self.Ry, self.formatRefs(refs, self.Br)) - case OP_blt : return fmt.Sprintf("blt %%%s, %%%s, %s", self.Rx, self.Ry, self.formatRefs(refs, self.Br)) - case OP_bltu : return fmt.Sprintf("bltu %%%s, %%%s, %s", self.Rx, self.Ry, self.formatRefs(refs, self.Br)) - case OP_bgeu : return fmt.Sprintf("bgeu %%%s, %%%s, %s", self.Rx, self.Ry, self.formatRefs(refs, self.Br)) - case OP_beqp : return fmt.Sprintf("beq %%%s, %%%s, %s", self.Ps, self.Pd, self.formatRefs(refs, self.Br)) - case OP_bnep : return fmt.Sprintf("bne %%%s, %%%s, %s", self.Ps, self.Pd, self.formatRefs(refs, self.Br)) - case OP_bsw : return fmt.Sprintf("bsw %%%s, %s", self.Rx, self.formatTable(refs)) - case OP_jmp : return fmt.Sprintf("jmp %s", self.formatRefs(refs, self.Br)) - case OP_bzero : return fmt.Sprintf("bzero $%d, %s", self.Iv, self.Pd) - case OP_bcopy : return fmt.Sprintf("bcopy %%%s, %%%s, %%%s", self.Ps, self.Rx, self.Pd) - case OP_ccall : return fmt.Sprintf("ccall %s, %s", LookupCall(self.Iv), self.formatCall()) - case OP_gcall : return fmt.Sprintf("gcall %s, %s", LookupCall(self.Iv), self.formatCall()) - case OP_icall : return fmt.Sprintf("icall $%d, {%%%s, %%%s}, %s", LookupCall(self.Iv).Slot, self.Ps, self.Pd, self.formatCall()) - case OP_ret : return fmt.Sprintf("ret %s", self.formatArgs(&self.Rr, self.Rn)) - case OP_break : return "break" - default : panic(fmt.Sprintf("invalid OpCode: 0x%02x", self.Op)) - } + switch self.Op { + case OP_nop: + return "nop" + case OP_ip: + return fmt.Sprintf("ip $%p, %%%s", self.Pr, self.Pd) + case OP_lb: + return fmt.Sprintf("lb %d(%%%s), %%%s", self.Iv, self.Ps, self.Rx) + case OP_lw: + return fmt.Sprintf("lw %d(%%%s), %%%s", self.Iv, self.Ps, self.Rx) + case OP_ll: + return fmt.Sprintf("ll %d(%%%s), %%%s", self.Iv, self.Ps, self.Rx) + case OP_lq: + return fmt.Sprintf("lq %d(%%%s), %%%s", self.Iv, self.Ps, self.Rx) + case OP_lp: + return fmt.Sprintf("lp %d(%%%s), %%%s", self.Iv, self.Ps, self.Pd) + case OP_sb: + return fmt.Sprintf("sb %%%s, %d(%%%s)", self.Rx, self.Iv, self.Pd) + case OP_sw: + return fmt.Sprintf("sw %%%s, %d(%%%s)", self.Rx, self.Iv, self.Pd) + case OP_sl: + return fmt.Sprintf("sl %%%s, %d(%%%s)", self.Rx, self.Iv, self.Pd) + case OP_sq: + return fmt.Sprintf("sq %%%s, %d(%%%s)", self.Rx, self.Iv, self.Pd) + case OP_sp: + return fmt.Sprintf("sp %%%s, %d(%%%s)", self.Ps, self.Iv, self.Pd) + case OP_ldaq: + return fmt.Sprintf("ldaq $%d, %%%s", self.Iv, self.Rx) + case OP_ldap: + return fmt.Sprintf("ldap $%d, %%%s", self.Iv, self.Pd) + case OP_addp: + return fmt.Sprintf("addp %%%s, %%%s, %%%s", self.Ps, self.Rx, self.Pd) + case OP_subp: + return fmt.Sprintf("subp %%%s, %%%s, %%%s", self.Ps, self.Rx, self.Pd) + case OP_addpi: + return fmt.Sprintf("addpi %%%s, $%d, %%%s", self.Ps, self.Iv, self.Pd) + case OP_add: + return fmt.Sprintf("add %%%s, %%%s, %%%s", self.Rx, self.Ry, self.Rz) + case OP_sub: + return fmt.Sprintf("sub %%%s, %%%s, %%%s", self.Rx, self.Ry, self.Rz) + case OP_bts: + return fmt.Sprintf("bts %%%s, %%%s, %%%s", self.Rx, self.Ry, self.Rz) + case OP_addi: + return fmt.Sprintf("addi %%%s, $%d, %%%s", self.Rx, self.Iv, self.Ry) + case OP_muli: + return fmt.Sprintf("muli %%%s, $%d, %%%s", self.Rx, self.Iv, self.Ry) + case OP_andi: + return fmt.Sprintf("andi %%%s, $%d, %%%s", self.Rx, self.Iv, self.Ry) + case OP_xori: + return fmt.Sprintf("xori %%%s, $%d, %%%s", self.Rx, self.Iv, self.Ry) + case OP_shri: + return fmt.Sprintf("shri %%%s, $%d, %%%s", self.Rx, self.Iv, self.Ry) + case OP_bsi: + return fmt.Sprintf("bsi %%%s, $%d, %%%s", self.Rx, self.Iv, self.Ry) + case OP_swapw: + return fmt.Sprintf("swapw %%%s, %%%s", self.Rx, self.Ry) + case OP_swapl: + return fmt.Sprintf("swapl %%%s, %%%s", self.Rx, self.Ry) + case OP_swapq: + return fmt.Sprintf("swapq %%%s, %%%s", self.Rx, self.Ry) + case OP_sxlq: + return fmt.Sprintf("sxlq %%%s, %%%s", self.Rx, self.Ry) + case OP_beq: + return fmt.Sprintf("beq %%%s, %%%s, %s", self.Rx, self.Ry, self.formatRefs(refs, self.Br)) + case OP_bne: + return fmt.Sprintf("bne %%%s, %%%s, %s", self.Rx, self.Ry, self.formatRefs(refs, self.Br)) + case OP_blt: + return fmt.Sprintf("blt %%%s, %%%s, %s", self.Rx, self.Ry, self.formatRefs(refs, self.Br)) + case OP_bltu: + return fmt.Sprintf("bltu %%%s, %%%s, %s", self.Rx, self.Ry, self.formatRefs(refs, self.Br)) + case OP_bgeu: + return fmt.Sprintf("bgeu %%%s, %%%s, %s", self.Rx, self.Ry, self.formatRefs(refs, self.Br)) + case OP_beqp: + return fmt.Sprintf("beq %%%s, %%%s, %s", self.Ps, self.Pd, self.formatRefs(refs, self.Br)) + case OP_bnep: + return fmt.Sprintf("bne %%%s, %%%s, %s", self.Ps, self.Pd, self.formatRefs(refs, self.Br)) + case OP_bsw: + return fmt.Sprintf("bsw %%%s, %s", self.Rx, self.formatTable(refs)) + case OP_jmp: + return fmt.Sprintf("jmp %s", self.formatRefs(refs, self.Br)) + case OP_bzero: + return fmt.Sprintf("bzero $%d, %s", self.Iv, self.Pd) + case OP_bcopy: + return fmt.Sprintf("bcopy %%%s, %%%s, %%%s", self.Ps, self.Rx, self.Pd) + case OP_ccall: + return fmt.Sprintf("ccall %s, %s", LookupCall(self.Iv), self.formatCall()) + case OP_gcall: + return fmt.Sprintf("gcall %s, %s", LookupCall(self.Iv), self.formatCall()) + case OP_icall: + return fmt.Sprintf("icall $%d, {%%%s, %%%s}, %s", LookupCall(self.Iv).Slot, self.Ps, self.Pd, self.formatCall()) + case OP_ret: + return fmt.Sprintf("ret %s", self.formatArgs(&self.Rr, self.Rn)) + case OP_break: + return "break" + default: + panic(fmt.Sprintf("invalid OpCode: 0x%02x", self.Op)) + } } diff --git a/internal/atm/hir/pools.go b/internal/atm/hir/pools.go index 2cefe99..48ca29a 100644 --- a/internal/atm/hir/pools.go +++ b/internal/atm/hir/pools.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,63 +17,63 @@ package hir import ( - `sync` + "sync" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) var ( - instrPool sync.Pool - builderPool sync.Pool + instrPool sync.Pool + builderPool sync.Pool ) func newInstr(op OpCode) *Ir { - if v := instrPool.Get(); v == nil { - return allocInstr(op) - } else { - return resetInstr(op, v.(*Ir)) - } + if v := instrPool.Get(); v == nil { + return allocInstr(op) + } else { + return resetInstr(op, v.(*Ir)) + } } func freeInstr(p *Ir) { - instrPool.Put(p) + instrPool.Put(p) } func allocInstr(op OpCode) (p *Ir) { - p = new(Ir) - p.Op = op - return + p = new(Ir) + p.Op = op + return } func resetInstr(op OpCode, p *Ir) *Ir { - *p = Ir{Op: op} - return p + *p = Ir{Op: op} + return p } func newBuilder() *Builder { - if v := builderPool.Get(); v == nil { - return allocBuilder() - } else { - return resetBuilder(v.(*Builder)) - } + if v := builderPool.Get(); v == nil { + return allocBuilder() + } else { + return resetBuilder(v.(*Builder)) + } } func freeBuilder(p *Builder) { - builderPool.Put(p) + builderPool.Put(p) } func allocBuilder() (p *Builder) { - p = new(Builder) - p.refs = make(map[string]*Ir, 64) - p.pends = make(map[string][]**Ir, 64) - return + p = new(Builder) + p.refs = make(map[string]*Ir, 64) + p.pends = make(map[string][]**Ir, 64) + return } func resetBuilder(p *Builder) *Builder { - p.i = 0 - p.head = nil - p.tail = nil - rt.MapClear(p.refs) - rt.MapClear(p.pends) - return p + p.i = 0 + p.head = nil + p.tail = nil + rt.MapClear(p.refs) + rt.MapClear(p.pends) + return p } diff --git a/internal/atm/hir/prog.go b/internal/atm/hir/prog.go index 9a17631..a36e379 100644 --- a/internal/atm/hir/prog.go +++ b/internal/atm/hir/prog.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,60 +17,60 @@ package hir import ( - `fmt` - `strings` + "fmt" + "strings" ) type Program struct { - Head *Ir + Head *Ir } func (self Program) Free() { - for p, q := self.Head, self.Head; p != nil; p = q { - q = p.Ln - p.Free() - } + for p, q := self.Head, self.Head; p != nil; p = q { + q = p.Ln + p.Free() + } } func (self Program) Disassemble() string { - ret := make([]string, 0, 64) - ref := make(map[*Ir]string) + ret := make([]string, 0, 64) + ref := make(map[*Ir]string) - /* scan all the branch target */ - for p := self.Head; p != nil; p = p.Ln { - if p.IsBranch() { - if p.Op != OP_bsw { - if _, ok := ref[p.Br]; !ok { - ref[p.Br] = fmt.Sprintf("L_%d", len(ref)) - } - } else { - for _, lb := range p.Switch() { - if lb != nil { - if _, ok := ref[lb]; !ok { - ref[lb] = fmt.Sprintf("L_%d", len(ref)) - } - } - } - } - } - } + /* scan all the branch target */ + for p := self.Head; p != nil; p = p.Ln { + if p.IsBranch() { + if p.Op != OP_bsw { + if _, ok := ref[p.Br]; !ok { + ref[p.Br] = fmt.Sprintf("L_%d", len(ref)) + } + } else { + for _, lb := range p.Switch() { + if lb != nil { + if _, ok := ref[lb]; !ok { + ref[lb] = fmt.Sprintf("L_%d", len(ref)) + } + } + } + } + } + } - /* dump all the instructions */ - for p := self.Head; p != nil; p = p.Ln { - var ok bool - var vv string + /* dump all the instructions */ + for p := self.Head; p != nil; p = p.Ln { + var ok bool + var vv string - /* check for label reference */ - if vv, ok = ref[p]; ok { - ret = append(ret, vv + ":") - } + /* check for label reference */ + if vv, ok = ref[p]; ok { + ret = append(ret, vv+":") + } - /* indent each line */ - for _, ln := range strings.Split(p.Disassemble(ref), "\n") { - ret = append(ret, " " + ln) - } - } + /* indent each line */ + for _, ln := range strings.Split(p.Disassemble(ref), "\n") { + ret = append(ret, " "+ln) + } + } - /* join them together */ - return strings.Join(ret, "\n") + /* join them together */ + return strings.Join(ret, "\n") } diff --git a/internal/atm/hir/regs.go b/internal/atm/hir/regs.go index 95ce29e..1cda6d0 100644 --- a/internal/atm/hir/regs.go +++ b/internal/atm/hir/regs.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,62 +17,62 @@ package hir import ( - `fmt` + "fmt" ) type Register interface { - fmt.Stringer - Z() bool - A() uint8 + fmt.Stringer + Z() bool + A() uint8 } type ( - GenericRegister uint8 - PointerRegister uint8 + GenericRegister uint8 + PointerRegister uint8 ) const ( - ArgMask = 0x7f - ArgGeneric = 0x00 - ArgPointer = 0x80 + ArgMask = 0x7f + ArgGeneric = 0x00 + ArgPointer = 0x80 ) const ( - R0 GenericRegister = iota - R1 - R2 - R3 - R4 - Rz + R0 GenericRegister = iota + R1 + R2 + R3 + R4 + Rz ) const ( - P0 PointerRegister = iota - P1 - P2 - P3 - P4 - P5 - Pn + P0 PointerRegister = iota + P1 + P2 + P3 + P4 + P5 + Pn ) -var GenericRegisters = map[GenericRegister]string { - R0: "r0", - R1: "r1", - R2: "r2", - R3: "r3", - R4: "r4", - Rz: "z", +var GenericRegisters = map[GenericRegister]string{ + R0: "r0", + R1: "r1", + R2: "r2", + R3: "r3", + R4: "r4", + Rz: "z", } -var PointerRegisters = map[PointerRegister]string { - P0: "p0", - P1: "p1", - P2: "p2", - P3: "p3", - P4: "p4", - P5: "p5", - Pn: "nil", +var PointerRegisters = map[PointerRegister]string{ + P0: "p0", + P1: "p1", + P2: "p2", + P3: "p3", + P4: "p4", + P5: "p5", + Pn: "nil", } func (self GenericRegister) Z() bool { return self == Rz } @@ -82,17 +82,17 @@ func (self GenericRegister) A() uint8 { return uint8(self) | ArgGeneric } func (self PointerRegister) A() uint8 { return uint8(self) | ArgPointer } func (self GenericRegister) String() string { - if v := GenericRegisters[self]; v == "" { - panic(fmt.Sprintf("invalid generic register: 0x%02x", uint8(self))) - } else { - return v - } + if v := GenericRegisters[self]; v == "" { + panic(fmt.Sprintf("invalid generic register: 0x%02x", uint8(self))) + } else { + return v + } } func (self PointerRegister) String() string { - if v := PointerRegisters[self]; v == "" { - panic(fmt.Sprintf("invalid pointer register: 0x%02x", uint8(self))) - } else { - return v - } + if v := PointerRegisters[self]; v == "" { + panic(fmt.Sprintf("invalid pointer register: 0x%02x", uint8(self))) + } else { + return v + } } diff --git a/internal/atm/pgen/asm.s b/internal/atm/pgen/asm.s index a8c6cc1..9c99547 100644 --- a/internal/atm/pgen/asm.s +++ b/internal/atm/pgen/asm.s @@ -1,4 +1,4 @@ -// Copyright 2022 ByteDance Inc. +// Copyright 2022 CloudWeGo Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/internal/atm/pgen/iasm_amd64.go b/internal/atm/pgen/iasm_amd64.go index bf19e48..9a751b8 100644 --- a/internal/atm/pgen/iasm_amd64.go +++ b/internal/atm/pgen/iasm_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,82 +17,82 @@ package pgen import ( - `math` + "math" - `github.com/cloudwego/iasm/x86_64` + "github.com/cloudwego/iasm/x86_64" ) const ( - AL = x86_64.AL - AX = x86_64.AX - EAX = x86_64.EAX - RAX = x86_64.RAX - RCX = x86_64.RCX - RDX = x86_64.RDX - RBX = x86_64.RBX - RSP = x86_64.RSP - RBP = x86_64.RBP - RSI = x86_64.RSI - RDI = x86_64.RDI - R8 = x86_64.R8 - R9 = x86_64.R9 - R10 = x86_64.R10 - R11 = x86_64.R11 - R12 = x86_64.R12 - R13 = x86_64.R13 - R14 = x86_64.R14 - R15 = x86_64.R15 - XMM15 = x86_64.XMM15 + AL = x86_64.AL + AX = x86_64.AX + EAX = x86_64.EAX + RAX = x86_64.RAX + RCX = x86_64.RCX + RDX = x86_64.RDX + RBX = x86_64.RBX + RSP = x86_64.RSP + RBP = x86_64.RBP + RSI = x86_64.RSI + RDI = x86_64.RDI + R8 = x86_64.R8 + R9 = x86_64.R9 + R10 = x86_64.R10 + R11 = x86_64.R11 + R12 = x86_64.R12 + R13 = x86_64.R13 + R14 = x86_64.R14 + R15 = x86_64.R15 + XMM15 = x86_64.XMM15 ) -var allocationOrder = [11]x86_64.Register64 { - R12, R13, R14, R15, // reserved registers first (except for RBX) - R10, R11, // then scratch registers - R9, R8, RCX, RDX, // then argument registers in reverse order (RDI, RSI and RAX are always free) - RBX, // finally the RBX, we put RBX here to reduce collision with Go register ABI +var allocationOrder = [11]x86_64.Register64{ + R12, R13, R14, R15, // reserved registers first (except for RBX) + R10, R11, // then scratch registers + R9, R8, RCX, RDX, // then argument registers in reverse order (RDI, RSI and RAX are always free) + RBX, // finally the RBX, we put RBX here to reduce collision with Go register ABI } func Abs(disp int32) *x86_64.MemoryOperand { - return x86_64.Abs(disp) + return x86_64.Abs(disp) } func Ptr(base x86_64.Register, disp int32) *x86_64.MemoryOperand { - return x86_64.Ptr(base, disp) + return x86_64.Ptr(base, disp) } func Sib(base x86_64.Register, index x86_64.Register64, scale uint8, disp int32) *x86_64.MemoryOperand { - return x86_64.Sib(base, index, scale, disp) + return x86_64.Sib(base, index, scale, disp) } func isPow2(v int64) bool { - return v & (v - 1) == 0 + return v&(v-1) == 0 } func isInt32(v int64) bool { - return v >= math.MinInt32 && v <= math.MaxInt32 + return v >= math.MinInt32 && v <= math.MaxInt32 } func isReg64(v x86_64.Register) (ok bool) { - _, ok = v.(x86_64.Register64) - return + _, ok = v.(x86_64.Register64) + return } func toAddress(p *x86_64.Label) uintptr { - if v, err := p.Evaluate(); err != nil { - panic(err) - } else { - return uintptr(v) - } + if v, err := p.Evaluate(); err != nil { + panic(err) + } else { + return uintptr(v) + } } func isSimpleMem(v *x86_64.MemoryOperand) bool { - return !v.Masked && - v.Broadcast == 0 && - v.Addr.Type == x86_64.Memory && - v.Addr.Offset == 0 && - v.Addr.Reference == nil && - v.Addr.Memory.Scale <= 1 && - v.Addr.Memory.Index == nil && - v.Addr.Memory.Displacement == 0 && - isReg64(v.Addr.Memory.Base) + return !v.Masked && + v.Broadcast == 0 && + v.Addr.Type == x86_64.Memory && + v.Addr.Offset == 0 && + v.Addr.Reference == nil && + v.Addr.Memory.Scale <= 1 && + v.Addr.Memory.Index == nil && + v.Addr.Memory.Displacement == 0 && + isReg64(v.Addr.Memory.Base) } diff --git a/internal/atm/pgen/pgen_abi_amd64.go b/internal/atm/pgen/pgen_abi_amd64.go index 9eedd81..e164ef1 100644 --- a/internal/atm/pgen/pgen_abi_amd64.go +++ b/internal/atm/pgen/pgen_abi_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,454 +17,454 @@ package pgen import ( - `fmt` - `math` - `unsafe` - - `github.com/cloudwego/iasm/x86_64` - `github.com/cloudwego/frugal/internal/atm/abi` - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/atm/rtx` - `github.com/cloudwego/frugal/internal/rt` + "fmt" + "math" + "unsafe" + + "github.com/cloudwego/frugal/internal/atm/abi" + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/atm/rtx" + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/iasm/x86_64" ) type _SwapPair struct { - rs hir.Register - rd hir.Register - rr x86_64.Register64 + rs hir.Register + rd hir.Register + rr x86_64.Register64 } type _CodeGenExtension struct { - rets []_SwapPair + rets []_SwapPair } /** Prologue & Epilogue **/ func (self *CodeGen) abiPrologue(p *x86_64.Program) { - for i, v := range self.ctxt.desc.Args { - if v.InRegister { - p.MOVQ(v.Reg, self.ctxt.argv(i)) - } - } + for i, v := range self.ctxt.desc.Args { + if v.InRegister { + p.MOVQ(v.Reg, self.ctxt.argv(i)) + } + } } func (self *CodeGen) abiEpilogue(p *x86_64.Program) { - for _, v := range self.abix.rets { - p.XCHGQ(self.r(v.rs), v.rr) - self.regs[v.rs], self.regs[v.rd] = self.regs[v.rd], self.regs[v.rs] - } + for _, v := range self.abix.rets { + p.XCHGQ(self.r(v.rs), v.rr) + self.regs[v.rs], self.regs[v.rd] = self.regs[v.rd], self.regs[v.rs] + } } /** Stack Growing **/ func (self *CodeGen) abiStackGrow(p *x86_64.Program) { - self.internalSpillArgs(p) - p.MOVQ(uintptr(rtx.F_morestack_noctxt), R12) - p.CALLQ(R12) - self.internalUnspillArgs(p) + self.internalSpillArgs(p) + p.MOVQ(uintptr(rtx.F_morestack_noctxt), R12) + p.CALLQ(R12) + self.internalUnspillArgs(p) } func (self *CodeGen) internalSpillArgs(p *x86_64.Program) { - for _, v := range self.ctxt.desc.Args { - if v.InRegister { - p.MOVQ(v.Reg, Ptr(RSP, int32(v.Mem) +abi.PtrSize)) - } - } + for _, v := range self.ctxt.desc.Args { + if v.InRegister { + p.MOVQ(v.Reg, Ptr(RSP, int32(v.Mem)+abi.PtrSize)) + } + } } func (self *CodeGen) internalUnspillArgs(p *x86_64.Program) { - for _, v := range self.ctxt.desc.Args { - if v.InRegister { - p.MOVQ(Ptr(RSP, int32(v.Mem) +abi.PtrSize), v.Reg) - } - } + for _, v := range self.ctxt.desc.Args { + if v.InRegister { + p.MOVQ(Ptr(RSP, int32(v.Mem)+abi.PtrSize), v.Reg) + } + } } /** Reserved Register Management **/ func (self *CodeGen) abiSaveReserved(p *x86_64.Program) { - for rr := range self.ctxt.regr { - p.MOVQ(rr, self.ctxt.rslot(rr)) - } + for rr := range self.ctxt.regr { + p.MOVQ(rr, self.ctxt.rslot(rr)) + } } func (self *CodeGen) abiLoadReserved(p *x86_64.Program) { - for rr := range self.ctxt.regr { - p.MOVQ(self.ctxt.rslot(rr), rr) - } + for rr := range self.ctxt.regr { + p.MOVQ(self.ctxt.rslot(rr), rr) + } } func (self *CodeGen) abiSpillReserved(p *x86_64.Program) { - for rr := range self.ctxt.regr { - if lr := self.rindex(rr); lr != nil { - p.MOVQ(rr, self.ctxt.slot(lr)) - } - } + for rr := range self.ctxt.regr { + if lr := self.rindex(rr); lr != nil { + p.MOVQ(rr, self.ctxt.slot(lr)) + } + } } func (self *CodeGen) abiRestoreReserved(p *x86_64.Program) { - for rr := range self.ctxt.regr { - if lr := self.rindex(rr); lr != nil { - p.MOVQ(self.ctxt.slot(lr), rr) - } - } + for rr := range self.ctxt.regr { + if lr := self.rindex(rr); lr != nil { + p.MOVQ(self.ctxt.slot(lr), rr) + } + } } /** Argument & Return Value Management **/ func (self *CodeGen) abiLoadInt(p *x86_64.Program, i int, d hir.GenericRegister) { - p.MOVQ(self.ctxt.argv(i), self.r(d)) + p.MOVQ(self.ctxt.argv(i), self.r(d)) } func (self *CodeGen) abiLoadPtr(p *x86_64.Program, i int, d hir.PointerRegister) { - p.MOVQ(self.ctxt.argv(i), self.r(d)) + p.MOVQ(self.ctxt.argv(i), self.r(d)) } func (self *CodeGen) abiStoreInt(p *x86_64.Program, s hir.GenericRegister, i int) { - self.internalStoreRet(p, s, i) + self.internalStoreRet(p, s, i) } func (self *CodeGen) abiStorePtr(p *x86_64.Program, s hir.PointerRegister, i int) { - self.internalStoreRet(p, s, i) + self.internalStoreRet(p, s, i) } func (self *CodeGen) internalStoreRet(p *x86_64.Program, s hir.Register, i int) { - var r hir.Register - var m abi.Parameter - - /* if return with stack, store directly */ - if m = self.ctxt.desc.Rets[i]; !m.InRegister { - p.MOVQ(self.r(s), self.ctxt.retv(i)) - return - } - - /* check if the value is the very register required for return */ - if self.r(s) == m.Reg { - return - } - - /* if return with free registers, simply overwrite with new value */ - if r = self.rindex(m.Reg); r == nil { - p.MOVQ(self.r(s), m.Reg) - return - } - - /* if not, mark the register to store later */ - self.abix.rets = append(self.abix.rets, _SwapPair { - rs: s, - rd: r, - rr: m.Reg, - }) + var r hir.Register + var m abi.Parameter + + /* if return with stack, store directly */ + if m = self.ctxt.desc.Rets[i]; !m.InRegister { + p.MOVQ(self.r(s), self.ctxt.retv(i)) + return + } + + /* check if the value is the very register required for return */ + if self.r(s) == m.Reg { + return + } + + /* if return with free registers, simply overwrite with new value */ + if r = self.rindex(m.Reg); r == nil { + p.MOVQ(self.r(s), m.Reg) + return + } + + /* if not, mark the register to store later */ + self.abix.rets = append(self.abix.rets, _SwapPair{ + rs: s, + rd: r, + rr: m.Reg, + }) } /** Memory Zeroing **/ func (self *CodeGen) abiBlockZero(p *x86_64.Program, pd hir.PointerRegister, nb int64) { - var dp int32 - var rd x86_64.Register64 - - /* check for block size */ - if nb <= 0 || nb > math.MaxInt32 { - panic("abiBlockZero: invalid block size") - } - - /* use XMM for larger blocks */ - if nb >= 16 { - p.PXOR(XMM15, XMM15) - } - - /* use loops to reduce the code length */ - if rd = self.r(pd); nb >= 128 { - r := x86_64.CreateLabel("loop") - t := x86_64.CreateLabel("begin") - - /* setup the zeroing loop, use 8x loop for more efficient pipelining */ - p.MOVQ (rd, RDI) - p.MOVL (nb / 128, EAX) - p.JMP (t) - p.Link (r) - p.ADDQ (128, RDI) - p.Link (t) - - /* generate the zeroing instructions */ - for i := int32(0); i < 8; i++ { - p.MOVDQU(XMM15, Ptr(RDI, i * 16)) - } - - /* decrease & check loop counter */ - p.SUBL (1, EAX) - p.JNZ (r) - - /* replace the register */ - rd = RDI - nb %= 128 - } - - /* clear every 16-byte block */ - for nb >= 16 { - p.MOVDQU(XMM15, Ptr(rd, dp)) - dp += 16 - nb -= 16 - } - - /* only 1 byte left */ - if nb == 1 { - p.MOVB(0, Ptr(rd, dp)) - return - } - - /* still bytes need to be zeroed */ - if nb != 0 { - p.XORL(EAX, EAX) - } - - /* clear every 8-byte block */ - if nb >= 8 { - p.MOVQ(RAX, Ptr(rd, dp)) - dp += 8 - nb -= 8 - } - - /* clear every 4-byte block */ - if nb >= 8 { - p.MOVL(EAX, Ptr(rd, dp)) - dp += 4 - nb -= 4 - } - - /* clear every 2-byte block */ - if nb >= 2 { - p.MOVW(AX, Ptr(rd, dp)) - dp += 2 - nb -= 2 - } - - /* last byte */ - if nb > 0 { - p.MOVB(AL, Ptr(rd, dp)) - } + var dp int32 + var rd x86_64.Register64 + + /* check for block size */ + if nb <= 0 || nb > math.MaxInt32 { + panic("abiBlockZero: invalid block size") + } + + /* use XMM for larger blocks */ + if nb >= 16 { + p.PXOR(XMM15, XMM15) + } + + /* use loops to reduce the code length */ + if rd = self.r(pd); nb >= 128 { + r := x86_64.CreateLabel("loop") + t := x86_64.CreateLabel("begin") + + /* setup the zeroing loop, use 8x loop for more efficient pipelining */ + p.MOVQ(rd, RDI) + p.MOVL(nb/128, EAX) + p.JMP(t) + p.Link(r) + p.ADDQ(128, RDI) + p.Link(t) + + /* generate the zeroing instructions */ + for i := int32(0); i < 8; i++ { + p.MOVDQU(XMM15, Ptr(RDI, i*16)) + } + + /* decrease & check loop counter */ + p.SUBL(1, EAX) + p.JNZ(r) + + /* replace the register */ + rd = RDI + nb %= 128 + } + + /* clear every 16-byte block */ + for nb >= 16 { + p.MOVDQU(XMM15, Ptr(rd, dp)) + dp += 16 + nb -= 16 + } + + /* only 1 byte left */ + if nb == 1 { + p.MOVB(0, Ptr(rd, dp)) + return + } + + /* still bytes need to be zeroed */ + if nb != 0 { + p.XORL(EAX, EAX) + } + + /* clear every 8-byte block */ + if nb >= 8 { + p.MOVQ(RAX, Ptr(rd, dp)) + dp += 8 + nb -= 8 + } + + /* clear every 4-byte block */ + if nb >= 8 { + p.MOVL(EAX, Ptr(rd, dp)) + dp += 4 + nb -= 4 + } + + /* clear every 2-byte block */ + if nb >= 2 { + p.MOVW(AX, Ptr(rd, dp)) + dp += 2 + nb -= 2 + } + + /* last byte */ + if nb > 0 { + p.MOVB(AL, Ptr(rd, dp)) + } } /** Function & Method Call **/ -var argumentOrder = [6]x86_64.Register64 { - RDI, - RSI, - RDX, - RCX, - R8, - R9, +var argumentOrder = [6]x86_64.Register64{ + RDI, + RSI, + RDX, + RCX, + R8, + R9, } -var argumentRegisters = map[x86_64.Register64]bool { - RDI : true, - RSI : true, - RDX : true, - RCX : true, - R8 : true, - R9 : true, +var argumentRegisters = map[x86_64.Register64]bool{ + RDI: true, + RSI: true, + RDX: true, + RCX: true, + R8: true, + R9: true, } -var reservedRegisters = map[x86_64.Register64]bool { - RBX: true, - R12: true, - R13: true, - R14: true, - R15: true, +var reservedRegisters = map[x86_64.Register64]bool{ + RBX: true, + R12: true, + R13: true, + R14: true, + R15: true, } func ri2reg(ri uint8) hir.Register { - if ri & hir.ArgPointer == 0 { - return hir.GenericRegister(ri & hir.ArgMask) - } else { - return hir.PointerRegister(ri & hir.ArgMask) - } + if ri&hir.ArgPointer == 0 { + return hir.GenericRegister(ri & hir.ArgMask) + } else { + return hir.PointerRegister(ri & hir.ArgMask) + } } func checkfp(fp unsafe.Pointer) uintptr { - if fp == nil { - panic("checkfp: nil function") - } else { - return uintptr(fp) - } + if fp == nil { + panic("checkfp: nil function") + } else { + return uintptr(fp) + } } func checkptr(ri uint8, arg abi.Parameter) bool { - return arg.IsPointer() == ((ri & hir.ArgPointer) != 0) + return arg.IsPointer() == ((ri & hir.ArgPointer) != 0) } func (self *CodeGen) abiCallGo(p *x86_64.Program, v *hir.Ir) { - self.internalCallFunction(p, v, nil, func(fp *hir.CallHandle) { - p.MOVQ(checkfp(fp.Func), R12) - p.CALLQ(R12) - }) + self.internalCallFunction(p, v, nil, func(fp *hir.CallHandle) { + p.MOVQ(checkfp(fp.Func), R12) + p.CALLQ(R12) + }) } func (self *CodeGen) abiCallNative(p *x86_64.Program, v *hir.Ir) { - rv := hir.Register(nil) - fp := hir.LookupCall(v.Iv) - - /* native function can have at most 1 return value */ - if v.Rn > 1 { - panic("abiCallNative: native function can only have at most 1 return value") - } - - /* passing arguments on stack is currently not implemented */ - if int(v.An) > len(argumentOrder) { - panic("abiCallNative: not implemented: passing arguments on stack for native functions") - } - - /* save all the allocated registers (except reserved registers) before function call */ - for _, lr := range self.ctxt.regs { - if rr := self.r(lr); !reservedRegisters[rr] { - p.MOVQ(rr, self.ctxt.slot(lr)) - } - } - - /* load all the parameters */ - for i := 0; i < int(v.An); i++ { - rr := ri2reg(v.Ar[i]) - rd := argumentOrder[i] - - /* check for zero source and spilled arguments */ - if rr.Z() { - p.XORL(x86_64.Register32(rd), x86_64.Register32(rd)) - } else if rs := self.r(rr); argumentRegisters[rs] { - p.MOVQ(self.ctxt.slot(rr), rd) - } else { - p.MOVQ(rs, rd) - } - } - - /* call the function */ - p.MOVQ(checkfp(fp.Func), RAX) - p.CALLQ(RAX) - - /* store the result */ - if v.Rn != 0 { - if rv = ri2reg(v.Rr[0]); !rv.Z() { - p.MOVQ(RAX, self.r(rv)) - } - } - - /* restore all the allocated registers (except reserved registers and result) after function call */ - for _, lr := range self.ctxt.regs { - if rr := self.r(lr); (lr != rv) && !reservedRegisters[rr] { - p.MOVQ(self.ctxt.slot(lr), rr) - } - } + rv := hir.Register(nil) + fp := hir.LookupCall(v.Iv) + + /* native function can have at most 1 return value */ + if v.Rn > 1 { + panic("abiCallNative: native function can only have at most 1 return value") + } + + /* passing arguments on stack is currently not implemented */ + if int(v.An) > len(argumentOrder) { + panic("abiCallNative: not implemented: passing arguments on stack for native functions") + } + + /* save all the allocated registers (except reserved registers) before function call */ + for _, lr := range self.ctxt.regs { + if rr := self.r(lr); !reservedRegisters[rr] { + p.MOVQ(rr, self.ctxt.slot(lr)) + } + } + + /* load all the parameters */ + for i := 0; i < int(v.An); i++ { + rr := ri2reg(v.Ar[i]) + rd := argumentOrder[i] + + /* check for zero source and spilled arguments */ + if rr.Z() { + p.XORL(x86_64.Register32(rd), x86_64.Register32(rd)) + } else if rs := self.r(rr); argumentRegisters[rs] { + p.MOVQ(self.ctxt.slot(rr), rd) + } else { + p.MOVQ(rs, rd) + } + } + + /* call the function */ + p.MOVQ(checkfp(fp.Func), RAX) + p.CALLQ(RAX) + + /* store the result */ + if v.Rn != 0 { + if rv = ri2reg(v.Rr[0]); !rv.Z() { + p.MOVQ(RAX, self.r(rv)) + } + } + + /* restore all the allocated registers (except reserved registers and result) after function call */ + for _, lr := range self.ctxt.regs { + if rr := self.r(lr); (lr != rv) && !reservedRegisters[rr] { + p.MOVQ(self.ctxt.slot(lr), rr) + } + } } func (self *CodeGen) abiCallMethod(p *x86_64.Program, v *hir.Ir) { - self.internalCallFunction(p, v, v.Pd, func(fp *hir.CallHandle) { - p.MOVQ(self.ctxt.slot(v.Ps), R12) - p.CALLQ(Ptr(R12, int32(rt.GoItabFuncBase) + int32(fp.Slot) * abi.PtrSize)) - }) + self.internalCallFunction(p, v, v.Pd, func(fp *hir.CallHandle) { + p.MOVQ(self.ctxt.slot(v.Ps), R12) + p.CALLQ(Ptr(R12, int32(rt.GoItabFuncBase)+int32(fp.Slot)*abi.PtrSize)) + }) } func (self *CodeGen) internalSetArg(p *x86_64.Program, ri uint8, arg abi.Parameter, clobberSet map[x86_64.Register64]bool) { - if !checkptr(ri, arg) { - panic("internalSetArg: passing arguments in different kind of registers") - } else if !arg.InRegister { - self.internalSetStack(p, ri2reg(ri), arg) - } else { - self.internalSetRegister(p, ri2reg(ri), arg, clobberSet) - } + if !checkptr(ri, arg) { + panic("internalSetArg: passing arguments in different kind of registers") + } else if !arg.InRegister { + self.internalSetStack(p, ri2reg(ri), arg) + } else { + self.internalSetRegister(p, ri2reg(ri), arg, clobberSet) + } } func (self *CodeGen) internalSetStack(p *x86_64.Program, rr hir.Register, arg abi.Parameter) { - if rr.Z() { - p.MOVQ(0, Ptr(RSP, int32(arg.Mem))) - } else { - p.MOVQ(self.r(rr), Ptr(RSP, int32(arg.Mem))) - } + if rr.Z() { + p.MOVQ(0, Ptr(RSP, int32(arg.Mem))) + } else { + p.MOVQ(self.r(rr), Ptr(RSP, int32(arg.Mem))) + } } func (self *CodeGen) internalSetRegister(p *x86_64.Program, rr hir.Register, arg abi.Parameter, clobberSet map[x86_64.Register64]bool) { - if rr.Z() { - p.XORL(x86_64.Register32(arg.Reg), x86_64.Register32(arg.Reg)) - } else if lr := self.r(rr); clobberSet[lr] { - p.MOVQ(self.ctxt.slot(rr), arg.Reg) - } else if clobberSet[arg.Reg] = true; self.rindex(arg.Reg) != nil { - p.MOVQ(self.ctxt.slot(rr), arg.Reg) - } else { - p.MOVQ(lr, arg.Reg) - } + if rr.Z() { + p.XORL(x86_64.Register32(arg.Reg), x86_64.Register32(arg.Reg)) + } else if lr := self.r(rr); clobberSet[lr] { + p.MOVQ(self.ctxt.slot(rr), arg.Reg) + } else if clobberSet[arg.Reg] = true; self.rindex(arg.Reg) != nil { + p.MOVQ(self.ctxt.slot(rr), arg.Reg) + } else { + p.MOVQ(lr, arg.Reg) + } } func (self *CodeGen) internalCallFunction(p *x86_64.Program, v *hir.Ir, this hir.Register, makeFuncCall func(fp *hir.CallHandle)) { - ac := 0 - fp := hir.LookupCall(v.Iv) - fv := abi.ABI.GetLayout(fp.Id) - rm := make(map[hir.Register]int32) - cs := make(map[x86_64.Register64]bool) - - /* find the function */ - if fv == nil { - panic(fmt.Sprintf("internalCallFunction: invalid function ID: %d", v.Iv)) - } - - /* "this" is an implicit argument, so exclude from argument count */ - if this != nil { - ac = 1 - } - - /* check for argument and return value count */ - if int(v.Rn) != len(fv.Rets) || int(v.An) != len(fv.Args) - ac { - panic("internalCallFunction: argument or return value count mismatch") - } - - /* save all the allocated registers before function call */ - for _, lr := range self.ctxt.regs { - p.MOVQ(self.r(lr), self.ctxt.slot(lr)) - } - - /* load all the arguments */ - for i, vv := range fv.Args { - if i == 0 && this != nil { - self.internalSetArg(p, this.A(), vv, cs) - } else { - self.internalSetArg(p, v.Ar[i - ac], vv, cs) - } - } - - /* call the function with reserved registers restored */ - self.abiLoadReserved(p) - makeFuncCall(fp) - self.abiSaveReserved(p) - - /* if the function returns a value with a used register, spill it on stack */ - for i, retv := range fv.Rets { - if rr := ri2reg(v.Rr[i]); !rr.Z() { - if !retv.InRegister { - rm[rr] = int32(retv.Mem) - } else if self.rindex(retv.Reg) != nil { - p.MOVQ(retv.Reg, self.ctxt.slot(rr)) - } - } - } - - /* save all the non-spilled arguments */ - for i, retv := range fv.Rets { - if rr := ri2reg(v.Rr[i]); !rr.Z() { - if retv.InRegister && self.rindex(retv.Reg) == nil { - rm[rr] = -1 - p.MOVQ(retv.Reg, self.r(rr)) - } - } - } - - /* restore all the allocated registers (except return values) after function call */ - for _, lr := range self.ctxt.regs { - if _, ok := rm[lr]; !ok { - p.MOVQ(self.ctxt.slot(lr), self.r(lr)) - } - } - - /* store all the stack-based return values */ - for rr, mem := range rm { - if mem != -1 { - p.MOVQ(Ptr(RSP, mem), self.r(rr)) - } - } + ac := 0 + fp := hir.LookupCall(v.Iv) + fv := abi.ABI.GetLayout(fp.Id) + rm := make(map[hir.Register]int32) + cs := make(map[x86_64.Register64]bool) + + /* find the function */ + if fv == nil { + panic(fmt.Sprintf("internalCallFunction: invalid function ID: %d", v.Iv)) + } + + /* "this" is an implicit argument, so exclude from argument count */ + if this != nil { + ac = 1 + } + + /* check for argument and return value count */ + if int(v.Rn) != len(fv.Rets) || int(v.An) != len(fv.Args)-ac { + panic("internalCallFunction: argument or return value count mismatch") + } + + /* save all the allocated registers before function call */ + for _, lr := range self.ctxt.regs { + p.MOVQ(self.r(lr), self.ctxt.slot(lr)) + } + + /* load all the arguments */ + for i, vv := range fv.Args { + if i == 0 && this != nil { + self.internalSetArg(p, this.A(), vv, cs) + } else { + self.internalSetArg(p, v.Ar[i-ac], vv, cs) + } + } + + /* call the function with reserved registers restored */ + self.abiLoadReserved(p) + makeFuncCall(fp) + self.abiSaveReserved(p) + + /* if the function returns a value with a used register, spill it on stack */ + for i, retv := range fv.Rets { + if rr := ri2reg(v.Rr[i]); !rr.Z() { + if !retv.InRegister { + rm[rr] = int32(retv.Mem) + } else if self.rindex(retv.Reg) != nil { + p.MOVQ(retv.Reg, self.ctxt.slot(rr)) + } + } + } + + /* save all the non-spilled arguments */ + for i, retv := range fv.Rets { + if rr := ri2reg(v.Rr[i]); !rr.Z() { + if retv.InRegister && self.rindex(retv.Reg) == nil { + rm[rr] = -1 + p.MOVQ(retv.Reg, self.r(rr)) + } + } + } + + /* restore all the allocated registers (except return values) after function call */ + for _, lr := range self.ctxt.regs { + if _, ok := rm[lr]; !ok { + p.MOVQ(self.ctxt.slot(lr), self.r(lr)) + } + } + + /* store all the stack-based return values */ + for rr, mem := range rm { + if mem != -1 { + p.MOVQ(Ptr(RSP, mem), self.r(rr)) + } + } } diff --git a/internal/atm/pgen/pgen_amd64.go b/internal/atm/pgen/pgen_amd64.go index b91a45e..21054f2 100644 --- a/internal/atm/pgen/pgen_amd64.go +++ b/internal/atm/pgen/pgen_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,61 +17,62 @@ package pgen import ( - `fmt` - `math` - `math/bits` - `reflect` - `sort` - `sync/atomic` - - `github.com/cloudwego/iasm/expr` - `github.com/cloudwego/iasm/x86_64` - `github.com/cloudwego/frugal/internal/atm/abi` - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "fmt" + "math" + "math/bits" + "reflect" + "sort" + "sync/atomic" + + "github.com/cloudwego/frugal/internal/atm/abi" + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/iasm/expr" + "github.com/cloudwego/iasm/x86_64" ) type _SwitchTable struct { - ref *x86_64.Label - tab []*x86_64.Label + ref *x86_64.Label + tab []*x86_64.Label } var ( - stabCount uint64 + stabCount uint64 ) func newSwitchTable(n int) (v _SwitchTable) { - return _SwitchTable { - tab: make([]*x86_64.Label, n), - ref: x86_64.CreateLabel(fmt.Sprintf("_table_%d", atomic.AddUint64(&stabCount, 1))), - } + return _SwitchTable{ + tab: make([]*x86_64.Label, n), + ref: x86_64.CreateLabel(fmt.Sprintf("_table_%d", atomic.AddUint64(&stabCount, 1))), + } } func (self *_SwitchTable) link(p *x86_64.Program) { - p.Link(self.ref) - self.refs(p, self.ref) + p.Link(self.ref) + self.refs(p, self.ref) } func (self *_SwitchTable) mark(i int, to *x86_64.Label) { - if i >= len(self.tab) { - panic("pgen: stab: index out of bound") - } else { - self.tab[i] = to - } + if i >= len(self.tab) { + panic("pgen: stab: index out of bound") + } else { + self.tab[i] = to + } } func (self *_SwitchTable) refs(p *x86_64.Program, to *x86_64.Label) { - for _, v := range self.tab { - p.Long(expr.Ref(v).Sub(expr.Ref(to))) - } + for _, v := range self.tab { + p.Long(expr.Ref(v).Sub(expr.Ref(to))) + } } type _DeferBlock struct { - ref *x86_64.Label - def func(p *x86_64.Program) + ref *x86_64.Label + def func(p *x86_64.Program) } type _RegSeq []hir.Register + func (self _RegSeq) Len() int { return len(self) } func (self _RegSeq) Swap(i int, j int) { self[i], self[j] = self[j], self[i] } func (self _RegSeq) Less(i int, j int) bool { return self[i].A() < self[j].A() } @@ -93,1164 +94,1247 @@ func (self _RegSeq) Less(i int, j int) bool { return self[i].A() < self[j].A() } */ type _FrameInfo struct { - alen int - regs _RegSeq - desc *abi.FunctionLayout - regi map[hir.Register]int32 - regr map[x86_64.Register64]int32 + alen int + regs _RegSeq + desc *abi.FunctionLayout + regi map[hir.Register]int32 + regr map[x86_64.Register64]int32 } func (self *_FrameInfo) regc() int { - return len(self.regs) + return len(self.regs) } func (self *_FrameInfo) argc() int { - return len(self.desc.Args) + return len(self.desc.Args) } func (self *_FrameInfo) retc() int { - return len(self.desc.Rets) + return len(self.desc.Rets) } func (self *_FrameInfo) save() int32 { - return int32(self.alen) + return int32(self.alen) } func (self *_FrameInfo) prev() int32 { - return self.size() + abi.PtrSize + return self.size() + abi.PtrSize } func (self *_FrameInfo) size() int32 { - return self.offs() + abi.PtrSize + return self.offs() + abi.PtrSize } func (self *_FrameInfo) offs() int32 { - return self.rsvd() + int32(len(self.regr)) *abi.PtrSize + return self.rsvd() + int32(len(self.regr))*abi.PtrSize } func (self *_FrameInfo) rsvd() int32 { - return self.save() + int32(len(self.regs)) *abi.PtrSize + return self.save() + int32(len(self.regs))*abi.PtrSize } func (self *_FrameInfo) argv(i int) *x86_64.MemoryOperand { - return Ptr(RSP, self.prev() + int32(self.desc.Args[i].Mem)) + return Ptr(RSP, self.prev()+int32(self.desc.Args[i].Mem)) } func (self *_FrameInfo) retv(i int) *x86_64.MemoryOperand { - return Ptr(RSP, self.prev() + int32(self.desc.Rets[i].Mem)) + return Ptr(RSP, self.prev()+int32(self.desc.Rets[i].Mem)) } func (self *_FrameInfo) slot(r hir.Register) *x86_64.MemoryOperand { - return Ptr(RSP, self.save() + self.regi[r] *abi.PtrSize) + return Ptr(RSP, self.save()+self.regi[r]*abi.PtrSize) } func (self *_FrameInfo) rslot(r x86_64.Register64) *x86_64.MemoryOperand { - return Ptr(RSP, self.rsvd() + self.regr[r] *abi.PtrSize) + return Ptr(RSP, self.rsvd()+self.regr[r]*abi.PtrSize) } func (self *_FrameInfo) ralloc(r hir.Register) { - self.regs = append(self.regs, r) - sort.Sort(self.regs) + self.regs = append(self.regs, r) + sort.Sort(self.regs) - /* assign slots in ascending order */ - for i, v := range self.regs { - self.regi[v] = int32(i) - } + /* assign slots in ascending order */ + for i, v := range self.regs { + self.regi[v] = int32(i) + } } func (self *_FrameInfo) require(n uintptr) { - if self.alen < int(n) { - self.alen = int(n) - } + if self.alen < int(n) { + self.alen = int(n) + } } func (self *_FrameInfo) ArgPtrs() *rt.StackMap { - return self.desc.StackMap() + return self.desc.StackMap() } func (self *_FrameInfo) LocalPtrs() *rt.StackMap { - var v hir.Register - var m rt.StackMapBuilder + var v hir.Register + var m rt.StackMapBuilder - /* register spill slots */ - for _, v = range self.regs { - m.AddField(v.A() & hir.ArgPointer != 0) - } + /* register spill slots */ + for _, v = range self.regs { + m.AddField(v.A()&hir.ArgPointer != 0) + } - /* reserved registers */ - m.AddFields(len(self.regr), false) - return m.Build() + /* reserved registers */ + m.AddFields(len(self.regr), false) + return m.Build() } func newContext(proto interface{}) (ret _FrameInfo) { - vt := reflect.TypeOf(proto) - vk := vt.Kind() + vt := reflect.TypeOf(proto) + vk := vt.Kind() - /* must be a function */ - if vk != reflect.Func { - panic("pgen: proto must be a function") - } + /* must be a function */ + if vk != reflect.Func { + panic("pgen: proto must be a function") + } - /* layout the function */ - ret.regr = abi.ABI.Reserved() - ret.desc = abi.ABI.LayoutFunc(-1, vt) - ret.regi = make(map[hir.Register]int32) - return + /* layout the function */ + ret.regr = abi.ABI.Reserved() + ret.desc = abi.ABI.LayoutFunc(-1, vt) + ret.regi = make(map[hir.Register]int32) + return } type ( - Operands uint16 + Operands uint16 ) const ( - Orx Operands = 1 << iota // read Rx register - Ory // read Ry register - Owx // write Rx register - Owy // write Ry register - Owz // write ir.Rz register - Ops // read Ps register - Opd // write Pd register - Ojmp // unconditional jumps - Oret // function returns - Ocall // function calls - Octrl // control OpCodes + Orx Operands = 1 << iota // read Rx register + Ory // read Ry register + Owx // write Rx register + Owy // write Ry register + Owz // write ir.Rz register + Ops // read Ps register + Opd // write Pd register + Ojmp // unconditional jumps + Oret // function returns + Ocall // function calls + Octrl // control OpCodes ) -var _OperandMask = [256]Operands { - hir.OP_nop : Octrl, - hir.OP_ip : Opd, - hir.OP_lb : Owx | Ops, - hir.OP_lw : Owx | Ops, - hir.OP_ll : Owx | Ops, - hir.OP_lq : Owx | Ops, - hir.OP_lp : Ops | Opd, - hir.OP_sb : Orx | Opd, - hir.OP_sw : Orx | Opd, - hir.OP_sl : Orx | Opd, - hir.OP_sq : Orx | Opd, - hir.OP_sp : Ops | Opd, - hir.OP_ldaq : Owx, - hir.OP_ldap : Opd, - hir.OP_addp : Orx | Ops | Opd, - hir.OP_subp : Orx | Ops | Opd, - hir.OP_addpi : Ops | Opd, - hir.OP_add : Orx | Ory | Owz, - hir.OP_sub : Orx | Ory | Owz, - hir.OP_bts : Orx | Ory | Owy | Owz, - hir.OP_addi : Orx | Owy, - hir.OP_muli : Orx | Owy, - hir.OP_andi : Orx | Owy, - hir.OP_xori : Orx | Owy, - hir.OP_shri : Orx | Owy, - hir.OP_bsi : Orx | Owy, - hir.OP_swapw : Orx | Owy, - hir.OP_swapl : Orx | Owy, - hir.OP_swapq : Orx | Owy, - hir.OP_sxlq : Orx | Owy, - hir.OP_beq : Orx | Ory, - hir.OP_bne : Orx | Ory, - hir.OP_blt : Orx | Ory, - hir.OP_bltu : Orx | Ory, - hir.OP_bgeu : Orx | Ory, - hir.OP_beqp : Ops | Opd, - hir.OP_bnep : Ops | Opd, - hir.OP_bsw : Orx, - hir.OP_jmp : Ojmp, - hir.OP_bzero : Opd, - hir.OP_bcopy : Orx | Ops | Opd, - hir.OP_ccall : Ocall, - hir.OP_gcall : Ocall, - hir.OP_icall : Ocall, - hir.OP_ret : Oret, - hir.OP_break : Octrl, +var _OperandMask = [256]Operands{ + hir.OP_nop: Octrl, + hir.OP_ip: Opd, + hir.OP_lb: Owx | Ops, + hir.OP_lw: Owx | Ops, + hir.OP_ll: Owx | Ops, + hir.OP_lq: Owx | Ops, + hir.OP_lp: Ops | Opd, + hir.OP_sb: Orx | Opd, + hir.OP_sw: Orx | Opd, + hir.OP_sl: Orx | Opd, + hir.OP_sq: Orx | Opd, + hir.OP_sp: Ops | Opd, + hir.OP_ldaq: Owx, + hir.OP_ldap: Opd, + hir.OP_addp: Orx | Ops | Opd, + hir.OP_subp: Orx | Ops | Opd, + hir.OP_addpi: Ops | Opd, + hir.OP_add: Orx | Ory | Owz, + hir.OP_sub: Orx | Ory | Owz, + hir.OP_bts: Orx | Ory | Owy | Owz, + hir.OP_addi: Orx | Owy, + hir.OP_muli: Orx | Owy, + hir.OP_andi: Orx | Owy, + hir.OP_xori: Orx | Owy, + hir.OP_shri: Orx | Owy, + hir.OP_bsi: Orx | Owy, + hir.OP_swapw: Orx | Owy, + hir.OP_swapl: Orx | Owy, + hir.OP_swapq: Orx | Owy, + hir.OP_sxlq: Orx | Owy, + hir.OP_beq: Orx | Ory, + hir.OP_bne: Orx | Ory, + hir.OP_blt: Orx | Ory, + hir.OP_bltu: Orx | Ory, + hir.OP_bgeu: Orx | Ory, + hir.OP_beqp: Ops | Opd, + hir.OP_bnep: Ops | Opd, + hir.OP_bsw: Orx, + hir.OP_jmp: Ojmp, + hir.OP_bzero: Opd, + hir.OP_bcopy: Orx | Ops | Opd, + hir.OP_ccall: Ocall, + hir.OP_gcall: Ocall, + hir.OP_icall: Ocall, + hir.OP_ret: Oret, + hir.OP_break: Octrl, } type Func struct { - Code []byte - Frame rt.Frame + Code []byte + Frame rt.Frame } type CodeGen struct { - regi int - ctxt _FrameInfo - arch *x86_64.Arch - head *x86_64.Label - tail *x86_64.Label - halt *x86_64.Label - defs []_DeferBlock - stab []_SwitchTable - abix _CodeGenExtension - jmps map[string]*x86_64.Label - regs map[hir.Register]x86_64.Register64 + regi int + ctxt _FrameInfo + arch *x86_64.Arch + head *x86_64.Label + tail *x86_64.Label + halt *x86_64.Label + defs []_DeferBlock + stab []_SwitchTable + abix _CodeGenExtension + jmps map[string]*x86_64.Label + regs map[hir.Register]x86_64.Register64 } func CreateCodeGen(proto interface{}) *CodeGen { - return &CodeGen { - ctxt: newContext(proto), - arch: x86_64.DefaultArch, - jmps: make(map[string]*x86_64.Label), - regs: make(map[hir.Register]x86_64.Register64), - } + return &CodeGen{ + ctxt: newContext(proto), + arch: x86_64.DefaultArch, + jmps: make(map[string]*x86_64.Label), + regs: make(map[hir.Register]x86_64.Register64), + } } func (self *CodeGen) Generate(s hir.Program, sp uintptr) *Func { - h := 0 - p := self.arch.CreateProgram() - - /* find the halting points */ - for v := s.Head; h < 2 && v != nil; v = v.Ln { - if v.Op == hir.OP_ret { - h++ - } - } - - /* program must halt exactly once */ - switch h { - case 1 : break - case 0 : panic("pgen: program does not halt") - default : panic("pgen: program halts more than once") - } - - /* static register allocation */ - for v := s.Head; v != nil; v = v.Ln { - self.rcheck(v, _OperandMask[v.Op]) - self.walloc(v, _OperandMask[v.Op]) - } - - /* argument space calculation */ - for v := s.Head; v != nil; v = v.Ln { - switch v.Op { - case hir.OP_gcall: fallthrough - case hir.OP_icall: self.ctxt.require(abi.ABI.GetLayout(hir.LookupCall(v.Iv).Id).Sp) - case hir.OP_bcopy: self.ctxt.require(_M_memcpyargs) - } - } - - /* create the labels for stack management */ - entry := x86_64.CreateLabel("_entry") - stack := x86_64.CreateLabel("_stack_grow") - - /* create key anchor points */ - self.head = x86_64.CreateLabel("_head") - self.tail = x86_64.CreateLabel("_tail") - self.halt = x86_64.CreateLabel("_halt") - - /* stack checking */ - p.Link(entry) - self.abiStackCheck(p, stack, sp) - - /* program prologue */ - p.SUBQ(self.ctxt.size(), RSP) - p.Link(self.head) - p.MOVQ(RBP, Ptr(RSP, self.ctxt.offs())) - p.LEAQ(Ptr(RSP, self.ctxt.offs()), RBP) - - /* ABI-specific prologue */ - self.abiSaveReserved(p) - self.abiPrologue(p) - - /* clear all the pointer registers */ - for lr := range self.regs { - if lr.A() & hir.ArgPointer != 0 { - self.clr(p, lr) - } - } - - /* clear all the spill slots, if any */ - if i, n := 0, self.ctxt.regc(); n != 0 { - if n >= 2 { p.PXOR (XMM15, XMM15) } - for n >= 2 { p.MOVDQU (XMM15, Ptr(RSP, self.ctxt.save() + int32(i) *abi.PtrSize)); i += 2; n -= 2 } - if n != 0 { p.MOVQ (0, Ptr(RSP, self.ctxt.save() + int32(i) *abi.PtrSize)) } - } - - /* translate the entire program */ - for v := s.Head; v != nil; v = v.Ln { - self.translate(p, v) - } - - /* generate all defered blocks */ - for _, fp := range self.defs { - p.Link(fp.ref) - fp.def(p) - } - - /* ABI-specific epilogue */ - p.Link(self.halt) - self.abiEpilogue(p) - self.abiLoadReserved(p) - - /* program epilogue */ - p.MOVQ(Ptr(RSP, self.ctxt.offs()), RBP) - p.ADDQ(self.ctxt.size(), RSP) - p.Link(self.tail) - p.RET() - - /* stack grow */ - p.Link(stack) - self.abiStackGrow(p) - p.JMP(entry) - - /* link all the lookup tables */ - for _, v := range self.stab { - v.link(p) - } - - /* stack ranges */ - code := p.Assemble(0) - head := toAddress(self.head) - tail := toAddress(self.tail) - args := uintptr(self.ctxt.save()) - size := uintptr(self.ctxt.size()) - - /* build the PC-SP tab */ - tab := []rt.Stack { - { Sp: 0, Nb: head }, - { Sp: size, Nb: tail - head }, - { Sp: 0, Nb: 0 }, - } - - /* assemble the function */ - ret := &Func { - Code : code, - Frame : rt.Frame { - SpTab : tab, - ArgSize : args, - ArgPtrs : self.ctxt.ArgPtrs(), - LocalPtrs : self.ctxt.LocalPtrs(), - }, - } - - /* free the assembler */ - p.Free() - return ret + h := 0 + p := self.arch.CreateProgram() + + /* find the halting points */ + for v := s.Head; h < 2 && v != nil; v = v.Ln { + if v.Op == hir.OP_ret { + h++ + } + } + + /* program must halt exactly once */ + switch h { + case 1: + break + case 0: + panic("pgen: program does not halt") + default: + panic("pgen: program halts more than once") + } + + /* static register allocation */ + for v := s.Head; v != nil; v = v.Ln { + self.rcheck(v, _OperandMask[v.Op]) + self.walloc(v, _OperandMask[v.Op]) + } + + /* argument space calculation */ + for v := s.Head; v != nil; v = v.Ln { + switch v.Op { + case hir.OP_gcall: + fallthrough + case hir.OP_icall: + self.ctxt.require(abi.ABI.GetLayout(hir.LookupCall(v.Iv).Id).Sp) + case hir.OP_bcopy: + self.ctxt.require(_M_memcpyargs) + } + } + + /* create the labels for stack management */ + entry := x86_64.CreateLabel("_entry") + stack := x86_64.CreateLabel("_stack_grow") + + /* create key anchor points */ + self.head = x86_64.CreateLabel("_head") + self.tail = x86_64.CreateLabel("_tail") + self.halt = x86_64.CreateLabel("_halt") + + /* stack checking */ + p.Link(entry) + self.abiStackCheck(p, stack, sp) + + /* program prologue */ + p.SUBQ(self.ctxt.size(), RSP) + p.Link(self.head) + p.MOVQ(RBP, Ptr(RSP, self.ctxt.offs())) + p.LEAQ(Ptr(RSP, self.ctxt.offs()), RBP) + + /* ABI-specific prologue */ + self.abiSaveReserved(p) + self.abiPrologue(p) + + /* clear all the pointer registers */ + for lr := range self.regs { + if lr.A()&hir.ArgPointer != 0 { + self.clr(p, lr) + } + } + + /* clear all the spill slots, if any */ + if i, n := 0, self.ctxt.regc(); n != 0 { + if n >= 2 { + p.PXOR(XMM15, XMM15) + } + for n >= 2 { + p.MOVDQU(XMM15, Ptr(RSP, self.ctxt.save()+int32(i)*abi.PtrSize)) + i += 2 + n -= 2 + } + if n != 0 { + p.MOVQ(0, Ptr(RSP, self.ctxt.save()+int32(i)*abi.PtrSize)) + } + } + + /* translate the entire program */ + for v := s.Head; v != nil; v = v.Ln { + self.translate(p, v) + } + + /* generate all deferred blocks */ + for _, fp := range self.defs { + p.Link(fp.ref) + fp.def(p) + } + + /* ABI-specific epilogue */ + p.Link(self.halt) + self.abiEpilogue(p) + self.abiLoadReserved(p) + + /* program epilogue */ + p.MOVQ(Ptr(RSP, self.ctxt.offs()), RBP) + p.ADDQ(self.ctxt.size(), RSP) + p.Link(self.tail) + p.RET() + + /* stack grow */ + p.Link(stack) + self.abiStackGrow(p) + p.JMP(entry) + + /* link all the lookup tables */ + for _, v := range self.stab { + v.link(p) + } + + /* stack ranges */ + code := p.Assemble(0) + head := toAddress(self.head) + tail := toAddress(self.tail) + args := uintptr(self.ctxt.save()) + size := uintptr(self.ctxt.size()) + + /* build the PC-SP tab */ + tab := []rt.Stack{ + {Sp: 0, Nb: head}, + {Sp: size, Nb: tail - head}, + {Sp: 0, Nb: 0}, + } + + /* assemble the function */ + ret := &Func{ + Code: code, + Frame: rt.Frame{ + SpTab: tab, + ArgSize: args, + ArgPtrs: self.ctxt.ArgPtrs(), + LocalPtrs: self.ctxt.LocalPtrs(), + }, + } + + /* free the assembler */ + p.Free() + return ret } func (self *CodeGen) later(ref *x86_64.Label, def func(*x86_64.Program)) { - self.defs = append(self.defs, _DeferBlock { - ref: ref, - def: def, - }) + self.defs = append(self.defs, _DeferBlock{ + ref: ref, + def: def, + }) } func (self *CodeGen) translate(p *x86_64.Program, v *hir.Ir) { - if p.Link(self.to(v)); v.Op != hir.OP_nop { - if fp := translators[v.Op]; fp != nil { - fp(self, p, v) - } else { - panic("pgen: invalid instruction: " + v.Disassemble(nil)) - } - } + if p.Link(self.to(v)); v.Op != hir.OP_nop { + if fp := translators[v.Op]; fp != nil { + fp(self, p, v) + } else { + panic("pgen: invalid instruction: " + v.Disassemble(nil)) + } + } } /** Register Allocation **/ type _Check struct { - bv Operands - fn func(*hir.Ir) hir.Register + bv Operands + fn func(*hir.Ir) hir.Register } -var _readChecks = [...]_Check { - { bv: Orx, fn: func(p *hir.Ir) hir.Register { return p.Rx } }, - { bv: Ory, fn: func(p *hir.Ir) hir.Register { return p.Ry } }, - { bv: Ops, fn: func(p *hir.Ir) hir.Register { return p.Ps } }, +var _readChecks = [...]_Check{ + {bv: Orx, fn: func(p *hir.Ir) hir.Register { return p.Rx }}, + {bv: Ory, fn: func(p *hir.Ir) hir.Register { return p.Ry }}, + {bv: Ops, fn: func(p *hir.Ir) hir.Register { return p.Ps }}, } -var _writeAllocs = [...]_Check { - { bv: Owx, fn: func(p *hir.Ir) hir.Register { return p.Rx } }, - { bv: Owy, fn: func(p *hir.Ir) hir.Register { return p.Ry } }, - { bv: Owz, fn: func(p *hir.Ir) hir.Register { return p.Rz } }, - { bv: Opd, fn: func(p *hir.Ir) hir.Register { return p.Pd } }, +var _writeAllocs = [...]_Check{ + {bv: Owx, fn: func(p *hir.Ir) hir.Register { return p.Rx }}, + {bv: Owy, fn: func(p *hir.Ir) hir.Register { return p.Ry }}, + {bv: Owz, fn: func(p *hir.Ir) hir.Register { return p.Rz }}, + {bv: Opd, fn: func(p *hir.Ir) hir.Register { return p.Pd }}, } func (self *CodeGen) rload(v *hir.Ir, r hir.Register) { - if _, ok := self.regs[r]; !ok && !r.Z() { - panic(fmt.Sprintf("pgen: access to unallocated register %s: %s", r.String(), v.Disassemble(nil))) - } + if _, ok := self.regs[r]; !ok && !r.Z() { + panic(fmt.Sprintf("pgen: access to unallocated register %s: %s", r.String(), v.Disassemble(nil))) + } } func (self *CodeGen) rstore(v *hir.Ir, r hir.Register) { - if _, ok := self.regs[r]; !ok && !r.Z() { - if self.ctxt.ralloc(r); self.regi < len(allocationOrder) { - self.regi, self.regs[r] = self.regi + 1, allocationOrder[self.regi] - } else { - panic("pgen: program is too complex to translate on x86_64 (requiring too many registers): " + v.Disassemble(nil)) - } - } + if _, ok := self.regs[r]; !ok && !r.Z() { + if self.ctxt.ralloc(r); self.regi < len(allocationOrder) { + self.regi, self.regs[r] = self.regi+1, allocationOrder[self.regi] + } else { + panic("pgen: program is too complex to translate on x86_64 (requiring too many registers): " + v.Disassemble(nil)) + } + } } func (self *CodeGen) ldret(v *hir.Ir) { - for i := 0; i < int(v.Rn); i++ { - if r := v.Rr[i]; r & hir.ArgPointer == 0 { - self.rload(v, hir.GenericRegister(r)) - } else { - self.rload(v, hir.PointerRegister(r & hir.ArgMask)) - } - } + for i := 0; i < int(v.Rn); i++ { + if r := v.Rr[i]; r&hir.ArgPointer == 0 { + self.rload(v, hir.GenericRegister(r)) + } else { + self.rload(v, hir.PointerRegister(r&hir.ArgMask)) + } + } } func (self *CodeGen) ldcall(v *hir.Ir) { - for i := 0; i < int(v.An); i++ { - if r := v.Ar[i]; r & hir.ArgPointer == 0 { - self.rload(v, hir.GenericRegister(r)) - } else { - self.rload(v, hir.PointerRegister(r & hir.ArgMask)) - } - } + for i := 0; i < int(v.An); i++ { + if r := v.Ar[i]; r&hir.ArgPointer == 0 { + self.rload(v, hir.GenericRegister(r)) + } else { + self.rload(v, hir.PointerRegister(r&hir.ArgMask)) + } + } } func (self *CodeGen) stcall(v *hir.Ir) { - for i := 0; i < int(v.Rn); i++ { - if r := v.Rr[i]; r & hir.ArgPointer == 0 { - self.rstore(v, hir.GenericRegister(r)) - } else { - self.rstore(v, hir.PointerRegister(r & hir.ArgMask)) - } - } + for i := 0; i < int(v.Rn); i++ { + if r := v.Rr[i]; r&hir.ArgPointer == 0 { + self.rstore(v, hir.GenericRegister(r)) + } else { + self.rstore(v, hir.PointerRegister(r&hir.ArgMask)) + } + } } func (self *CodeGen) rcheck(v *hir.Ir, p Operands) { - for _, cc := range _readChecks { - if p & Oret != 0 { self.ldret(v) } - if p & Ocall != 0 { self.ldcall(v) } - if p & cc.bv != 0 { self.rload(v, cc.fn(v)) } - } + for _, cc := range _readChecks { + if p&Oret != 0 { + self.ldret(v) + } + if p&Ocall != 0 { + self.ldcall(v) + } + if p&cc.bv != 0 { + self.rload(v, cc.fn(v)) + } + } } func (self *CodeGen) walloc(v *hir.Ir, p Operands) { - for _, cc := range _writeAllocs { - if p & Ocall != 0 { self.stcall(v) } - if p & cc.bv != 0 { self.rstore(v, cc.fn(v)) } - } + for _, cc := range _writeAllocs { + if p&Ocall != 0 { + self.stcall(v) + } + if p&cc.bv != 0 { + self.rstore(v, cc.fn(v)) + } + } } func (self *CodeGen) rindex(r x86_64.Register64) hir.Register { - for k, v := range self.regs { if v == r { return k } } - return nil + for k, v := range self.regs { + if v == r { + return k + } + } + return nil } /** Generator Helpers **/ func (self *CodeGen) r(reg hir.Register) x86_64.Register64 { - if rr, ok := self.regs[reg]; !ok { - panic("pgen: access to unallocated register: " + reg.String()) - } else { - return rr - } + if rr, ok := self.regs[reg]; !ok { + panic("pgen: access to unallocated register: " + reg.String()) + } else { + return rr + } } func (self *CodeGen) to(v *hir.Ir) *x86_64.Label { - return self.ref(fmt.Sprintf("_PC_%p", v)) + return self.ref(fmt.Sprintf("_PC_%p", v)) } func (self *CodeGen) tab(i int64) *_SwitchTable { - p := len(self.stab) - self.stab = append(self.stab, newSwitchTable(int(i))) - return &self.stab[p] + p := len(self.stab) + self.stab = append(self.stab, newSwitchTable(int(i))) + return &self.stab[p] } func (self *CodeGen) ref(s string) *x86_64.Label { - var k bool - var p *x86_64.Label + var k bool + var p *x86_64.Label - /* check for existance */ - if p, k = self.jmps[s]; k { - return p - } + /* check for existence */ + if p, k = self.jmps[s]; k { + return p + } - /* create a new label if not */ - p = x86_64.CreateLabel(s) - self.jmps[s] = p - return p + /* create a new label if not */ + p = x86_64.CreateLabel(s) + self.jmps[s] = p + return p } func (self *CodeGen) i32(p *x86_64.Program, v *hir.Ir) interface{} { - if isInt32(v.Iv) { - return v.Iv - } else { - p.MOVQ(v.Iv, RAX) - return RAX - } + if isInt32(v.Iv) { + return v.Iv + } else { + p.MOVQ(v.Iv, RAX) + return RAX + } } func (self *CodeGen) ptr(p *x86_64.Program, r hir.PointerRegister, d int64) *x86_64.MemoryOperand { - if isInt32(d) { - return Ptr(self.r(r), int32(d)) - } else { - p.MOVQ(d, RAX) - return Sib(self.r(r), RAX, 1, 0) - } + if isInt32(d) { + return Ptr(self.r(r), int32(d)) + } else { + p.MOVQ(d, RAX) + return Sib(self.r(r), RAX, 1, 0) + } } func (self *CodeGen) clr(p *x86_64.Program, r hir.Register) { - rx := self.r(r) - p.XORL(x86_64.Register32(rx), x86_64.Register32(rx)) + rx := self.r(r) + p.XORL(x86_64.Register32(rx), x86_64.Register32(rx)) } func (self *CodeGen) set(p *x86_64.Program, r hir.Register, i int64) { - if i == 0 { - self.clr(p, r) - } else { - p.MOVQ(i, self.r(r)) - } + if i == 0 { + self.clr(p, r) + } else { + p.MOVQ(i, self.r(r)) + } } func (self *CodeGen) dup(p *x86_64.Program, r hir.Register, d hir.Register) { - if r != d { - p.MOVQ(self.r(r), self.r(d)) - } + if r != d { + p.MOVQ(self.r(r), self.r(d)) + } } /** OpCode Generators **/ -var translators = [256]func(*CodeGen, *x86_64.Program, *hir.Ir) { - hir.OP_ip : (*CodeGen).translate_OP_ip, - hir.OP_lb : (*CodeGen).translate_OP_lb, - hir.OP_lw : (*CodeGen).translate_OP_lw, - hir.OP_ll : (*CodeGen).translate_OP_ll, - hir.OP_lq : (*CodeGen).translate_OP_lq, - hir.OP_lp : (*CodeGen).translate_OP_lp, - hir.OP_sb : (*CodeGen).translate_OP_sb, - hir.OP_sw : (*CodeGen).translate_OP_sw, - hir.OP_sl : (*CodeGen).translate_OP_sl, - hir.OP_sq : (*CodeGen).translate_OP_sq, - hir.OP_sp : (*CodeGen).translate_OP_sp, - hir.OP_ldaq : (*CodeGen).translate_OP_ldaq, - hir.OP_ldap : (*CodeGen).translate_OP_ldap, - hir.OP_addp : (*CodeGen).translate_OP_addp, - hir.OP_subp : (*CodeGen).translate_OP_subp, - hir.OP_addpi : (*CodeGen).translate_OP_addpi, - hir.OP_add : (*CodeGen).translate_OP_add, - hir.OP_sub : (*CodeGen).translate_OP_sub, - hir.OP_bts : (*CodeGen).translate_OP_bts, - hir.OP_addi : (*CodeGen).translate_OP_addi, - hir.OP_muli : (*CodeGen).translate_OP_muli, - hir.OP_andi : (*CodeGen).translate_OP_andi, - hir.OP_xori : (*CodeGen).translate_OP_xori, - hir.OP_shri : (*CodeGen).translate_OP_shri, - hir.OP_bsi : (*CodeGen).translate_OP_bsi, - hir.OP_swapw : (*CodeGen).translate_OP_swapw, - hir.OP_swapl : (*CodeGen).translate_OP_swapl, - hir.OP_swapq : (*CodeGen).translate_OP_swapq, - hir.OP_sxlq : (*CodeGen).translate_OP_sxlq, - hir.OP_beq : (*CodeGen).translate_OP_beq, - hir.OP_bne : (*CodeGen).translate_OP_bne, - hir.OP_blt : (*CodeGen).translate_OP_blt, - hir.OP_bltu : (*CodeGen).translate_OP_bltu, - hir.OP_bgeu : (*CodeGen).translate_OP_bgeu, - hir.OP_beqp : (*CodeGen).translate_OP_beqp, - hir.OP_bnep : (*CodeGen).translate_OP_bnep, - hir.OP_bsw : (*CodeGen).translate_OP_bsw, - hir.OP_jmp : (*CodeGen).translate_OP_jmp, - hir.OP_bzero : (*CodeGen).translate_OP_bzero, - hir.OP_bcopy : (*CodeGen).translate_OP_bcopy, - hir.OP_ccall : (*CodeGen).translate_OP_ccall, - hir.OP_gcall : (*CodeGen).translate_OP_gcall, - hir.OP_icall : (*CodeGen).translate_OP_icall, - hir.OP_ret : (*CodeGen).translate_OP_ret, - hir.OP_break : (*CodeGen).translate_OP_break, +var translators = [256]func(*CodeGen, *x86_64.Program, *hir.Ir){ + hir.OP_ip: (*CodeGen).translate_OP_ip, + hir.OP_lb: (*CodeGen).translate_OP_lb, + hir.OP_lw: (*CodeGen).translate_OP_lw, + hir.OP_ll: (*CodeGen).translate_OP_ll, + hir.OP_lq: (*CodeGen).translate_OP_lq, + hir.OP_lp: (*CodeGen).translate_OP_lp, + hir.OP_sb: (*CodeGen).translate_OP_sb, + hir.OP_sw: (*CodeGen).translate_OP_sw, + hir.OP_sl: (*CodeGen).translate_OP_sl, + hir.OP_sq: (*CodeGen).translate_OP_sq, + hir.OP_sp: (*CodeGen).translate_OP_sp, + hir.OP_ldaq: (*CodeGen).translate_OP_ldaq, + hir.OP_ldap: (*CodeGen).translate_OP_ldap, + hir.OP_addp: (*CodeGen).translate_OP_addp, + hir.OP_subp: (*CodeGen).translate_OP_subp, + hir.OP_addpi: (*CodeGen).translate_OP_addpi, + hir.OP_add: (*CodeGen).translate_OP_add, + hir.OP_sub: (*CodeGen).translate_OP_sub, + hir.OP_bts: (*CodeGen).translate_OP_bts, + hir.OP_addi: (*CodeGen).translate_OP_addi, + hir.OP_muli: (*CodeGen).translate_OP_muli, + hir.OP_andi: (*CodeGen).translate_OP_andi, + hir.OP_xori: (*CodeGen).translate_OP_xori, + hir.OP_shri: (*CodeGen).translate_OP_shri, + hir.OP_bsi: (*CodeGen).translate_OP_bsi, + hir.OP_swapw: (*CodeGen).translate_OP_swapw, + hir.OP_swapl: (*CodeGen).translate_OP_swapl, + hir.OP_swapq: (*CodeGen).translate_OP_swapq, + hir.OP_sxlq: (*CodeGen).translate_OP_sxlq, + hir.OP_beq: (*CodeGen).translate_OP_beq, + hir.OP_bne: (*CodeGen).translate_OP_bne, + hir.OP_blt: (*CodeGen).translate_OP_blt, + hir.OP_bltu: (*CodeGen).translate_OP_bltu, + hir.OP_bgeu: (*CodeGen).translate_OP_bgeu, + hir.OP_beqp: (*CodeGen).translate_OP_beqp, + hir.OP_bnep: (*CodeGen).translate_OP_bnep, + hir.OP_bsw: (*CodeGen).translate_OP_bsw, + hir.OP_jmp: (*CodeGen).translate_OP_jmp, + hir.OP_bzero: (*CodeGen).translate_OP_bzero, + hir.OP_bcopy: (*CodeGen).translate_OP_bcopy, + hir.OP_ccall: (*CodeGen).translate_OP_ccall, + hir.OP_gcall: (*CodeGen).translate_OP_gcall, + hir.OP_icall: (*CodeGen).translate_OP_icall, + hir.OP_ret: (*CodeGen).translate_OP_ret, + hir.OP_break: (*CodeGen).translate_OP_break, } func (self *CodeGen) translate_OP_ip(p *x86_64.Program, v *hir.Ir) { - if v.Pd != hir.Pn { - if addr := uintptr(v.Pr); addr > math.MaxUint32 { - p.MOVQ(addr, self.r(v.Pd)) - } else { - p.MOVL(addr, x86_64.Register32(self.r(v.Pd))) - } - } + if v.Pd != hir.Pn { + if addr := uintptr(v.Pr); addr > math.MaxUint32 { + p.MOVQ(addr, self.r(v.Pd)) + } else { + p.MOVL(addr, x86_64.Register32(self.r(v.Pd))) + } + } } func (self *CodeGen) translate_OP_lb(p *x86_64.Program, v *hir.Ir) { - if v.Rx != hir.Rz { - if v.Ps == hir.Pn { - panic("lb: load from nil pointer") - } else { - p.MOVZBQ(self.ptr(p, v.Ps, v.Iv), self.r(v.Rx)) - } - } + if v.Rx != hir.Rz { + if v.Ps == hir.Pn { + panic("lb: load from nil pointer") + } else { + p.MOVZBQ(self.ptr(p, v.Ps, v.Iv), self.r(v.Rx)) + } + } } func (self *CodeGen) translate_OP_lw(p *x86_64.Program, v *hir.Ir) { - if v.Rx != hir.Rz { - if v.Ps == hir.Pn { - panic("lw: load from nil pointer") - } else { - p.MOVZWQ(self.ptr(p, v.Ps, v.Iv), self.r(v.Rx)) - } - } + if v.Rx != hir.Rz { + if v.Ps == hir.Pn { + panic("lw: load from nil pointer") + } else { + p.MOVZWQ(self.ptr(p, v.Ps, v.Iv), self.r(v.Rx)) + } + } } func (self *CodeGen) translate_OP_ll(p *x86_64.Program, v *hir.Ir) { - if v.Rx != hir.Rz { - if v.Ps == hir.Pn { - panic("ll: load from nil pointer") - } else { - p.MOVL(self.ptr(p, v.Ps, v.Iv), x86_64.Register32(self.r(v.Rx))) - } - } + if v.Rx != hir.Rz { + if v.Ps == hir.Pn { + panic("ll: load from nil pointer") + } else { + p.MOVL(self.ptr(p, v.Ps, v.Iv), x86_64.Register32(self.r(v.Rx))) + } + } } func (self *CodeGen) translate_OP_lq(p *x86_64.Program, v *hir.Ir) { - if v.Rx != hir.Rz { - if v.Ps == hir.Pn { - panic("lq: load from nil pointer") - } else { - p.MOVQ(self.ptr(p, v.Ps, v.Iv), self.r(v.Rx)) - } - } + if v.Rx != hir.Rz { + if v.Ps == hir.Pn { + panic("lq: load from nil pointer") + } else { + p.MOVQ(self.ptr(p, v.Ps, v.Iv), self.r(v.Rx)) + } + } } func (self *CodeGen) translate_OP_lp(p *x86_64.Program, v *hir.Ir) { - if v.Pd != hir.Pn { - if v.Ps == hir.Pn { - panic("lp: load from nil pointer") - } else { - p.MOVQ(self.ptr(p, v.Ps, v.Iv), self.r(v.Pd)) - } - } + if v.Pd != hir.Pn { + if v.Ps == hir.Pn { + panic("lp: load from nil pointer") + } else { + p.MOVQ(self.ptr(p, v.Ps, v.Iv), self.r(v.Pd)) + } + } } func (self *CodeGen) translate_OP_sb(p *x86_64.Program, v *hir.Ir) { - if v.Pd == hir.Pn { - panic("sb: store to nil pointer") - } else if v.Rx == hir.Rz { - p.MOVB(0, self.ptr(p, v.Pd, v.Iv)) - } else { - p.MOVB(x86_64.Register8(self.r(v.Rx)), self.ptr(p, v.Pd, v.Iv)) - } + if v.Pd == hir.Pn { + panic("sb: store to nil pointer") + } else if v.Rx == hir.Rz { + p.MOVB(0, self.ptr(p, v.Pd, v.Iv)) + } else { + p.MOVB(x86_64.Register8(self.r(v.Rx)), self.ptr(p, v.Pd, v.Iv)) + } } func (self *CodeGen) translate_OP_sw(p *x86_64.Program, v *hir.Ir) { - if v.Pd == hir.Pn { - panic("sw: store to nil pointer") - } else if v.Rx == hir.Rz { - p.MOVW(0, self.ptr(p, v.Pd, v.Iv)) - } else { - p.MOVW(x86_64.Register16(self.r(v.Rx)), self.ptr(p, v.Pd, v.Iv)) - } + if v.Pd == hir.Pn { + panic("sw: store to nil pointer") + } else if v.Rx == hir.Rz { + p.MOVW(0, self.ptr(p, v.Pd, v.Iv)) + } else { + p.MOVW(x86_64.Register16(self.r(v.Rx)), self.ptr(p, v.Pd, v.Iv)) + } } func (self *CodeGen) translate_OP_sl(p *x86_64.Program, v *hir.Ir) { - if v.Pd == hir.Pn { - panic("sl: store to nil pointer") - } else if v.Rx == hir.Rz { - p.MOVL(0, self.ptr(p, v.Pd, v.Iv)) - } else { - p.MOVL(x86_64.Register32(self.r(v.Rx)), self.ptr(p, v.Pd, v.Iv)) - } + if v.Pd == hir.Pn { + panic("sl: store to nil pointer") + } else if v.Rx == hir.Rz { + p.MOVL(0, self.ptr(p, v.Pd, v.Iv)) + } else { + p.MOVL(x86_64.Register32(self.r(v.Rx)), self.ptr(p, v.Pd, v.Iv)) + } } func (self *CodeGen) translate_OP_sq(p *x86_64.Program, v *hir.Ir) { - if v.Pd == hir.Pn { - panic("sq: store to nil pointer") - } else if v.Rx == hir.Rz { - p.MOVQ(0, self.ptr(p, v.Pd, v.Iv)) - } else { - p.MOVQ(self.r(v.Rx), self.ptr(p, v.Pd, v.Iv)) - } + if v.Pd == hir.Pn { + panic("sq: store to nil pointer") + } else if v.Rx == hir.Rz { + p.MOVQ(0, self.ptr(p, v.Pd, v.Iv)) + } else { + p.MOVQ(self.r(v.Rx), self.ptr(p, v.Pd, v.Iv)) + } } func (self *CodeGen) translate_OP_sp(p *x86_64.Program, v *hir.Ir) { - if v.Pd == hir.Pn { - panic("sp: store to nil pointer") - } else { - self.wbStorePointer(p, v.Ps, self.ptr(p, v.Pd, v.Iv)) - } + if v.Pd == hir.Pn { + panic("sp: store to nil pointer") + } else { + self.wbStorePointer(p, v.Ps, self.ptr(p, v.Pd, v.Iv)) + } } func (self *CodeGen) translate_OP_ldaq(p *x86_64.Program, v *hir.Ir) { - if v.Rx != hir.Rz { - if i := int(v.Iv); i < self.ctxt.argc() { - self.abiLoadInt(p, i, v.Rx) - } else { - panic(fmt.Sprintf("ldaq: argument index out of range: %d", i)) - } - } + if v.Rx != hir.Rz { + if i := int(v.Iv); i < self.ctxt.argc() { + self.abiLoadInt(p, i, v.Rx) + } else { + panic(fmt.Sprintf("ldaq: argument index out of range: %d", i)) + } + } } func (self *CodeGen) translate_OP_ldap(p *x86_64.Program, v *hir.Ir) { - if v.Pd != hir.Pn { - if i := int(v.Iv); i < self.ctxt.argc() { - self.abiLoadPtr(p, i, v.Pd) - } else { - panic(fmt.Sprintf("ldap: argument index out of range: %d", v.Iv)) - } - } + if v.Pd != hir.Pn { + if i := int(v.Iv); i < self.ctxt.argc() { + self.abiLoadPtr(p, i, v.Pd) + } else { + panic(fmt.Sprintf("ldap: argument index out of range: %d", v.Iv)) + } + } } func (self *CodeGen) translate_OP_addp(p *x86_64.Program, v *hir.Ir) { - if v.Pd != hir.Pn { - if v.Ps == hir.Pn { - if v.Rx != hir.Rz { - panic("addp: direct conversion of integer to pointer") - } else { - self.clr(p, v.Pd) - } - } else { - if v.Rx != hir.Rz { - p.LEAQ(Sib(self.r(v.Ps), self.r(v.Rx), 1, 0), self.r(v.Pd)) - } else { - self.dup(p, v.Ps, v.Pd) - } - } - } + if v.Pd != hir.Pn { + if v.Ps == hir.Pn { + if v.Rx != hir.Rz { + panic("addp: direct conversion of integer to pointer") + } else { + self.clr(p, v.Pd) + } + } else { + if v.Rx != hir.Rz { + p.LEAQ(Sib(self.r(v.Ps), self.r(v.Rx), 1, 0), self.r(v.Pd)) + } else { + self.dup(p, v.Ps, v.Pd) + } + } + } } func (self *CodeGen) translate_OP_subp(p *x86_64.Program, v *hir.Ir) { - if v.Pd != hir.Pn { - if v.Ps == hir.Pn { - panic("subp: direct conversion of integer to pointer") - } else if self.dup(p, v.Ps, v.Pd); v.Rx != hir.Rz { - p.SUBQ(self.r(v.Rx), self.r(v.Pd)) - } - } + if v.Pd != hir.Pn { + if v.Ps == hir.Pn { + panic("subp: direct conversion of integer to pointer") + } else if self.dup(p, v.Ps, v.Pd); v.Rx != hir.Rz { + p.SUBQ(self.r(v.Rx), self.r(v.Pd)) + } + } } func (self *CodeGen) translate_OP_addpi(p *x86_64.Program, v *hir.Ir) { - if v.Pd != hir.Pn { - if v.Ps == hir.Pn { - if v.Iv != 0 { - panic("addpi: direct conversion of integer to pointer") - } else { - self.clr(p, v.Pd) - } - } else { - if !isInt32(v.Iv) { - panic("addpi: offset too large, may result in an invalid pointer") - } else { - p.LEAQ(Ptr(self.r(v.Ps), int32(v.Iv)), self.r(v.Pd)) - } - } - } + if v.Pd != hir.Pn { + if v.Ps == hir.Pn { + if v.Iv != 0 { + panic("addpi: direct conversion of integer to pointer") + } else { + self.clr(p, v.Pd) + } + } else { + if !isInt32(v.Iv) { + panic("addpi: offset too large, may result in an invalid pointer") + } else { + p.LEAQ(Ptr(self.r(v.Ps), int32(v.Iv)), self.r(v.Pd)) + } + } + } } func (self *CodeGen) translate_OP_add(p *x86_64.Program, v *hir.Ir) { - if v.Rz != hir.Rz { - if v.Rx == hir.Rz { - if v.Ry == hir.Rz { - self.clr(p, v.Rz) - } else { - self.dup(p, v.Ry, v.Rz) - } - } else { - if v.Ry == hir.Rz { - self.dup(p, v.Rx, v.Rz) - } else if v.Ry == v.Rz { - p.ADDQ(self.r(v.Rx), self.r(v.Rz)) - } else { - self.dup(p, v.Rx, v.Rz) - p.ADDQ(self.r(v.Ry), self.r(v.Rz)) - } - } - } + if v.Rz != hir.Rz { + if v.Rx == hir.Rz { + if v.Ry == hir.Rz { + self.clr(p, v.Rz) + } else { + self.dup(p, v.Ry, v.Rz) + } + } else { + if v.Ry == hir.Rz { + self.dup(p, v.Rx, v.Rz) + } else if v.Ry == v.Rz { + p.ADDQ(self.r(v.Rx), self.r(v.Rz)) + } else { + self.dup(p, v.Rx, v.Rz) + p.ADDQ(self.r(v.Ry), self.r(v.Rz)) + } + } + } } func (self *CodeGen) translate_OP_sub(p *x86_64.Program, v *hir.Ir) { - if v.Rz != hir.Rz { - if v.Rx == hir.Rz { - if v.Ry == hir.Rz { - self.clr(p, v.Rz) - } else { - self.dup(p, v.Ry, v.Rz) - } - } else { - if v.Ry == hir.Rz { - self.dup(p, v.Rx, v.Rz) - } else if v.Ry == v.Rz { - p.SUBQ(self.r(v.Rx), self.r(v.Rz)) - p.NEGQ(self.r(v.Rz)) - } else { - self.dup(p, v.Rx, v.Rz) - p.SUBQ(self.r(v.Ry), self.r(v.Rz)) - } - } - } + if v.Rz != hir.Rz { + if v.Rx == hir.Rz { + if v.Ry == hir.Rz { + self.clr(p, v.Rz) + } else { + self.dup(p, v.Ry, v.Rz) + } + } else { + if v.Ry == hir.Rz { + self.dup(p, v.Rx, v.Rz) + } else if v.Ry == v.Rz { + p.SUBQ(self.r(v.Rx), self.r(v.Rz)) + p.NEGQ(self.r(v.Rz)) + } else { + self.dup(p, v.Rx, v.Rz) + p.SUBQ(self.r(v.Ry), self.r(v.Rz)) + } + } + } } func (self *CodeGen) translate_OP_bts(p *x86_64.Program, v *hir.Ir) { - x := v.Rx - y := v.Ry - z := v.Rz + x := v.Rx + y := v.Ry + z := v.Rz - /* special case: y is zero */ - if y == hir.Rz { - return - } + /* special case: y is zero */ + if y == hir.Rz { + return + } - /* testing and setting the bits at the same time */ - if x == hir.Rz { - p.BTSQ(0, self.r(y)) - } else { - p.BTSQ(self.r(x), self.r(y)) - } + /* testing and setting the bits at the same time */ + if x == hir.Rz { + p.BTSQ(0, self.r(y)) + } else { + p.BTSQ(self.r(x), self.r(y)) + } - /* set the result if expected */ - if z != hir.Rz { - p.SETC(x86_64.Register8(self.r(z))) - p.ANDL(1, x86_64.Register32(self.r(z))) - } + /* set the result if expected */ + if z != hir.Rz { + p.SETC(x86_64.Register8(self.r(z))) + p.ANDL(1, x86_64.Register32(self.r(z))) + } } func (self *CodeGen) translate_OP_addi(p *x86_64.Program, v *hir.Ir) { - if v.Ry != hir.Rz { - if v.Rx != hir.Rz { - if self.dup(p, v.Rx, v.Ry); v.Iv != 0 { - p.ADDQ(self.i32(p, v), self.r(v.Ry)) - } - } else { - if v.Iv == 0 { - self.clr(p, v.Ry) - } else if !isInt32(v.Iv) { - p.MOVQ(v.Iv, self.r(v.Ry)) - } else { - p.MOVL(v.Iv, x86_64.Register32(self.r(v.Ry))) - } - } - } + if v.Ry != hir.Rz { + if v.Rx != hir.Rz { + if self.dup(p, v.Rx, v.Ry); v.Iv != 0 { + p.ADDQ(self.i32(p, v), self.r(v.Ry)) + } + } else { + if v.Iv == 0 { + self.clr(p, v.Ry) + } else if !isInt32(v.Iv) { + p.MOVQ(v.Iv, self.r(v.Ry)) + } else { + p.MOVL(v.Iv, x86_64.Register32(self.r(v.Ry))) + } + } + } } func (self *CodeGen) translate_OP_muli(p *x86_64.Program, v *hir.Ir) { - var z x86_64.Register - var x x86_64.Register64 - var y x86_64.Register64 - - /* no need to calculate if the result was to be discarded */ - if v.Ry == hir.Rz { - return - } - - /* multiply anything by zero is zero */ - if v.Rx == hir.Rz { - self.clr(p, v.Ry) - return - } - - /* get the allocated registers */ - x = self.r(v.Rx) - y = self.r(v.Ry) - - /* optimized multiplication */ - switch { - case v.Iv == 0: self.clr(p, v.Ry) // x * 0 == 0 - case v.Iv == 1: self.dup(p, v.Rx, v.Ry) // x * 1 == x - - /* multiply by 2, 4 or 8, choose between ADD / SHL and LEA */ - case v.Iv == 2: if x == y { p.ADDQ(x, y) } else { p.LEAQ(Sib(x, x, 1, 0), y) } - case v.Iv == 4: if x == y { p.SHLQ(2, y) } else { p.LEAQ(Sib(z, x, 4, 0), y) } - case v.Iv == 8: if x == y { p.SHLQ(3, y) } else { p.LEAQ(Sib(z, x, 8, 0), y) } - - /* small multipliers, use optimized multiplication algorithm */ - case v.Iv == 3 : p.LEAQ(Sib(x, x, 2, 0), y) // x * 3 == x + x * 2 - case v.Iv == 5 : p.LEAQ(Sib(x, x, 4, 0), y) // x * 5 == x + x * 4 - case v.Iv == 6 : p.LEAQ(Sib(x, x, 2, 0), y); p.ADDQ(y, y) // x * 6 == x * 3 * 2 - case v.Iv == 9 : p.LEAQ(Sib(x, x, 8, 0), y) // x * 9 == x + x * 8 - case v.Iv == 10 : p.LEAQ(Sib(x, x, 4, 0), y); p.ADDQ(y, y) // x * 10 == x * 5 * 2 - case v.Iv == 12 : p.LEAQ(Sib(x, x, 2, 0), y); p.SHLQ(2, y) // x * 12 == x * 3 * 4 - case v.Iv == 15 : p.LEAQ(Sib(x, x, 4, 0), y); p.LEAQ(Sib(y, y, 2, 0), y) // x * 15 == x * 5 * 3 - case v.Iv == 18 : p.LEAQ(Sib(x, x, 8, 0), y); p.ADDQ(y, y) // x * 18 == x * 9 * 2 - case v.Iv == 20 : p.LEAQ(Sib(x, x, 4, 0), y); p.SHLQ(2, y) // x * 20 == x * 5 * 4 - case v.Iv == 24 : p.LEAQ(Sib(x, x, 2, 0), y); p.SHLQ(3, y) // x * 24 == x * 3 * 8 - case v.Iv == 25 : p.LEAQ(Sib(x, x, 4, 0), y); p.LEAQ(Sib(y, y, 4, 0), y) // x * 25 == x * 5 * 5 - case v.Iv == 27 : p.LEAQ(Sib(x, x, 8, 0), y); p.LEAQ(Sib(y, y, 2, 0), y) // x * 27 == x * 9 * 3 - case v.Iv == 36 : p.LEAQ(Sib(x, x, 8, 0), y); p.SHLQ(2, y) // x * 36 == x * 9 * 4 - case v.Iv == 40 : p.LEAQ(Sib(x, x, 4, 0), y); p.SHLQ(3, y) // x * 40 == x * 5 * 8 - case v.Iv == 45 : p.LEAQ(Sib(x, x, 8, 0), y); p.LEAQ(Sib(y, y, 4, 0), y) // x * 45 == x * 9 * 5 - case v.Iv == 48 : p.LEAQ(Sib(x, x, 2, 0), y); p.SHLQ(4, y) // x * 48 == x * 3 * 16 - case v.Iv == 72 : p.LEAQ(Sib(x, x, 8, 0), y); p.SHLQ(3, y) // x * 72 == x * 9 * 8 - case v.Iv == 80 : p.LEAQ(Sib(x, x, 4, 0), y); p.SHLQ(4, y) // x * 80 == x * 5 * 16 - case v.Iv == 81 : p.LEAQ(Sib(x, x, 8, 0), y); p.LEAQ(Sib(y, y, 8, 0), y) // x * 81 == x * 9 * 9 - - /* multiplier is a power of 2, use shifts */ - case isPow2(v.Iv): { - self.dup(p, v.Rx, v.Ry) - p.SHLQ(bits.TrailingZeros64(uint64(v.Iv)), y) - } - - /* multiplier can fit into a 32-bit integer, use 3-operand IMUL instruction */ - case isInt32(v.Iv): { - p.IMULQ(v.Iv, x, y) - } - - /* none of above matches, we need an extra temporary register */ - default: { - self.dup(p, v.Rx, v.Ry) - p.MOVQ(v.Iv, RAX) - p.IMULQ(RAX, y) - } - } + var z x86_64.Register + var x x86_64.Register64 + var y x86_64.Register64 + + /* no need to calculate if the result was to be discarded */ + if v.Ry == hir.Rz { + return + } + + /* multiply anything by zero is zero */ + if v.Rx == hir.Rz { + self.clr(p, v.Ry) + return + } + + /* get the allocated registers */ + x = self.r(v.Rx) + y = self.r(v.Ry) + + /* optimized multiplication */ + switch { + case v.Iv == 0: + self.clr(p, v.Ry) // x * 0 == 0 + case v.Iv == 1: + self.dup(p, v.Rx, v.Ry) // x * 1 == x + + /* multiply by 2, 4 or 8, choose between ADD / SHL and LEA */ + case v.Iv == 2: + if x == y { + p.ADDQ(x, y) + } else { + p.LEAQ(Sib(x, x, 1, 0), y) + } + case v.Iv == 4: + if x == y { + p.SHLQ(2, y) + } else { + p.LEAQ(Sib(z, x, 4, 0), y) + } + case v.Iv == 8: + if x == y { + p.SHLQ(3, y) + } else { + p.LEAQ(Sib(z, x, 8, 0), y) + } + + /* small multipliers, use optimized multiplication algorithm */ + case v.Iv == 3: + p.LEAQ(Sib(x, x, 2, 0), y) // x * 3 == x + x * 2 + case v.Iv == 5: + p.LEAQ(Sib(x, x, 4, 0), y) // x * 5 == x + x * 4 + case v.Iv == 6: + p.LEAQ(Sib(x, x, 2, 0), y) + p.ADDQ(y, y) // x * 6 == x * 3 * 2 + case v.Iv == 9: + p.LEAQ(Sib(x, x, 8, 0), y) // x * 9 == x + x * 8 + case v.Iv == 10: + p.LEAQ(Sib(x, x, 4, 0), y) + p.ADDQ(y, y) // x * 10 == x * 5 * 2 + case v.Iv == 12: + p.LEAQ(Sib(x, x, 2, 0), y) + p.SHLQ(2, y) // x * 12 == x * 3 * 4 + case v.Iv == 15: + p.LEAQ(Sib(x, x, 4, 0), y) + p.LEAQ(Sib(y, y, 2, 0), y) // x * 15 == x * 5 * 3 + case v.Iv == 18: + p.LEAQ(Sib(x, x, 8, 0), y) + p.ADDQ(y, y) // x * 18 == x * 9 * 2 + case v.Iv == 20: + p.LEAQ(Sib(x, x, 4, 0), y) + p.SHLQ(2, y) // x * 20 == x * 5 * 4 + case v.Iv == 24: + p.LEAQ(Sib(x, x, 2, 0), y) + p.SHLQ(3, y) // x * 24 == x * 3 * 8 + case v.Iv == 25: + p.LEAQ(Sib(x, x, 4, 0), y) + p.LEAQ(Sib(y, y, 4, 0), y) // x * 25 == x * 5 * 5 + case v.Iv == 27: + p.LEAQ(Sib(x, x, 8, 0), y) + p.LEAQ(Sib(y, y, 2, 0), y) // x * 27 == x * 9 * 3 + case v.Iv == 36: + p.LEAQ(Sib(x, x, 8, 0), y) + p.SHLQ(2, y) // x * 36 == x * 9 * 4 + case v.Iv == 40: + p.LEAQ(Sib(x, x, 4, 0), y) + p.SHLQ(3, y) // x * 40 == x * 5 * 8 + case v.Iv == 45: + p.LEAQ(Sib(x, x, 8, 0), y) + p.LEAQ(Sib(y, y, 4, 0), y) // x * 45 == x * 9 * 5 + case v.Iv == 48: + p.LEAQ(Sib(x, x, 2, 0), y) + p.SHLQ(4, y) // x * 48 == x * 3 * 16 + case v.Iv == 72: + p.LEAQ(Sib(x, x, 8, 0), y) + p.SHLQ(3, y) // x * 72 == x * 9 * 8 + case v.Iv == 80: + p.LEAQ(Sib(x, x, 4, 0), y) + p.SHLQ(4, y) // x * 80 == x * 5 * 16 + case v.Iv == 81: + p.LEAQ(Sib(x, x, 8, 0), y) + p.LEAQ(Sib(y, y, 8, 0), y) // x * 81 == x * 9 * 9 + + /* multiplier is a power of 2, use shifts */ + case isPow2(v.Iv): + { + self.dup(p, v.Rx, v.Ry) + p.SHLQ(bits.TrailingZeros64(uint64(v.Iv)), y) + } + + /* multiplier can fit into a 32-bit integer, use 3-operand IMUL instruction */ + case isInt32(v.Iv): + { + p.IMULQ(v.Iv, x, y) + } + + /* none of above matches, we need an extra temporary register */ + default: + { + self.dup(p, v.Rx, v.Ry) + p.MOVQ(v.Iv, RAX) + p.IMULQ(RAX, y) + } + } } func (self *CodeGen) translate_OP_andi(p *x86_64.Program, v *hir.Ir) { - if v.Ry != hir.Rz { - if v.Iv == 0 || v.Rx == hir.Rz { - self.clr(p, v.Ry) - } else { - self.dup(p, v.Rx, v.Ry) - p.ANDQ(self.i32(p, v), self.r(v.Ry)) - } - } + if v.Ry != hir.Rz { + if v.Iv == 0 || v.Rx == hir.Rz { + self.clr(p, v.Ry) + } else { + self.dup(p, v.Rx, v.Ry) + p.ANDQ(self.i32(p, v), self.r(v.Ry)) + } + } } func (self *CodeGen) translate_OP_xori(p *x86_64.Program, v *hir.Ir) { - if v.Ry != hir.Rz { - if v.Rx != hir.Rz { - if self.dup(p, v.Rx, v.Ry); v.Iv != 0 { - p.XORQ(self.i32(p, v), self.r(v.Ry)) - } - } else { - if v.Iv == 0 { - self.clr(p, v.Ry) - } else if !isInt32(v.Iv) { - p.MOVQ(v.Iv, self.r(v.Ry)) - } else { - p.MOVL(v.Iv, x86_64.Register32(self.r(v.Ry))) - } - } - } + if v.Ry != hir.Rz { + if v.Rx != hir.Rz { + if self.dup(p, v.Rx, v.Ry); v.Iv != 0 { + p.XORQ(self.i32(p, v), self.r(v.Ry)) + } + } else { + if v.Iv == 0 { + self.clr(p, v.Ry) + } else if !isInt32(v.Iv) { + p.MOVQ(v.Iv, self.r(v.Ry)) + } else { + p.MOVL(v.Iv, x86_64.Register32(self.r(v.Ry))) + } + } + } } func (self *CodeGen) translate_OP_shri(p *x86_64.Program, v *hir.Ir) { - if v.Ry != hir.Rz { - if v.Iv < 0 { - panic("shri: negative bit count") - } else if v.Iv >= 64 || v.Rx == hir.Rz { - p.XORL(self.r(v.Ry), self.r(v.Ry)) - } else if self.dup(p, v.Rx, v.Ry); v.Iv != 0 { - p.SHRQ(v.Iv, self.r(v.Ry)) - } - } + if v.Ry != hir.Rz { + if v.Iv < 0 { + panic("shri: negative bit count") + } else if v.Iv >= 64 || v.Rx == hir.Rz { + p.XORL(self.r(v.Ry), self.r(v.Ry)) + } else if self.dup(p, v.Rx, v.Ry); v.Iv != 0 { + p.SHRQ(v.Iv, self.r(v.Ry)) + } + } } func (self *CodeGen) translate_OP_bsi(p *x86_64.Program, v *hir.Ir) { - if v.Ry != hir.Rz { - if v.Iv < 0 { - panic("sbiti: negative bit index") - } else if v.Rx == hir.Rz { - self.set(p, v.Ry, 1 << v.Iv) - } else if self.dup(p, v.Rx, v.Ry); v.Iv < 31 { - p.ORQ(1 << v.Iv, self.r(v.Ry)) - } else if v.Iv < 64 { - p.BTSQ(v.Iv, self.r(v.Ry)) - } - } + if v.Ry != hir.Rz { + if v.Iv < 0 { + panic("sbiti: negative bit index") + } else if v.Rx == hir.Rz { + self.set(p, v.Ry, 1< RAX since R11 will be clobbered by gcWriteBarrier2 */ - p.MOVQ(uintptr(rtx.F_gcWriteBarrier2), RSI) - p.CALLQ(RSI) /* apply 2 slots and save the beginning address in R11 */ - p.XCHGQ(RAX, R11) /* Restore R11 <- RAX, and save R11(slotAddr) -> RAX */ - self.abiSaveReserved(p) - self.abiRestoreReserved(p) - wbStoreNewPointerForGC(RAX) /* MOV r(s), 0[R11] */ - wbStoreOldPointerForGC(RAX) /* MOV RDI, 8[R11] */ - p.JMP(rt) - } + /* write barrier wrapper */ + wbStoreFn := func(p *x86_64.Program) { + self.abiSpillReserved(p) + self.abiLoadReserved(p) + p.MOVQ(R11, RAX) /* Save R11 -> RAX since R11 will be clobbered by gcWriteBarrier2 */ + p.MOVQ(uintptr(rtx.F_gcWriteBarrier2), RSI) + p.CALLQ(RSI) /* apply 2 slots and save the beginning address in R11 */ + p.XCHGQ(RAX, R11) /* Restore R11 <- RAX, and save R11(slotAddr) -> RAX */ + self.abiSaveReserved(p) + self.abiRestoreReserved(p) + wbStoreNewPointerForGC(RAX) /* MOV r(s), 0[R11] */ + wbStoreOldPointerForGC(RAX) /* MOV RDI, 8[R11] */ + p.JMP(rt) + } - /* defer the call to the end of generated code */ - p.Link(rt) - wbUpdatePointer() /* Need to do it in go1.21+ both for direct_store or wb_store */ - self.later(wb, wbStoreFn) + /* defer the call to the end of generated code */ + p.Link(rt) + wbUpdatePointer() /* Need to do it in go1.21+ both for direct_store or wb_store */ + self.later(wb, wbStoreFn) } diff --git a/internal/atm/pgen/pgen_legacy_amd64.go b/internal/atm/pgen/pgen_legacy_amd64.go index 21025fb..04adc34 100644 --- a/internal/atm/pgen/pgen_legacy_amd64.go +++ b/internal/atm/pgen/pgen_legacy_amd64.go @@ -1,7 +1,8 @@ +//go:build !go1.17 // +build !go1.17 /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,62 +20,65 @@ package pgen import ( - `runtime` + "runtime" - `github.com/cloudwego/iasm/x86_64` - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/atm/rtx` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/atm/rtx" + "github.com/cloudwego/iasm/x86_64" ) /** Stack Checking **/ const ( - _M_memcpyargs = 24 - _G_stackguard0 = 0x10 + _M_memcpyargs = 24 + _G_stackguard0 = 0x10 ) func (self *CodeGen) abiStackCheck(p *x86_64.Program, to *x86_64.Label, sp uintptr) { - ctxt := self.ctxt - size := ctxt.size() + int32(sp) + ctxt := self.ctxt + size := ctxt.size() + int32(sp) - /* get the current goroutine */ - switch runtime.GOOS { - case "linux" : p.MOVQ (Abs(-8), RCX).FS() - case "darwin" : p.MOVQ (Abs(0x30), RCX).GS() - default : panic("unsupported operating system") - } + /* get the current goroutine */ + switch runtime.GOOS { + case "linux": + p.MOVQ(Abs(-8), RCX).FS() + case "darwin": + p.MOVQ(Abs(0x30), RCX).GS() + default: + panic("unsupported operating system") + } - /* check the stack guard */ - p.LEAQ (Ptr(RSP, -size), RAX) - p.CMPQ (Ptr(RCX, _G_stackguard0), RAX) - p.JBE (to) + /* check the stack guard */ + p.LEAQ(Ptr(RSP, -size), RAX) + p.CMPQ(Ptr(RCX, _G_stackguard0), RAX) + p.JBE(to) } /** Efficient Block Copy Algorithm **/ func (self *CodeGen) abiBlockCopy(p *x86_64.Program, pd hir.PointerRegister, ps hir.PointerRegister, nb hir.GenericRegister) { - rd := self.r(pd) - rs := self.r(ps) - rl := self.r(nb) + rd := self.r(pd) + rs := self.r(ps) + rl := self.r(nb) - /* save all the registers, if they will be clobbered */ - for _, lr := range self.ctxt.regs { - if rr := self.r(lr); rtx.R_memmove[rr] { - p.MOVQ(rr, self.ctxt.slot(lr)) - } - } + /* save all the registers, if they will be clobbered */ + for _, lr := range self.ctxt.regs { + if rr := self.r(lr); rtx.R_memmove[rr] { + p.MOVQ(rr, self.ctxt.slot(lr)) + } + } - /* load the args and call the function */ - p.MOVQ(rd, Ptr(RSP, 0)) - p.MOVQ(rs, Ptr(RSP, 8)) - p.MOVQ(rl, Ptr(RSP, 16)) - p.MOVQ(uintptr(rtx.F_memmove), RDI) - p.CALLQ(RDI) + /* load the args and call the function */ + p.MOVQ(rd, Ptr(RSP, 0)) + p.MOVQ(rs, Ptr(RSP, 8)) + p.MOVQ(rl, Ptr(RSP, 16)) + p.MOVQ(uintptr(rtx.F_memmove), RDI) + p.CALLQ(RDI) - /* restore all the registers, if they were clobbered */ - for _, lr := range self.ctxt.regs { - if rr := self.r(lr); rtx.R_memmove[rr] { - p.MOVQ(self.ctxt.slot(lr), rr) - } - } + /* restore all the registers, if they were clobbered */ + for _, lr := range self.ctxt.regs { + if rr := self.r(lr); rtx.R_memmove[rr] { + p.MOVQ(self.ctxt.slot(lr), rr) + } + } } diff --git a/internal/atm/pgen/pgen_regabi_amd64.go b/internal/atm/pgen/pgen_regabi_amd64.go index 04a8de2..a9ea87d 100644 --- a/internal/atm/pgen/pgen_regabi_amd64.go +++ b/internal/atm/pgen/pgen_regabi_amd64.go @@ -1,7 +1,8 @@ +//go:build go1.17 // +build go1.17 /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,63 +20,79 @@ package pgen import ( - `github.com/cloudwego/iasm/x86_64` - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/atm/rtx` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/atm/rtx" + "github.com/cloudwego/iasm/x86_64" ) /** Stack Checking **/ const ( - _M_memcpyargs = 0 - _G_stackguard0 = 0x10 + _M_memcpyargs = 0 + _G_stackguard0 = 0x10 ) func (self *CodeGen) abiStackCheck(p *x86_64.Program, to *x86_64.Label, sp uintptr) { - p.LEAQ (Ptr(RSP, -self.ctxt.size() - int32(sp)), R12) - p.CMPQ (Ptr(R14, _G_stackguard0), R12) - p.JBE (to) + p.LEAQ(Ptr(RSP, -self.ctxt.size()-int32(sp)), R12) + p.CMPQ(Ptr(R14, _G_stackguard0), R12) + p.JBE(to) } /** Efficient Block Copy Algorithm **/ -var memcpyargs = [256]bool { - RAX: true, - RBX: true, - RCX: true, +var memcpyargs = [256]bool{ + RAX: true, + RBX: true, + RCX: true, } func (self *CodeGen) abiBlockCopy(p *x86_64.Program, pd hir.PointerRegister, ps hir.PointerRegister, nb hir.GenericRegister) { - rd := self.r(pd) - rs := self.r(ps) - rl := self.r(nb) + rd := self.r(pd) + rs := self.r(ps) + rl := self.r(nb) - /* save all the registers, if they will be clobbered */ - for _, lr := range self.ctxt.regs { - if rr := self.r(lr); rtx.R_memmove[rr] || memcpyargs[rr] { - p.MOVQ(rr, self.ctxt.slot(lr)) - } - } + /* save all the registers, if they will be clobbered */ + for _, lr := range self.ctxt.regs { + if rr := self.r(lr); rtx.R_memmove[rr] || memcpyargs[rr] { + p.MOVQ(rr, self.ctxt.slot(lr)) + } + } - /* enumerate different register cases */ - switch { - case rs == RBX && rl == RCX : p.MOVQ(rd, RAX) - case rs == RBX && rl != RCX : p.MOVQ(rd, RAX); p.MOVQ (rl, RCX) - case rs != RBX && rl == RCX : p.MOVQ(rd, RAX); p.MOVQ (rs, RBX) - case rs == RCX && rl == RBX : p.MOVQ(rd, RAX); p.XCHGQ (RBX, RCX) - case rs == RCX && rl != RBX : p.MOVQ(rd, RAX); p.MOVQ (RCX, RBX); p.MOVQ(rl, RCX) - case rs != RCX && rl == RBX : p.MOVQ(rd, RAX); p.MOVQ (RBX, RCX); p.MOVQ(rs, RBX) - default : p.MOVQ(rd, RAX); p.MOVQ (rs, RBX); p.MOVQ(rl, RCX) - } + /* enumerate different register cases */ + switch { + case rs == RBX && rl == RCX: + p.MOVQ(rd, RAX) + case rs == RBX && rl != RCX: + p.MOVQ(rd, RAX) + p.MOVQ(rl, RCX) + case rs != RBX && rl == RCX: + p.MOVQ(rd, RAX) + p.MOVQ(rs, RBX) + case rs == RCX && rl == RBX: + p.MOVQ(rd, RAX) + p.XCHGQ(RBX, RCX) + case rs == RCX && rl != RBX: + p.MOVQ(rd, RAX) + p.MOVQ(RCX, RBX) + p.MOVQ(rl, RCX) + case rs != RCX && rl == RBX: + p.MOVQ(rd, RAX) + p.MOVQ(RBX, RCX) + p.MOVQ(rs, RBX) + default: + p.MOVQ(rd, RAX) + p.MOVQ(rs, RBX) + p.MOVQ(rl, RCX) + } - /* call the function */ - p.MOVQ(uintptr(rtx.F_memmove), RDI) - p.CALLQ(RDI) + /* call the function */ + p.MOVQ(uintptr(rtx.F_memmove), RDI) + p.CALLQ(RDI) - /* restore all the registers, if they were clobbered */ - for _, lr := range self.ctxt.regs { - if rr := self.r(lr); rtx.R_memmove[rr] || memcpyargs[rr] { - p.MOVQ(self.ctxt.slot(lr), rr) - } - } + /* restore all the registers, if they were clobbered */ + for _, lr := range self.ctxt.regs { + if rr := self.r(lr); rtx.R_memmove[rr] || memcpyargs[rr] { + p.MOVQ(self.ctxt.slot(lr), rr) + } + } } diff --git a/internal/atm/rtx/asm.s b/internal/atm/rtx/asm.s index a8c6cc1..9c99547 100644 --- a/internal/atm/rtx/asm.s +++ b/internal/atm/rtx/asm.s @@ -1,4 +1,4 @@ -// Copyright 2022 ByteDance Inc. +// Copyright 2022 CloudWeGo Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/internal/atm/rtx/clobber_amd64.go b/internal/atm/rtx/clobber_amd64.go index c5c7af2..5a06883 100644 --- a/internal/atm/rtx/clobber_amd64.go +++ b/internal/atm/rtx/clobber_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,209 +17,209 @@ package rtx import ( - `unsafe` + "unsafe" - `github.com/cloudwego/iasm/x86_64` - `github.com/cloudwego/frugal/internal/rt` - `github.com/oleiade/lane` - `golang.org/x/arch/x86/x86asm` + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/iasm/x86_64" + "github.com/oleiade/lane" + "golang.org/x/arch/x86/x86asm" ) -var branchTable = map[x86asm.Op]bool { - x86asm.JA : true, - x86asm.JAE : true, - x86asm.JB : true, - x86asm.JBE : true, - x86asm.JCXZ : true, - x86asm.JE : true, - x86asm.JECXZ : true, - x86asm.JG : true, - x86asm.JGE : true, - x86asm.JL : true, - x86asm.JLE : true, - x86asm.JMP : true, - x86asm.JNE : true, - x86asm.JNO : true, - x86asm.JNP : true, - x86asm.JNS : true, - x86asm.JO : true, - x86asm.JP : true, - x86asm.JRCXZ : true, - x86asm.JS : true, +var branchTable = map[x86asm.Op]bool{ + x86asm.JA: true, + x86asm.JAE: true, + x86asm.JB: true, + x86asm.JBE: true, + x86asm.JCXZ: true, + x86asm.JE: true, + x86asm.JECXZ: true, + x86asm.JG: true, + x86asm.JGE: true, + x86asm.JL: true, + x86asm.JLE: true, + x86asm.JMP: true, + x86asm.JNE: true, + x86asm.JNO: true, + x86asm.JNP: true, + x86asm.JNS: true, + x86asm.JO: true, + x86asm.JP: true, + x86asm.JRCXZ: true, + x86asm.JS: true, } -var registerTable = map[x86asm.Reg]x86_64.Register64 { - x86asm.AL : x86_64.RAX, - x86asm.CL : x86_64.RCX, - x86asm.DL : x86_64.RDX, - x86asm.BL : x86_64.RBX, - x86asm.AH : x86_64.RAX, - x86asm.CH : x86_64.RCX, - x86asm.DH : x86_64.RDX, - x86asm.BH : x86_64.RBX, - x86asm.SPB : x86_64.RSP, - x86asm.BPB : x86_64.RBP, - x86asm.SIB : x86_64.RSI, - x86asm.DIB : x86_64.RDI, - x86asm.R8B : x86_64.R8, - x86asm.R9B : x86_64.R9, - x86asm.R10B : x86_64.R10, - x86asm.R11B : x86_64.R11, - x86asm.R12B : x86_64.R12, - x86asm.R13B : x86_64.R13, - x86asm.R14B : x86_64.R14, - x86asm.R15B : x86_64.R15, - x86asm.AX : x86_64.RAX, - x86asm.CX : x86_64.RCX, - x86asm.DX : x86_64.RDX, - x86asm.BX : x86_64.RBX, - x86asm.SP : x86_64.RSP, - x86asm.BP : x86_64.RBP, - x86asm.SI : x86_64.RSI, - x86asm.DI : x86_64.RDI, - x86asm.R8W : x86_64.R8, - x86asm.R9W : x86_64.R9, - x86asm.R10W : x86_64.R10, - x86asm.R11W : x86_64.R11, - x86asm.R12W : x86_64.R12, - x86asm.R13W : x86_64.R13, - x86asm.R14W : x86_64.R14, - x86asm.R15W : x86_64.R15, - x86asm.EAX : x86_64.RAX, - x86asm.ECX : x86_64.RCX, - x86asm.EDX : x86_64.RDX, - x86asm.EBX : x86_64.RBX, - x86asm.ESP : x86_64.RSP, - x86asm.EBP : x86_64.RBP, - x86asm.ESI : x86_64.RSI, - x86asm.EDI : x86_64.RDI, - x86asm.R8L : x86_64.R8, - x86asm.R9L : x86_64.R9, - x86asm.R10L : x86_64.R10, - x86asm.R11L : x86_64.R11, - x86asm.R12L : x86_64.R12, - x86asm.R13L : x86_64.R13, - x86asm.R14L : x86_64.R14, - x86asm.R15L : x86_64.R15, - x86asm.RAX : x86_64.RAX, - x86asm.RCX : x86_64.RCX, - x86asm.RDX : x86_64.RDX, - x86asm.RBX : x86_64.RBX, - x86asm.RSP : x86_64.RSP, - x86asm.RBP : x86_64.RBP, - x86asm.RSI : x86_64.RSI, - x86asm.RDI : x86_64.RDI, - x86asm.R8 : x86_64.R8, - x86asm.R9 : x86_64.R9, - x86asm.R10 : x86_64.R10, - x86asm.R11 : x86_64.R11, - x86asm.R12 : x86_64.R12, - x86asm.R13 : x86_64.R13, - x86asm.R14 : x86_64.R14, - x86asm.R15 : x86_64.R15, +var registerTable = map[x86asm.Reg]x86_64.Register64{ + x86asm.AL: x86_64.RAX, + x86asm.CL: x86_64.RCX, + x86asm.DL: x86_64.RDX, + x86asm.BL: x86_64.RBX, + x86asm.AH: x86_64.RAX, + x86asm.CH: x86_64.RCX, + x86asm.DH: x86_64.RDX, + x86asm.BH: x86_64.RBX, + x86asm.SPB: x86_64.RSP, + x86asm.BPB: x86_64.RBP, + x86asm.SIB: x86_64.RSI, + x86asm.DIB: x86_64.RDI, + x86asm.R8B: x86_64.R8, + x86asm.R9B: x86_64.R9, + x86asm.R10B: x86_64.R10, + x86asm.R11B: x86_64.R11, + x86asm.R12B: x86_64.R12, + x86asm.R13B: x86_64.R13, + x86asm.R14B: x86_64.R14, + x86asm.R15B: x86_64.R15, + x86asm.AX: x86_64.RAX, + x86asm.CX: x86_64.RCX, + x86asm.DX: x86_64.RDX, + x86asm.BX: x86_64.RBX, + x86asm.SP: x86_64.RSP, + x86asm.BP: x86_64.RBP, + x86asm.SI: x86_64.RSI, + x86asm.DI: x86_64.RDI, + x86asm.R8W: x86_64.R8, + x86asm.R9W: x86_64.R9, + x86asm.R10W: x86_64.R10, + x86asm.R11W: x86_64.R11, + x86asm.R12W: x86_64.R12, + x86asm.R13W: x86_64.R13, + x86asm.R14W: x86_64.R14, + x86asm.R15W: x86_64.R15, + x86asm.EAX: x86_64.RAX, + x86asm.ECX: x86_64.RCX, + x86asm.EDX: x86_64.RDX, + x86asm.EBX: x86_64.RBX, + x86asm.ESP: x86_64.RSP, + x86asm.EBP: x86_64.RBP, + x86asm.ESI: x86_64.RSI, + x86asm.EDI: x86_64.RDI, + x86asm.R8L: x86_64.R8, + x86asm.R9L: x86_64.R9, + x86asm.R10L: x86_64.R10, + x86asm.R11L: x86_64.R11, + x86asm.R12L: x86_64.R12, + x86asm.R13L: x86_64.R13, + x86asm.R14L: x86_64.R14, + x86asm.R15L: x86_64.R15, + x86asm.RAX: x86_64.RAX, + x86asm.RCX: x86_64.RCX, + x86asm.RDX: x86_64.RDX, + x86asm.RBX: x86_64.RBX, + x86asm.RSP: x86_64.RSP, + x86asm.RBP: x86_64.RBP, + x86asm.RSI: x86_64.RSI, + x86asm.RDI: x86_64.RDI, + x86asm.R8: x86_64.R8, + x86asm.R9: x86_64.R9, + x86asm.R10: x86_64.R10, + x86asm.R11: x86_64.R11, + x86asm.R12: x86_64.R12, + x86asm.R13: x86_64.R13, + x86asm.R14: x86_64.R14, + x86asm.R15: x86_64.R15, } -var freeRegisters = map[x86_64.Register64]bool { - x86_64.RAX: true, - x86_64.RSI: true, - x86_64.RDI: true, +var freeRegisters = map[x86_64.Register64]bool{ + x86_64.RAX: true, + x86_64.RSI: true, + x86_64.RDI: true, } type _InstrBlock struct { - ret bool - size uintptr - entry unsafe.Pointer - links [2]*_InstrBlock + ret bool + size uintptr + entry unsafe.Pointer + links [2]*_InstrBlock } func newInstrBlock(entry unsafe.Pointer) *_InstrBlock { - return &_InstrBlock{entry: entry} + return &_InstrBlock{entry: entry} } func (self *_InstrBlock) pc() unsafe.Pointer { - return unsafe.Pointer(uintptr(self.entry) + self.size) + return unsafe.Pointer(uintptr(self.entry) + self.size) } func (self *_InstrBlock) code() []byte { - return rt.BytesFrom(self.pc(), 15, 15) + return rt.BytesFrom(self.pc(), 15, 15) } func (self *_InstrBlock) commit(size int) { - self.size += uintptr(size) + self.size += uintptr(size) } func resolveClobberSet(fn interface{}) map[x86_64.Register64]bool { - buf := lane.NewQueue() - ret := make(map[x86_64.Register64]bool) - bmp := make(map[unsafe.Pointer]*_InstrBlock) - - /* build the CFG with BFS */ - for buf.Enqueue(newInstrBlock(rt.FuncAddr(fn))); !buf.Empty(); { - val := buf.Dequeue() - cfg := val.(*_InstrBlock) - - /* parse every instruction in the block */ - for !cfg.ret { - var err error - var ins x86asm.Inst - - /* decode one instruction */ - if ins, err = x86asm.Decode(cfg.code(), 64); err != nil { - panic(err) - } else { - cfg.commit(ins.Len) - } - - /* calling to other functions, cannot analyze */ - if ins.Op == x86asm.CALL { - return nil - } - - /* simple algorithm: every write to register is treated as clobbering */ - if ins.Op == x86asm.MOV { - if reg, ok := ins.Args[0].(x86asm.Reg); ok { - if rr, rok := registerTable[reg]; rok && !freeRegisters[rr] { - ret[rr] = true - } - } - } - - /* check for returns */ - if ins.Op == x86asm.RET { - cfg.ret = true - break - } - - /* check for branches */ - if !branchTable[ins.Op] { - continue - } - - /* calculate branch address */ - links := [2]unsafe.Pointer { - cfg.pc(), - unsafe.Pointer(uintptr(cfg.pc()) + uintptr(ins.Args[0].(x86asm.Rel))), - } - - /* link the next blocks */ - for i := 0; i < 2; i++ { - if cfg.links[i] = bmp[links[i]]; cfg.links[i] == nil { - cfg.links[i] = newInstrBlock(links[i]) - bmp[links[i]] = cfg.links[i] - } - } - - /* add the branches if not returned, if either one returns, mark the block returned */ - for i := 0; i < 2; i++ { - if cfg.links[i].ret { - cfg.ret = true - } else { - buf.Enqueue(cfg.links[i]) - } - } - } - } - - /* all done */ - return ret + buf := lane.NewQueue() + ret := make(map[x86_64.Register64]bool) + bmp := make(map[unsafe.Pointer]*_InstrBlock) + + /* build the CFG with BFS */ + for buf.Enqueue(newInstrBlock(rt.FuncAddr(fn))); !buf.Empty(); { + val := buf.Dequeue() + cfg := val.(*_InstrBlock) + + /* parse every instruction in the block */ + for !cfg.ret { + var err error + var ins x86asm.Inst + + /* decode one instruction */ + if ins, err = x86asm.Decode(cfg.code(), 64); err != nil { + panic(err) + } else { + cfg.commit(ins.Len) + } + + /* calling to other functions, cannot analyze */ + if ins.Op == x86asm.CALL { + return nil + } + + /* simple algorithm: every write to register is treated as clobbering */ + if ins.Op == x86asm.MOV { + if reg, ok := ins.Args[0].(x86asm.Reg); ok { + if rr, rok := registerTable[reg]; rok && !freeRegisters[rr] { + ret[rr] = true + } + } + } + + /* check for returns */ + if ins.Op == x86asm.RET { + cfg.ret = true + break + } + + /* check for branches */ + if !branchTable[ins.Op] { + continue + } + + /* calculate branch address */ + links := [2]unsafe.Pointer{ + cfg.pc(), + unsafe.Pointer(uintptr(cfg.pc()) + uintptr(ins.Args[0].(x86asm.Rel))), + } + + /* link the next blocks */ + for i := 0; i < 2; i++ { + if cfg.links[i] = bmp[links[i]]; cfg.links[i] == nil { + cfg.links[i] = newInstrBlock(links[i]) + bmp[links[i]] = cfg.links[i] + } + } + + /* add the branches if not returned, if either one returns, mark the block returned */ + for i := 0; i < 2; i++ { + if cfg.links[i].ret { + cfg.ret = true + } else { + buf.Enqueue(cfg.links[i]) + } + } + } + } + + /* all done */ + return ret } diff --git a/internal/atm/rtx/clobber_amd64_test.go b/internal/atm/rtx/clobber_amd64_test.go index a444943..0c94a24 100644 --- a/internal/atm/rtx/clobber_amd64_test.go +++ b/internal/atm/rtx/clobber_amd64_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,11 +17,11 @@ package rtx import ( - `testing` + "testing" - `github.com/davecgh/go-spew/spew` + "github.com/davecgh/go-spew/spew" ) func TestClobber_ClobberSet(t *testing.T) { - spew.Dump(R_memmove) + spew.Dump(R_memmove) } diff --git a/internal/atm/rtx/gcwb_go116_120.go b/internal/atm/rtx/gcwb_go116_120.go index 82c3e49..b66bcbb 100644 --- a/internal/atm/rtx/gcwb_go116_120.go +++ b/internal/atm/rtx/gcwb_go116_120.go @@ -2,7 +2,7 @@ // +build !go1.21 /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,9 +20,9 @@ package rtx import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) //go:linkname writeBarrier runtime.writeBarrier @@ -33,6 +33,6 @@ var writeBarrier uintptr func gcWriteBarrier() var ( - V_pWriteBarrier = unsafe.Pointer(&writeBarrier) - F_gcWriteBarrier = rt.FuncAddr(gcWriteBarrier) + V_pWriteBarrier = unsafe.Pointer(&writeBarrier) + F_gcWriteBarrier = rt.FuncAddr(gcWriteBarrier) ) diff --git a/internal/atm/rtx/gcwb_go121_121.go b/internal/atm/rtx/gcwb_go121_121.go index 700a598..ea0a272 100644 --- a/internal/atm/rtx/gcwb_go121_121.go +++ b/internal/atm/rtx/gcwb_go121_121.go @@ -2,7 +2,7 @@ // +build go1.21 /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,9 +20,9 @@ package rtx import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) //go:linkname writeBarrier runtime.writeBarrier @@ -33,12 +33,12 @@ var writeBarrier uintptr func gcWriteBarrier2() func gcWriteBarrier() { - // obsoleted in go1.21+, but it's referenced by ssa and we're not going to update - // ssa at the moment, we just leave an empty function here + // obsoleted in go1.21+, but it's referenced by ssa and we're not going to update + // ssa at the moment, we just leave an empty function here } var ( - V_pWriteBarrier = unsafe.Pointer(&writeBarrier) - F_gcWriteBarrier = rt.FuncAddr(gcWriteBarrier) // referenced by ssa (but not used) - F_gcWriteBarrier2 = rt.FuncAddr(gcWriteBarrier2) + V_pWriteBarrier = unsafe.Pointer(&writeBarrier) + F_gcWriteBarrier = rt.FuncAddr(gcWriteBarrier) // referenced by ssa (but not used) + F_gcWriteBarrier2 = rt.FuncAddr(gcWriteBarrier2) ) diff --git a/internal/atm/rtx/memcpy.go b/internal/atm/rtx/memcpy.go index d32027d..5e1dfd3 100644 --- a/internal/atm/rtx/memcpy.go +++ b/internal/atm/rtx/memcpy.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,11 +17,11 @@ package rtx import ( - `reflect` - `unsafe` + "reflect" + "unsafe" - `github.com/cloudwego/frugal/internal/atm/abi` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/abi" + "github.com/cloudwego/frugal/internal/rt" ) //go:noescape @@ -30,7 +30,7 @@ import ( func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) var ( - F_memmove = rt.FuncAddr(memmove) - R_memmove = resolveClobberSet(memmove) - S_memmove = abi.ABI.LayoutFunc(-1, reflect.TypeOf(memmove)) + F_memmove = rt.FuncAddr(memmove) + R_memmove = resolveClobberSet(memmove) + S_memmove = abi.ABI.LayoutFunc(-1, reflect.TypeOf(memmove)) ) diff --git a/internal/atm/rtx/memzero.go b/internal/atm/rtx/memzero.go index b910e17..eaa7e4b 100644 --- a/internal/atm/rtx/memzero.go +++ b/internal/atm/rtx/memzero.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,18 +17,18 @@ package rtx import ( - `unsafe` + "unsafe" ) type MemZeroFn struct { - Fn unsafe.Pointer - Sz []uintptr + Fn unsafe.Pointer + Sz []uintptr } var ( - MemZero = asmmemzero() + MemZero = asmmemzero() ) func (self MemZeroFn) ForSize(n uintptr) unsafe.Pointer { - return unsafe.Pointer(uintptr(self.Fn) + self.Sz[n / ZeroStep]) + return unsafe.Pointer(uintptr(self.Fn) + self.Sz[n/ZeroStep]) } diff --git a/internal/atm/rtx/memzero_amd64.go b/internal/atm/rtx/memzero_amd64.go index 9d951a1..57d1395 100644 --- a/internal/atm/rtx/memzero_amd64.go +++ b/internal/atm/rtx/memzero_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,59 +17,59 @@ package rtx import ( - `fmt` - `unsafe` + "fmt" + "unsafe" - `github.com/cloudwego/iasm/x86_64` - `github.com/cloudwego/frugal/internal/loader` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/loader" + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/iasm/x86_64" ) const ( - ZeroStep = 16 - MaxZeroSize = 65536 + ZeroStep = 16 + MaxZeroSize = 65536 ) func toaddr(p *x86_64.Label) uintptr { - if v, err := p.Evaluate(); err != nil { - panic(err) - } else { - return uintptr(v) - } + if v, err := p.Evaluate(); err != nil { + panic(err) + } else { + return uintptr(v) + } } func asmmemzero() MemZeroFn { - p := x86_64.DefaultArch.CreateProgram() - x := make([]*x86_64.Label, MaxZeroSize / ZeroStep + 1) + p := x86_64.DefaultArch.CreateProgram() + x := make([]*x86_64.Label, MaxZeroSize/ZeroStep+1) - /* create all the labels */ - for i := range x { - x[i] = x86_64.CreateLabel(fmt.Sprintf("zero_%d", i * ZeroStep)) - } + /* create all the labels */ + for i := range x { + x[i] = x86_64.CreateLabel(fmt.Sprintf("zero_%d", i*ZeroStep)) + } - /* fill backwards */ - for n := MaxZeroSize; n >= ZeroStep; n -= ZeroStep { - p.Link(x[n / ZeroStep]) - p.MOVDQU(x86_64.XMM15, x86_64.Ptr(x86_64.RDI, int32(n - ZeroStep))) - } + /* fill backwards */ + for n := MaxZeroSize; n >= ZeroStep; n -= ZeroStep { + p.Link(x[n/ZeroStep]) + p.MOVDQU(x86_64.XMM15, x86_64.Ptr(x86_64.RDI, int32(n-ZeroStep))) + } - /* finish the function */ - p.Link(x[0]) - p.RET() + /* finish the function */ + p.Link(x[0]) + p.RET() - /* assemble the function */ - c := p.Assemble(0) - r := make([]uintptr, len(x)) + /* assemble the function */ + c := p.Assemble(0) + r := make([]uintptr, len(x)) - /* resolve all the labels */ - for i, v := range x { - r[i] = toaddr(v) - } + /* resolve all the labels */ + for i, v := range x { + r[i] = toaddr(v) + } - /* load the function */ - defer p.Free() - return MemZeroFn { - Sz: r, - Fn: *(*unsafe.Pointer)(loader.Loader(c).Load("_frugal_memzero", rt.Frame{})), - } + /* load the function */ + defer p.Free() + return MemZeroFn{ + Sz: r, + Fn: *(*unsafe.Pointer)(loader.Loader(c).Load("_frugal_memzero", rt.Frame{})), + } } diff --git a/internal/atm/rtx/memzero_test.go b/internal/atm/rtx/memzero_test.go index d28be4d..f8988d5 100644 --- a/internal/atm/rtx/memzero_test.go +++ b/internal/atm/rtx/memzero_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,14 +17,14 @@ package rtx import ( - `fmt` - `math/rand` - `testing` - `unsafe` + "fmt" + "math/rand" + "testing" + "unsafe" - `github.com/cloudwego/frugal/internal/rt` - `github.com/davecgh/go-spew/spew` - `golang.org/x/arch/x86/x86asm` + "github.com/cloudwego/frugal/internal/rt" + "github.com/davecgh/go-spew/spew" + "golang.org/x/arch/x86/x86asm" ) //go:nosplit @@ -33,29 +33,29 @@ import ( func callnative1(fn unsafe.Pointer, a0 uintptr) func disasmfunc(p unsafe.Pointer) { - pc := uintptr(0) - for { - pp := unsafe.Pointer(uintptr(p) + pc) - ins, err := x86asm.Decode(rt.BytesFrom(pp, 15, 15), 64) - if err != nil { - panic(err) - } - fmt.Printf("%#x(%d) %s\n", uintptr(pp), ins.Len, x86asm.GNUSyntax(ins, uint64(uintptr(pp)), nil)) - pc += uintptr(ins.Len) - if ins.Op == x86asm.RET { - break - } - } + pc := uintptr(0) + for { + pp := unsafe.Pointer(uintptr(p) + pc) + ins, err := x86asm.Decode(rt.BytesFrom(pp, 15, 15), 64) + if err != nil { + panic(err) + } + fmt.Printf("%#x(%d) %s\n", uintptr(pp), ins.Len, x86asm.GNUSyntax(ins, uint64(uintptr(pp)), nil)) + pc += uintptr(ins.Len) + if ins.Op == x86asm.RET { + break + } + } } func zeromemsized(p unsafe.Pointer, n uintptr) { - callnative1(unsafe.Pointer(uintptr(MemZero.Fn) + MemZero.Sz[n / ZeroStep]), uintptr(p)) + callnative1(unsafe.Pointer(uintptr(MemZero.Fn)+MemZero.Sz[n/ZeroStep]), uintptr(p)) } func TestMemZero_Clear(t *testing.T) { - mm := make([]byte, 256) - rand.Read(mm) - zeromemsized(unsafe.Pointer(&mm[0]), 48) - spew.Dump(mm) - disasmfunc(MemZero.Fn) + mm := make([]byte, 256) + rand.Read(mm) + zeromemsized(unsafe.Pointer(&mm[0]), 48) + spew.Dump(mm) + disasmfunc(MemZero.Fn) } diff --git a/internal/atm/rtx/stack.go b/internal/atm/rtx/stack.go index 6ad5c1e..6fa8be6 100644 --- a/internal/atm/rtx/stack.go +++ b/internal/atm/rtx/stack.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,14 +17,14 @@ package rtx import ( - _ `unsafe` + _ "unsafe" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) //go:linkname morestack_noctxt runtime.morestack_noctxt func morestack_noctxt() var ( - F_morestack_noctxt = rt.FuncAddr(morestack_noctxt) + F_morestack_noctxt = rt.FuncAddr(morestack_noctxt) ) diff --git a/internal/atm/ssa/block.go b/internal/atm/ssa/block.go index dc44a6f..fadd8d8 100644 --- a/internal/atm/ssa/block.go +++ b/internal/atm/ssa/block.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,516 +17,551 @@ package ssa import ( - `fmt` + "fmt" - `github.com/cloudwego/frugal/internal/atm/abi` - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/atm/rtx` + "github.com/cloudwego/frugal/internal/atm/abi" + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/atm/rtx" ) -var _MemSize = [...]uint8 { - hir.OP_lb: 1, - hir.OP_lw: 2, - hir.OP_ll: 4, - hir.OP_lq: 8, - hir.OP_sb: 1, - hir.OP_sw: 2, - hir.OP_sl: 4, - hir.OP_sq: 8, +var _MemSize = [...]uint8{ + hir.OP_lb: 1, + hir.OP_lw: 2, + hir.OP_ll: 4, + hir.OP_lq: 8, + hir.OP_sb: 1, + hir.OP_sw: 2, + hir.OP_sl: 4, + hir.OP_sq: 8, } -var _UnaryOps = [...]IrUnaryOp { - hir.OP_swapw : IrOpSwap16, - hir.OP_swapl : IrOpSwap32, - hir.OP_swapq : IrOpSwap64, - hir.OP_sxlq : IrOpSx32to64, +var _UnaryOps = [...]IrUnaryOp{ + hir.OP_swapw: IrOpSwap16, + hir.OP_swapl: IrOpSwap32, + hir.OP_swapq: IrOpSwap64, + hir.OP_sxlq: IrOpSx32to64, } -var _BinaryOps = [...]IrBinaryOp { - hir.OP_add : IrOpAdd, - hir.OP_sub : IrOpSub, - hir.OP_addi : IrOpAdd, - hir.OP_muli : IrOpMul, - hir.OP_andi : IrOpAnd, - hir.OP_xori : IrOpXor, - hir.OP_shri : IrOpShr, +var _BinaryOps = [...]IrBinaryOp{ + hir.OP_add: IrOpAdd, + hir.OP_sub: IrOpSub, + hir.OP_addi: IrOpAdd, + hir.OP_muli: IrOpMul, + hir.OP_andi: IrOpAnd, + hir.OP_xori: IrOpXor, + hir.OP_shri: IrOpShr, } -var _ConstnessMap = [...]Constness { - hir.Const : Const, - hir.Volatile : Volatile, +var _ConstnessMap = [...]Constness{ + hir.Const: Const, + hir.Volatile: Volatile, } type BasicBlock struct { - Id int - Phi []*IrPhi - Ins []IrNode - Pred []*BasicBlock - Term IrTerminator + Id int + Phi []*IrPhi + Ins []IrNode + Pred []*BasicBlock + Term IrTerminator } func (self *BasicBlock) String() string { - return fmt.Sprintf("bb_%d", self.Id) + return fmt.Sprintf("bb_%d", self.Id) } func (self *BasicBlock) addPred(p *BasicBlock) { - for _, b := range self.Pred { if b == p { return } } - self.Pred = append(self.Pred, p) + for _, b := range self.Pred { + if b == p { + return + } + } + self.Pred = append(self.Pred, p) } func (self *BasicBlock) addInstr(p *hir.Ir) { - switch p.Op { - default: { - panic("invalid instruction: " + p.Disassemble(nil)) - } - - /* no operation */ - case hir.OP_nop: { - break - } - - /* ptr(Pr) -> Pd */ - case hir.OP_ip: { - self.Ins = append( - self.Ins, - &IrConstPtr { - P: p.Pr, - R: Rv(p.Pd), - M: _ConstnessMap[p.Constness()], - }, - ) - } - - /* *(Ps + Iv) -> Rx */ - case hir.OP_lb, hir.OP_lw, hir.OP_ll, hir.OP_lq: { - self.Ins = append( - self.Ins, - &IrConstInt { - R: Tr(0), - V: p.Iv, - }, - &IrLEA { - R : Pr(0), - Mem : Rv(p.Ps), - Off : Tr(0), - }, - &IrLoad { - R : Rv(p.Rx), - Mem : Pr(0), - Size : _MemSize[p.Op], - }, - ) - } - - /* *(Ps + Iv) -> Pd */ - case hir.OP_lp: { - self.Ins = append( - self.Ins, - &IrConstInt { - R: Tr(0), - V: p.Iv, - }, - &IrLEA { - R : Pr(0), - Mem : Rv(p.Ps), - Off : Tr(0), - }, - &IrLoad { - R : Rv(p.Pd), - Mem : Pr(0), - Size : abi.PtrSize, - }, - ) - } - - /* Rx -> *(Pd + Iv) */ - case hir.OP_sb, hir.OP_sw, hir.OP_sl, hir.OP_sq: { - self.Ins = append( - self.Ins, - &IrConstInt { - R: Tr(0), - V: p.Iv, - }, - &IrLEA { - R : Pr(0), - Mem : Rv(p.Pd), - Off : Tr(0), - }, - &IrStore { - R : Rv(p.Rx), - Mem : Pr(0), - Size : _MemSize[p.Op], - }, - ) - } - - /* Ps -> *(Pd + Iv) */ - case hir.OP_sp: { - self.Ins = append( - self.Ins, - &IrConstInt { - R: Tr(0), - V: p.Iv, - }, - &IrLEA { - R : Pr(0), - Mem : Rv(p.Pd), - Off : Tr(0), - }, - &IrConstPtr { - R: Pr(1), - P: rtx.V_pWriteBarrier, - M: Volatile, - }, - &IrConstPtr { - R: Pr(2), - P: rtx.F_gcWriteBarrier, - }, - &IrWriteBarrier { - R : Rv(p.Ps), - M : Pr(0), - Fn : Pr(2), - Var : Pr(1), - }, - ) - } - - /* arg[Iv] -> Rx */ - case hir.OP_ldaq: { - self.Ins = append( - self.Ins, - &IrLoadArg { - R: Rv(p.Rx), - I: int(p.Iv), - }, - ) - } - - /* arg[Iv] -> Pd */ - case hir.OP_ldap: { - self.Ins = append( - self.Ins, - &IrLoadArg { - R: Rv(p.Pd), - I: int(p.Iv), - }, - ) - } - - /* Ps + Rx -> Pd */ - case hir.OP_addp: { - self.Ins = append( - self.Ins, - &IrLEA { - R : Rv(p.Pd), - Mem : Rv(p.Ps), - Off : Rv(p.Rx), - }, - ) - } - - /* Ps - Rx -> Pd */ - case hir.OP_subp: { - self.Ins = append( - self.Ins, - &IrUnaryExpr { - R : Tr(0), - V : Rv(p.Rx), - Op : IrOpNegate, - }, - &IrLEA { - R : Rv(p.Pd), - Mem : Rv(p.Ps), - Off : Tr(0), - }, - ) - } - - /* Ps + Iv -> Pd */ - case hir.OP_addpi: { - self.Ins = append( - self.Ins, - &IrConstInt { - R: Tr(0), - V: p.Iv, - }, - &IrLEA { - R : Rv(p.Pd), - Mem : Rv(p.Ps), - Off : Tr(0), - }, - ) - } - - /* Rx ± Ry -> Rz */ - case hir.OP_add, hir.OP_sub: { - self.Ins = append( - self.Ins, - &IrBinaryExpr { - R : Rv(p.Rz), - X : Rv(p.Rx), - Y : Rv(p.Ry), - Op : _BinaryOps[p.Op], - }, - ) - } - - /* Ry & (1 << (Rx % PTR_BITS)) != 0 -> Rz - * Ry |= 1 << (Rx % PTR_BITS) */ - case hir.OP_bts: { - self.Ins = append( - self.Ins, - &IrBitTestSet { - T: Rv(p.Rz), - S: Rv(p.Ry), - X: Rv(p.Ry), - Y: Rv(p.Rx), - }, - ) - } - - /* Rx {+,*,&,^,>>} Iv -> Ry */ - case hir.OP_addi, hir.OP_muli, hir.OP_andi, hir.OP_xori, hir.OP_shri: { - self.Ins = append( - self.Ins, - &IrConstInt { - R: Tr(0), - V: p.Iv, - }, - &IrBinaryExpr { - R : Rv(p.Ry), - X : Rv(p.Rx), - Y : Tr(0), - Op : _BinaryOps[p.Op], - }, - ) - } - - /* Rx | (1 << Iv) -> Ry */ - case hir.OP_bsi: { - self.Ins = append( - self.Ins, - &IrConstInt { - R: Tr(0), - V: 1 << p.Iv, - }, - &IrBinaryExpr { - R : Rv(p.Ry), - X : Rv(p.Rx), - Y : Tr(0), - Op : IrOpOr, - }, - ) - } - - /* {bswap{16/32/64}/sign_extend_32_to_64}(Rx) -> Ry */ - case hir.OP_swapw, hir.OP_swapl, hir.OP_swapq, hir.OP_sxlq: { - self.Ins = append( - self.Ins, - &IrUnaryExpr { - R : Rv(p.Ry), - V : Rv(p.Rx), - Op : _UnaryOps[p.Op], - }, - ) - } - - /* memset(Pd, 0, Iv) */ - case hir.OP_bzero: { - r := Rv(p.Pd) - d := uintptr(0) - n := uintptr(p.Iv) - - /* call memory zero for large blocks */ - for ; n >= rtx.MaxZeroSize; n -= rtx.MaxZeroSize { - self.zeroBlock(r, d, rtx.MaxZeroSize) - d += rtx.MaxZeroSize - } - - /* call memory zero for smaller blocks */ - if n >= rtx.ZeroStep { - self.zeroBlock(r, d, n) - d += n / rtx.ZeroStep * rtx.ZeroStep - n %= rtx.ZeroStep - } - - /* use scalar code for remaining bytes */ - for _, v := range []uintptr { 8, 4, 2, 1 } { - for n >= v { - self.zeroUnit(r, d, v) - d += v - n -= v - } - } - } - - /* memcpy(Pd, Ps, Rx) */ - case hir.OP_bcopy: { - self.Ins = append( - self.Ins, - &IrConstPtr { - R: Pr(0), - P: rtx.F_memmove, - }, - &IrCallFunc { - R : Pr(0), - In : []Reg { Rv(p.Pd), Rv(p.Ps), Rv(p.Rx) }, - Func : rtx.S_memmove, - }, - ) - } - - /* C subroutine calls */ - case hir.OP_ccall: { - self.Ins = append( - self.Ins, - &IrConstPtr { - R: Pr(0), - P: hir.LookupCall(p.Iv).Func, - }, - &IrCallNative { - R : Pr(0), - In : ri2regs(p.Ar[:p.An]), - Out : ri2regz(p.Rr[:p.Rn]), - }, - ) - } - - /* Go subroutine calls */ - case hir.OP_gcall: { - self.Ins = append( - self.Ins, - &IrConstPtr { - R: Pr(0), - P: hir.LookupCall(p.Iv).Func, - }, - &IrCallFunc { - R : Pr(0), - In : ri2regs(p.Ar[:p.An]), - Out : ri2regs(p.Rr[:p.Rn]), - Func : abi.ABI.GetLayout(hir.LookupCall(p.Iv).Id), - }, - ) - } - - /* interface method calls */ - case hir.OP_icall: { - self.Ins = append( - self.Ins, - &IrCallMethod { - T : Rv(p.Ps), - V : Rv(p.Pd), - In : ri2regs(p.Ar[:p.An]), - Out : ri2regs(p.Rr[:p.Rn]), - Slot : hir.LookupCall(p.Iv).Slot, - Func : abi.ABI.GetLayout(hir.LookupCall(p.Iv).Id), - }, - ) - } - - /* trigger a debugger breakpoint */ - case hir.OP_break: { - self.Ins = append( - self.Ins, - new(IrBreakpoint), - ) - } - } + switch p.Op { + default: + { + panic("invalid instruction: " + p.Disassemble(nil)) + } + + /* no operation */ + case hir.OP_nop: + { + break + } + + /* ptr(Pr) -> Pd */ + case hir.OP_ip: + { + self.Ins = append( + self.Ins, + &IrConstPtr{ + P: p.Pr, + R: Rv(p.Pd), + M: _ConstnessMap[p.Constness()], + }, + ) + } + + /* *(Ps + Iv) -> Rx */ + case hir.OP_lb, hir.OP_lw, hir.OP_ll, hir.OP_lq: + { + self.Ins = append( + self.Ins, + &IrConstInt{ + R: Tr(0), + V: p.Iv, + }, + &IrLEA{ + R: Pr(0), + Mem: Rv(p.Ps), + Off: Tr(0), + }, + &IrLoad{ + R: Rv(p.Rx), + Mem: Pr(0), + Size: _MemSize[p.Op], + }, + ) + } + + /* *(Ps + Iv) -> Pd */ + case hir.OP_lp: + { + self.Ins = append( + self.Ins, + &IrConstInt{ + R: Tr(0), + V: p.Iv, + }, + &IrLEA{ + R: Pr(0), + Mem: Rv(p.Ps), + Off: Tr(0), + }, + &IrLoad{ + R: Rv(p.Pd), + Mem: Pr(0), + Size: abi.PtrSize, + }, + ) + } + + /* Rx -> *(Pd + Iv) */ + case hir.OP_sb, hir.OP_sw, hir.OP_sl, hir.OP_sq: + { + self.Ins = append( + self.Ins, + &IrConstInt{ + R: Tr(0), + V: p.Iv, + }, + &IrLEA{ + R: Pr(0), + Mem: Rv(p.Pd), + Off: Tr(0), + }, + &IrStore{ + R: Rv(p.Rx), + Mem: Pr(0), + Size: _MemSize[p.Op], + }, + ) + } + + /* Ps -> *(Pd + Iv) */ + case hir.OP_sp: + { + self.Ins = append( + self.Ins, + &IrConstInt{ + R: Tr(0), + V: p.Iv, + }, + &IrLEA{ + R: Pr(0), + Mem: Rv(p.Pd), + Off: Tr(0), + }, + &IrConstPtr{ + R: Pr(1), + P: rtx.V_pWriteBarrier, + M: Volatile, + }, + &IrConstPtr{ + R: Pr(2), + P: rtx.F_gcWriteBarrier, + }, + &IrWriteBarrier{ + R: Rv(p.Ps), + M: Pr(0), + Fn: Pr(2), + Var: Pr(1), + }, + ) + } + + /* arg[Iv] -> Rx */ + case hir.OP_ldaq: + { + self.Ins = append( + self.Ins, + &IrLoadArg{ + R: Rv(p.Rx), + I: int(p.Iv), + }, + ) + } + + /* arg[Iv] -> Pd */ + case hir.OP_ldap: + { + self.Ins = append( + self.Ins, + &IrLoadArg{ + R: Rv(p.Pd), + I: int(p.Iv), + }, + ) + } + + /* Ps + Rx -> Pd */ + case hir.OP_addp: + { + self.Ins = append( + self.Ins, + &IrLEA{ + R: Rv(p.Pd), + Mem: Rv(p.Ps), + Off: Rv(p.Rx), + }, + ) + } + + /* Ps - Rx -> Pd */ + case hir.OP_subp: + { + self.Ins = append( + self.Ins, + &IrUnaryExpr{ + R: Tr(0), + V: Rv(p.Rx), + Op: IrOpNegate, + }, + &IrLEA{ + R: Rv(p.Pd), + Mem: Rv(p.Ps), + Off: Tr(0), + }, + ) + } + + /* Ps + Iv -> Pd */ + case hir.OP_addpi: + { + self.Ins = append( + self.Ins, + &IrConstInt{ + R: Tr(0), + V: p.Iv, + }, + &IrLEA{ + R: Rv(p.Pd), + Mem: Rv(p.Ps), + Off: Tr(0), + }, + ) + } + + /* Rx ± Ry -> Rz */ + case hir.OP_add, hir.OP_sub: + { + self.Ins = append( + self.Ins, + &IrBinaryExpr{ + R: Rv(p.Rz), + X: Rv(p.Rx), + Y: Rv(p.Ry), + Op: _BinaryOps[p.Op], + }, + ) + } + + /* Ry & (1 << (Rx % PTR_BITS)) != 0 -> Rz + * Ry |= 1 << (Rx % PTR_BITS) */ + case hir.OP_bts: + { + self.Ins = append( + self.Ins, + &IrBitTestSet{ + T: Rv(p.Rz), + S: Rv(p.Ry), + X: Rv(p.Ry), + Y: Rv(p.Rx), + }, + ) + } + + /* Rx {+,*,&,^,>>} Iv -> Ry */ + case hir.OP_addi, hir.OP_muli, hir.OP_andi, hir.OP_xori, hir.OP_shri: + { + self.Ins = append( + self.Ins, + &IrConstInt{ + R: Tr(0), + V: p.Iv, + }, + &IrBinaryExpr{ + R: Rv(p.Ry), + X: Rv(p.Rx), + Y: Tr(0), + Op: _BinaryOps[p.Op], + }, + ) + } + + /* Rx | (1 << Iv) -> Ry */ + case hir.OP_bsi: + { + self.Ins = append( + self.Ins, + &IrConstInt{ + R: Tr(0), + V: 1 << p.Iv, + }, + &IrBinaryExpr{ + R: Rv(p.Ry), + X: Rv(p.Rx), + Y: Tr(0), + Op: IrOpOr, + }, + ) + } + + /* {bswap{16/32/64}/sign_extend_32_to_64}(Rx) -> Ry */ + case hir.OP_swapw, hir.OP_swapl, hir.OP_swapq, hir.OP_sxlq: + { + self.Ins = append( + self.Ins, + &IrUnaryExpr{ + R: Rv(p.Ry), + V: Rv(p.Rx), + Op: _UnaryOps[p.Op], + }, + ) + } + + /* memset(Pd, 0, Iv) */ + case hir.OP_bzero: + { + r := Rv(p.Pd) + d := uintptr(0) + n := uintptr(p.Iv) + + /* call memory zero for large blocks */ + for ; n >= rtx.MaxZeroSize; n -= rtx.MaxZeroSize { + self.zeroBlock(r, d, rtx.MaxZeroSize) + d += rtx.MaxZeroSize + } + + /* call memory zero for smaller blocks */ + if n >= rtx.ZeroStep { + self.zeroBlock(r, d, n) + d += n / rtx.ZeroStep * rtx.ZeroStep + n %= rtx.ZeroStep + } + + /* use scalar code for remaining bytes */ + for _, v := range []uintptr{8, 4, 2, 1} { + for n >= v { + self.zeroUnit(r, d, v) + d += v + n -= v + } + } + } + + /* memcpy(Pd, Ps, Rx) */ + case hir.OP_bcopy: + { + self.Ins = append( + self.Ins, + &IrConstPtr{ + R: Pr(0), + P: rtx.F_memmove, + }, + &IrCallFunc{ + R: Pr(0), + In: []Reg{Rv(p.Pd), Rv(p.Ps), Rv(p.Rx)}, + Func: rtx.S_memmove, + }, + ) + } + + /* C subroutine calls */ + case hir.OP_ccall: + { + self.Ins = append( + self.Ins, + &IrConstPtr{ + R: Pr(0), + P: hir.LookupCall(p.Iv).Func, + }, + &IrCallNative{ + R: Pr(0), + In: ri2regs(p.Ar[:p.An]), + Out: ri2regz(p.Rr[:p.Rn]), + }, + ) + } + + /* Go subroutine calls */ + case hir.OP_gcall: + { + self.Ins = append( + self.Ins, + &IrConstPtr{ + R: Pr(0), + P: hir.LookupCall(p.Iv).Func, + }, + &IrCallFunc{ + R: Pr(0), + In: ri2regs(p.Ar[:p.An]), + Out: ri2regs(p.Rr[:p.Rn]), + Func: abi.ABI.GetLayout(hir.LookupCall(p.Iv).Id), + }, + ) + } + + /* interface method calls */ + case hir.OP_icall: + { + self.Ins = append( + self.Ins, + &IrCallMethod{ + T: Rv(p.Ps), + V: Rv(p.Pd), + In: ri2regs(p.Ar[:p.An]), + Out: ri2regs(p.Rr[:p.Rn]), + Slot: hir.LookupCall(p.Iv).Slot, + Func: abi.ABI.GetLayout(hir.LookupCall(p.Iv).Id), + }, + ) + } + + /* trigger a debugger breakpoint */ + case hir.OP_break: + { + self.Ins = append( + self.Ins, + new(IrBreakpoint), + ) + } + } } func (self *BasicBlock) zeroUnit(r Reg, d uintptr, n uintptr) { - self.Ins = append(self.Ins, - &IrConstInt { - R: Tr(0), - V: int64(d), - }, - &IrLEA { - R : Pr(0), - Mem : r, - Off : Tr(0), - }, - &IrStore { - R : Rz, - Mem : Pr(0), - Size : uint8(n), - }, - ) + self.Ins = append(self.Ins, + &IrConstInt{ + R: Tr(0), + V: int64(d), + }, + &IrLEA{ + R: Pr(0), + Mem: r, + Off: Tr(0), + }, + &IrStore{ + R: Rz, + Mem: Pr(0), + Size: uint8(n), + }, + ) } func (self *BasicBlock) zeroBlock(r Reg, d uintptr, n uintptr) { - self.Ins = append(self.Ins, - &IrConstInt { - R: Tr(0), - V: int64(d), - }, - &IrLEA { - R : Pr(0), - Mem : r, - Off : Tr(0), - }, - &IrConstPtr { - R: Pr(1), - P: rtx.MemZero.ForSize(n), - }, - &IrCallNative { - R : Pr(1), - In : []Reg { Pr(0) }, - }, - ) + self.Ins = append(self.Ins, + &IrConstInt{ + R: Tr(0), + V: int64(d), + }, + &IrLEA{ + R: Pr(0), + Mem: r, + Off: Tr(0), + }, + &IrConstPtr{ + R: Pr(1), + P: rtx.MemZero.ForSize(n), + }, + &IrCallNative{ + R: Pr(1), + In: []Reg{Pr(0)}, + }, + ) } func (self *BasicBlock) termReturn(p *hir.Ir) { - self.Term = &IrReturn { - R: ri2regs(p.Rr[:p.Rn]), - } + self.Term = &IrReturn{ + R: ri2regs(p.Rr[:p.Rn]), + } } func (self *BasicBlock) termBranch(to *BasicBlock) { - to.addPred(self) - self.Term = &IrSwitch { Ln: IrLikely(to) } + to.addPred(self) + self.Term = &IrSwitch{Ln: IrLikely(to)} } func (self *BasicBlock) termCondition(p *hir.Ir, t *BasicBlock, f *BasicBlock) { - var cmp IrBinaryOp - var lhs hir.Register - var rhs hir.Register - - /* check for OpCode */ - switch p.Op { - case hir.OP_beq : cmp, lhs, rhs = IrCmpEq , p.Rx, p.Ry - case hir.OP_bne : cmp, lhs, rhs = IrCmpNe , p.Rx, p.Ry - case hir.OP_blt : cmp, lhs, rhs = IrCmpLt , p.Rx, p.Ry - case hir.OP_bltu : cmp, lhs, rhs = IrCmpLtu , p.Rx, p.Ry - case hir.OP_bgeu : cmp, lhs, rhs = IrCmpGeu , p.Rx, p.Ry - case hir.OP_beqp : cmp, lhs, rhs = IrCmpEq , p.Ps, p.Pd - case hir.OP_bnep : cmp, lhs, rhs = IrCmpNe , p.Ps, p.Pd - default : panic("invalid branch: " + p.Disassemble(nil)) - } - - /* construct the instruction */ - ins := &IrBinaryExpr { - R : Tr(0), - X : Rv(lhs), - Y : Rv(rhs), - Op : cmp, - } - - /* add predecessors */ - t.addPred(self) - f.addPred(self) - - /* create branch targets */ - to := IrUnlikely(t) - br := IrUnlikely(f) - - /* assign the correct likeliness */ - if p.Likeliness() == hir.Likely { - to.Likeliness = Likely - } else { - br.Likeliness = Likely - } - - /* attach to the block */ - self.Ins = append(self.Ins, ins) - self.Term = &IrSwitch { V: Tr(0), Ln: to, Br: map[int32]*IrBranch { 0: br } } + var cmp IrBinaryOp + var lhs hir.Register + var rhs hir.Register + + /* check for OpCode */ + switch p.Op { + case hir.OP_beq: + cmp, lhs, rhs = IrCmpEq, p.Rx, p.Ry + case hir.OP_bne: + cmp, lhs, rhs = IrCmpNe, p.Rx, p.Ry + case hir.OP_blt: + cmp, lhs, rhs = IrCmpLt, p.Rx, p.Ry + case hir.OP_bltu: + cmp, lhs, rhs = IrCmpLtu, p.Rx, p.Ry + case hir.OP_bgeu: + cmp, lhs, rhs = IrCmpGeu, p.Rx, p.Ry + case hir.OP_beqp: + cmp, lhs, rhs = IrCmpEq, p.Ps, p.Pd + case hir.OP_bnep: + cmp, lhs, rhs = IrCmpNe, p.Ps, p.Pd + default: + panic("invalid branch: " + p.Disassemble(nil)) + } + + /* construct the instruction */ + ins := &IrBinaryExpr{ + R: Tr(0), + X: Rv(lhs), + Y: Rv(rhs), + Op: cmp, + } + + /* add predecessors */ + t.addPred(self) + f.addPred(self) + + /* create branch targets */ + to := IrUnlikely(t) + br := IrUnlikely(f) + + /* assign the correct likeliness */ + if p.Likeliness() == hir.Likely { + to.Likeliness = Likely + } else { + br.Likeliness = Likely + } + + /* attach to the block */ + self.Ins = append(self.Ins, ins) + self.Term = &IrSwitch{V: Tr(0), Ln: to, Br: map[int32]*IrBranch{0: br}} } diff --git a/internal/atm/ssa/blockiter.go b/internal/atm/ssa/blockiter.go index 0d19aae..f9b458e 100644 --- a/internal/atm/ssa/blockiter.go +++ b/internal/atm/ssa/blockiter.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,75 +17,75 @@ package ssa import ( - `github.com/oleiade/lane` + "github.com/oleiade/lane" ) type BasicBlockIter struct { - g *CFG - b *BasicBlock - s *lane.Stack - v map[int]struct{} + g *CFG + b *BasicBlock + s *lane.Stack + v map[int]struct{} } func newBasicBlockIter(cfg *CFG) *BasicBlockIter { - return &BasicBlockIter { - g: cfg, - s: stacknew(cfg.Root), - v: map[int]struct{} { cfg.Root.Id: {} }, - } + return &BasicBlockIter{ + g: cfg, + s: stacknew(cfg.Root), + v: map[int]struct{}{cfg.Root.Id: {}}, + } } func (self *BasicBlockIter) Next() bool { - var tail bool - var this *BasicBlock + var tail bool + var this *BasicBlock - /* scan until the stack is empty */ - for !self.s.Empty() { - tail = true - this = self.s.Head().(*BasicBlock) + /* scan until the stack is empty */ + for !self.s.Empty() { + tail = true + this = self.s.Head().(*BasicBlock) - /* add all the successors */ - for _, p := range self.g.DominatorOf[this.Id] { - if _, ok := self.v[p.Id]; !ok { - tail = false - self.v[p.Id] = struct{}{} - self.s.Push(p) - break - } - } + /* add all the successors */ + for _, p := range self.g.DominatorOf[this.Id] { + if _, ok := self.v[p.Id]; !ok { + tail = false + self.v[p.Id] = struct{}{} + self.s.Push(p) + break + } + } - /* all the successors are visited, pop the current node */ - if tail { - self.b = self.s.Pop().(*BasicBlock) - return true - } - } + /* all the successors are visited, pop the current node */ + if tail { + self.b = self.s.Pop().(*BasicBlock) + return true + } + } - /* clear the basic block pointer to indicate no more blocks */ - self.b = nil - return false + /* clear the basic block pointer to indicate no more blocks */ + self.b = nil + return false } func (self *BasicBlockIter) Block() *BasicBlock { - return self.b + return self.b } func (self *BasicBlockIter) ForEach(action func(bb *BasicBlock)) { - for self.Next() { - action(self.b) - } + for self.Next() { + action(self.b) + } } func (self *BasicBlockIter) Reversed() []*BasicBlock { - nb := len(self.g.Depth) - ret := make([]*BasicBlock, 0, nb) + nb := len(self.g.Depth) + ret := make([]*BasicBlock, 0, nb) - /* dump all the blocks */ - for self.Next() { - ret = append(ret, self.b) - } + /* dump all the blocks */ + for self.Next() { + ret = append(ret, self.b) + } - /* reverse the order */ - blockreverse(ret) - return ret + /* reverse the order */ + blockreverse(ret) + return ret } diff --git a/internal/atm/ssa/builder.go b/internal/atm/ssa/builder.go index 89b5e7d..76719ba 100644 --- a/internal/atm/ssa/builder.go +++ b/internal/atm/ssa/builder.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,166 +17,170 @@ package ssa import ( - `fmt` - `sort` + "fmt" + "sort" - `github.com/cloudwego/frugal/internal/atm/hir` + "github.com/cloudwego/frugal/internal/atm/hir" ) var ( - _GenericRegs []hir.GenericRegister - _PointerRegs []hir.PointerRegister + _GenericRegs []hir.GenericRegister + _PointerRegs []hir.PointerRegister ) func init() { - _GenericRegs = make([]hir.GenericRegister, 0, len(hir.GenericRegisters)) - _PointerRegs = make([]hir.PointerRegister, 0, len(hir.PointerRegisters)) - - /* extract all generic registers */ - for i := range hir.GenericRegisters { - if i != hir.Rz { - _GenericRegs = append(_GenericRegs, i) - } - } - - /* extract all pointer registers */ - for m := range hir.PointerRegisters { - if m != hir.Pn { - _PointerRegs = append(_PointerRegs, m) - } - } - - /* sort by register ID */ - sort.Slice(_GenericRegs, func(i int, j int) bool { return _GenericRegs[i] < _GenericRegs[j] }) - sort.Slice(_PointerRegs, func(i int, j int) bool { return _PointerRegs[i] < _PointerRegs[j] }) + _GenericRegs = make([]hir.GenericRegister, 0, len(hir.GenericRegisters)) + _PointerRegs = make([]hir.PointerRegister, 0, len(hir.PointerRegisters)) + + /* extract all generic registers */ + for i := range hir.GenericRegisters { + if i != hir.Rz { + _GenericRegs = append(_GenericRegs, i) + } + } + + /* extract all pointer registers */ + for m := range hir.PointerRegisters { + if m != hir.Pn { + _PointerRegs = append(_PointerRegs, m) + } + } + + /* sort by register ID */ + sort.Slice(_GenericRegs, func(i int, j int) bool { return _GenericRegs[i] < _GenericRegs[j] }) + sort.Slice(_PointerRegs, func(i int, j int) bool { return _PointerRegs[i] < _PointerRegs[j] }) } type _GraphBuilder struct { - p map[*hir.Ir]bool - g map[*hir.Ir]*BasicBlock + p map[*hir.Ir]bool + g map[*hir.Ir]*BasicBlock } func newGraphBuilder() *_GraphBuilder { - return &_GraphBuilder { - p: make(map[*hir.Ir]bool), - g: make(map[*hir.Ir]*BasicBlock), - } + return &_GraphBuilder{ + p: make(map[*hir.Ir]bool), + g: make(map[*hir.Ir]*BasicBlock), + } } func (self *_GraphBuilder) build(p hir.Program) *CFG { - ret := &CFG { - Depth : make(map[int]int), - DominatedBy : make(map[int]*BasicBlock), - DominatorOf : make(map[int][]*BasicBlock), - DominanceFrontier : make(map[int][]*BasicBlock), - } - - /* create the root block */ - ret.Root = ret.CreateBlock() - ret.Root.Ins = make([]IrNode, 0, len(_GenericRegs) + len(_PointerRegs) + N_size) - - /* implicit defination of all generic registers */ - for _, v := range _GenericRegs { - ret.Root.Ins = append(ret.Root.Ins, &IrConstInt { R: Rv(v), V: 0 }) - } - - /* implicit defination of all pointer registers */ - for _, v := range _PointerRegs { - ret.Root.Ins = append(ret.Root.Ins, &IrConstPtr { R: Rv(v), P: nil, M: Volatile }) - } - - /* implicitly define all the temporary registers */ - for i := 0; i < N_size; i++ { - ret.Root.Ins = append( - ret.Root.Ins, - &IrConstInt { R: Tr(i), V: 0 }, - &IrConstPtr { R: Pr(i), P: nil, M: Volatile }, - ) - } - - /* mark all the branch targets */ - for v := p.Head; v != nil; v = v.Ln { - if v.IsBranch() { - if v.Op != hir.OP_bsw { - self.p[v.Br] = true - } else { - for _, lb := range v.Switch() { - self.p[lb] = true - } - } - } - } - - /* process the root block */ - self.g[p.Head] = ret.Root - self.block(ret, p.Head, ret.Root) - - /* build the CFG */ - ret.Rebuild() - return ret + ret := &CFG{ + Depth: make(map[int]int), + DominatedBy: make(map[int]*BasicBlock), + DominatorOf: make(map[int][]*BasicBlock), + DominanceFrontier: make(map[int][]*BasicBlock), + } + + /* create the root block */ + ret.Root = ret.CreateBlock() + ret.Root.Ins = make([]IrNode, 0, len(_GenericRegs)+len(_PointerRegs)+N_size) + + /* implicit definition of all generic registers */ + for _, v := range _GenericRegs { + ret.Root.Ins = append(ret.Root.Ins, &IrConstInt{R: Rv(v), V: 0}) + } + + /* implicit definition of all pointer registers */ + for _, v := range _PointerRegs { + ret.Root.Ins = append(ret.Root.Ins, &IrConstPtr{R: Rv(v), P: nil, M: Volatile}) + } + + /* implicitly define all the temporary registers */ + for i := 0; i < N_size; i++ { + ret.Root.Ins = append( + ret.Root.Ins, + &IrConstInt{R: Tr(i), V: 0}, + &IrConstPtr{R: Pr(i), P: nil, M: Volatile}, + ) + } + + /* mark all the branch targets */ + for v := p.Head; v != nil; v = v.Ln { + if v.IsBranch() { + if v.Op != hir.OP_bsw { + self.p[v.Br] = true + } else { + for _, lb := range v.Switch() { + self.p[lb] = true + } + } + } + } + + /* process the root block */ + self.g[p.Head] = ret.Root + self.block(ret, p.Head, ret.Root) + + /* build the CFG */ + ret.Rebuild() + return ret } func (self *_GraphBuilder) block(cfg *CFG, p *hir.Ir, bb *BasicBlock) { - bb.Phi = nil - bb.Term = nil - - /* traverse down until it hits a branch instruction */ - for p != nil && !p.IsBranch() && p.Op != hir.OP_ret { - bb.addInstr(p) - p = p.Ln - - /* hit a merge point, merge with existing block */ - if self.p[p] { - bb.termBranch(self.branch(cfg, p)) - return - } - } - - /* basic block must terminate */ - if p == nil { - panic(fmt.Sprintf("basic block %d does not terminate", bb.Id)) - } - - /* add terminators */ - switch p.Op { - case hir.OP_bsw : self.termbsw(cfg, p, bb) - case hir.OP_ret : bb.termReturn(p) - case hir.OP_jmp : bb.termBranch(self.branch(cfg, p.Br)) - default : bb.termCondition(p, self.branch(cfg, p.Br), self.branch(cfg, p.Ln)) - } + bb.Phi = nil + bb.Term = nil + + /* traverse down until it hits a branch instruction */ + for p != nil && !p.IsBranch() && p.Op != hir.OP_ret { + bb.addInstr(p) + p = p.Ln + + /* hit a merge point, merge with existing block */ + if self.p[p] { + bb.termBranch(self.branch(cfg, p)) + return + } + } + + /* basic block must terminate */ + if p == nil { + panic(fmt.Sprintf("basic block %d does not terminate", bb.Id)) + } + + /* add terminators */ + switch p.Op { + case hir.OP_bsw: + self.termbsw(cfg, p, bb) + case hir.OP_ret: + bb.termReturn(p) + case hir.OP_jmp: + bb.termBranch(self.branch(cfg, p.Br)) + default: + bb.termCondition(p, self.branch(cfg, p.Br), self.branch(cfg, p.Ln)) + } } func (self *_GraphBuilder) branch(cfg *CFG, p *hir.Ir) *BasicBlock { - var ok bool - var bb *BasicBlock - - /* check for existing basic blocks */ - if bb, ok = self.g[p]; ok { - return bb - } - - /* create and process the new block */ - bb = cfg.CreateBlock() - self.g[p] = bb - self.block(cfg, p, bb) - return bb + var ok bool + var bb *BasicBlock + + /* check for existing basic blocks */ + if bb, ok = self.g[p]; ok { + return bb + } + + /* create and process the new block */ + bb = cfg.CreateBlock() + self.g[p] = bb + self.block(cfg, p, bb) + return bb } func (self *_GraphBuilder) termbsw(cfg *CFG, p *hir.Ir, bb *BasicBlock) { - sw := new(IrSwitch) - sw.Br = make(map[int32]*IrBranch, p.Iv) - bb.Term = sw - - /* add every branch of the switch instruction */ - for i, br := range p.Switch() { - if br != nil { - to := self.branch(cfg, br) - to.addPred(bb) - sw.Br[int32(i)] = IrLikely(to) - } - } - - /* add the default branch */ - sw.Ln = IrUnlikely(self.branch(cfg, p.Ln)) - sw.Ln.To.addPred(bb) + sw := new(IrSwitch) + sw.Br = make(map[int32]*IrBranch, p.Iv) + bb.Term = sw + + /* add every branch of the switch instruction */ + for i, br := range p.Switch() { + if br != nil { + to := self.branch(cfg, br) + to.addPred(bb) + sw.Br[int32(i)] = IrLikely(to) + } + } + + /* add the default branch */ + sw.Ln = IrUnlikely(self.branch(cfg, p.Ln)) + sw.Ln.To.addPred(bb) } diff --git a/internal/atm/ssa/cfg.go b/internal/atm/ssa/cfg.go index 9b92942..661dc71 100644 --- a/internal/atm/ssa/cfg.go +++ b/internal/atm/ssa/cfg.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,67 +17,67 @@ package ssa import ( - `sync/atomic` + "sync/atomic" - `github.com/cloudwego/frugal/internal/atm/abi` + "github.com/cloudwego/frugal/internal/atm/abi" ) type _CFGPrivate struct { - reg uint64 - block uint64 + reg uint64 + block uint64 } func (self *_CFGPrivate) allocreg() int { - return int(atomic.AddUint64(&self.reg, 1)) - 1 + return int(atomic.AddUint64(&self.reg, 1)) - 1 } func (self *_CFGPrivate) allocblock() int { - return int(atomic.AddUint64(&self.block, 1)) - 1 + return int(atomic.AddUint64(&self.block, 1)) - 1 } type CFG struct { - _CFGPrivate - Func FuncData - Root *BasicBlock - Depth map[int]int - Layout *abi.FunctionLayout - DominatedBy map[int]*BasicBlock - DominatorOf map[int][]*BasicBlock - DominanceFrontier map[int][]*BasicBlock + _CFGPrivate + Func FuncData + Root *BasicBlock + Depth map[int]int + Layout *abi.FunctionLayout + DominatedBy map[int]*BasicBlock + DominatorOf map[int][]*BasicBlock + DominanceFrontier map[int][]*BasicBlock } func (self *CFG) Rebuild() { - updateDominatorTree(self) - updateDominatorDepth(self) - updateDominatorFrontier(self) + updateDominatorTree(self) + updateDominatorDepth(self) + updateDominatorFrontier(self) } func (self *CFG) MaxBlock() int { - return int(self.block) + return int(self.block) } func (self *CFG) PostOrder() *BasicBlockIter { - return newBasicBlockIter(self) + return newBasicBlockIter(self) } func (self *CFG) CreateBlock() (r *BasicBlock) { - r = new(BasicBlock) - r.Id = self.allocblock() - return + r = new(BasicBlock) + r.Id = self.allocblock() + return } func (self *CFG) CreateRegister(ptr bool) Reg { - if i := self.allocreg(); ptr { - return mkreg(1, K_norm, 0).Derive(i) - } else { - return mkreg(0, K_norm, 0).Derive(i) - } + if i := self.allocreg(); ptr { + return mkreg(1, K_norm, 0).Derive(i) + } else { + return mkreg(0, K_norm, 0).Derive(i) + } } func (self *CFG) CreateUnreachable(bb *BasicBlock) (ret *BasicBlock) { - ret = self.CreateBlock() - ret.Ins = []IrNode { new(IrBreakpoint) } - ret.Term = &IrSwitch { Ln: IrLikely(ret) } - ret.Pred = []*BasicBlock { bb, ret } - return + ret = self.CreateBlock() + ret.Ins = []IrNode{new(IrBreakpoint)} + ret.Term = &IrSwitch{Ln: IrLikely(ret)} + ret.Pred = []*BasicBlock{bb, ret} + return } diff --git a/internal/atm/ssa/cfg_test.go b/internal/atm/ssa/cfg_test.go index a36b46c..e3c91a8 100644 --- a/internal/atm/ssa/cfg_test.go +++ b/internal/atm/ssa/cfg_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,123 +17,123 @@ package ssa import ( - `fmt` - `html` - `io/ioutil` - `strings` - `testing` + "fmt" + "html" + "io/ioutil" + "strings" + "testing" - `github.com/cloudwego/frugal/internal/atm/hir` + "github.com/cloudwego/frugal/internal/atm/hir" ) func dumpbb(bb *BasicBlock) string { - var w int - var phi []string - var ins []string - var term []string - for _, v := range bb.Phi { - for _, ss := range strings.Split(v.String(), "\n") { - vv := html.EscapeString(ss) - vv = strings.ReplaceAll(vv, "$", "$$") - phi = append(phi, fmt.Sprintf("%s\n", vv)) - if len(ss) > w { - w = len(ss) - } - } - } - for _, v := range bb.Ins { - for _, ss := range strings.Split(v.String(), "\n") { - vv := html.EscapeString(ss) - vv = strings.ReplaceAll(vv, "$", "$$") - ins = append(ins, fmt.Sprintf("%s\n", vv)) - if len(ss) > w { - w = len(ss) - } - } - } - for _, ss := range strings.Split(bb.Term.String(), "\n") { - vv := html.EscapeString(ss) - vv = strings.ReplaceAll(vv, "$", "$$") - term = append(term, fmt.Sprintf("%s\n", vv)) - if len(ss) > w { - w = len(ss) - } - } - buf := []string { - "\n", - fmt.Sprintf("\n", w * 10 + 5, bb.Id), - } - if len(bb.Phi) != 0 { - buf = append(buf, "
\n") - buf = append(buf, phi...) - } - if len(bb.Ins) != 0 { - buf = append(buf, "
\n") - buf = append(buf, ins...) - } - buf = append(buf, "
\n") - buf = append(buf, term...) - buf = append(buf, "
bb_%d
") - return strings.Join(buf, "") + var w int + var phi []string + var ins []string + var term []string + for _, v := range bb.Phi { + for _, ss := range strings.Split(v.String(), "\n") { + vv := html.EscapeString(ss) + vv = strings.ReplaceAll(vv, "$", "$$") + phi = append(phi, fmt.Sprintf("%s\n", vv)) + if len(ss) > w { + w = len(ss) + } + } + } + for _, v := range bb.Ins { + for _, ss := range strings.Split(v.String(), "\n") { + vv := html.EscapeString(ss) + vv = strings.ReplaceAll(vv, "$", "$$") + ins = append(ins, fmt.Sprintf("%s\n", vv)) + if len(ss) > w { + w = len(ss) + } + } + } + for _, ss := range strings.Split(bb.Term.String(), "\n") { + vv := html.EscapeString(ss) + vv = strings.ReplaceAll(vv, "$", "$$") + term = append(term, fmt.Sprintf("%s\n", vv)) + if len(ss) > w { + w = len(ss) + } + } + buf := []string{ + "\n", + fmt.Sprintf("\n", w*10+5, bb.Id), + } + if len(bb.Phi) != 0 { + buf = append(buf, "
\n") + buf = append(buf, phi...) + } + if len(bb.Ins) != 0 { + buf = append(buf, "
\n") + buf = append(buf, ins...) + } + buf = append(buf, "
\n") + buf = append(buf, term...) + buf = append(buf, "
bb_%d
") + return strings.Join(buf, "") } func cfgdot(cfg *CFG, fn string) { - e := make(map[[2]int]bool) - buf := []string { - "digraph CFG {", - ` xdotversion = "15"`, - ` graph [ fontname = "Fira Code" ]`, - ` node [ fontname = "Fira Code" fontsize = "16" shape = "plaintext" ]`, - ` edge [ fontname = "Fira Code" ]`, - ` START [ shape = "circle" ]`, - fmt.Sprintf(` START -> bb_%d`, cfg.Root.Id), - } - for _, p := range cfg.PostOrder().Reversed() { - f := true - it := p.Term.Successors() - buf = append(buf, fmt.Sprintf(` bb_%d [ label = < %s > ]`, p.Id, dumpbb(p))) - for it.Next() { - ln := it.Block() - edge := [2]int{p.Id, ln.Id} - if !e[edge] { - e[edge] = true - if v, ok := it.Value(); ok { - f = false - buf = append(buf, fmt.Sprintf(` bb_%d -> bb_%d [ label = "%d" ]`, p.Id, ln.Id, v)) - } else if f { - buf = append(buf, fmt.Sprintf(` bb_%d -> bb_%d [ label = "goto" ]`, p.Id, ln.Id)) - } else { - buf = append(buf, fmt.Sprintf(` bb_%d -> bb_%d [ label = "otherwise" ]`, p.Id, ln.Id)) - } - } - } - } - buf = append(buf, "}") - err := ioutil.WriteFile(fn, []byte(strings.Join(buf, "\n")), 0644) - if err != nil { - panic(err) - } + e := make(map[[2]int]bool) + buf := []string{ + "digraph CFG {", + ` xdotversion = "15"`, + ` graph [ fontname = "Fira Code" ]`, + ` node [ fontname = "Fira Code" fontsize = "16" shape = "plaintext" ]`, + ` edge [ fontname = "Fira Code" ]`, + ` START [ shape = "circle" ]`, + fmt.Sprintf(` START -> bb_%d`, cfg.Root.Id), + } + for _, p := range cfg.PostOrder().Reversed() { + f := true + it := p.Term.Successors() + buf = append(buf, fmt.Sprintf(` bb_%d [ label = < %s > ]`, p.Id, dumpbb(p))) + for it.Next() { + ln := it.Block() + edge := [2]int{p.Id, ln.Id} + if !e[edge] { + e[edge] = true + if v, ok := it.Value(); ok { + f = false + buf = append(buf, fmt.Sprintf(` bb_%d -> bb_%d [ label = "%d" ]`, p.Id, ln.Id, v)) + } else if f { + buf = append(buf, fmt.Sprintf(` bb_%d -> bb_%d [ label = "goto" ]`, p.Id, ln.Id)) + } else { + buf = append(buf, fmt.Sprintf(` bb_%d -> bb_%d [ label = "otherwise" ]`, p.Id, ln.Id)) + } + } + } + } + buf = append(buf, "}") + err := ioutil.WriteFile(fn, []byte(strings.Join(buf, "\n")), 0644) + if err != nil { + panic(err) + } } var ( - ftest = hir.RegisterGCall(func (i int) int { return i + 1 }, nil) + ftest = hir.RegisterGCall(func(i int) int { return i + 1 }, nil) ) func TestCFG_Build(t *testing.T) { - p := hir.CreateBuilder() - p.LDAP (0, hir.P0) - p.LDAP (1, hir.P1) - p.Label ("loop") - p.LQ (hir.P0, 8, hir.R0) - p.SUBI (hir.R0, 1, hir.R0) - p.GCALL (ftest).A0(hir.R0).R0(hir.R2) - p.SQ (hir.R2, hir.P0, 8) - p.BNE (hir.R0, hir.Rz, "loop") - p.LQ (hir.P1, 8, hir.R1) - p.RET ().R0(hir.R0).R1(hir.R1) - t.Logf("Generating CFG ...") - c := p.Build() - g := Compile(c, (func(*int, *int) (int, int))(nil)) - t.Logf("Generating DOT file ...") - cfgdot(g, "/tmp/cfg.gv") + p := hir.CreateBuilder() + p.LDAP(0, hir.P0) + p.LDAP(1, hir.P1) + p.Label("loop") + p.LQ(hir.P0, 8, hir.R0) + p.SUBI(hir.R0, 1, hir.R0) + p.GCALL(ftest).A0(hir.R0).R0(hir.R2) + p.SQ(hir.R2, hir.P0, 8) + p.BNE(hir.R0, hir.Rz, "loop") + p.LQ(hir.P1, 8, hir.R1) + p.RET().R0(hir.R0).R1(hir.R1) + t.Logf("Generating CFG ...") + c := p.Build() + g := Compile(c, (func(*int, *int) (int, int))(nil)) + t.Logf("Generating DOT file ...") + cfgdot(g, "/tmp/cfg.gv") } diff --git a/internal/atm/ssa/compile.go b/internal/atm/ssa/compile.go index bb37a63..fb238a0 100644 --- a/internal/atm/ssa/compile.go +++ b/internal/atm/ssa/compile.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,65 +17,65 @@ package ssa import ( - `reflect` + "reflect" - `github.com/cloudwego/frugal/internal/atm/abi` - `github.com/cloudwego/frugal/internal/atm/hir` + "github.com/cloudwego/frugal/internal/atm/abi" + "github.com/cloudwego/frugal/internal/atm/hir" ) type Pass interface { - Apply(*CFG) + Apply(*CFG) } type PassDescriptor struct { - Pass Pass - Name string + Pass Pass + Name string } -var Passes = [...]PassDescriptor { - { Name: "Early Constant Propagation" , Pass: new(ConstProp) }, - { Name: "Early Reduction" , Pass: new(Reduce) }, - { Name: "Branch Elimination" , Pass: new(BranchElim) }, - { Name: "Return Spreading" , Pass: new(ReturnSpread) }, - { Name: "Value Reordering" , Pass: new(Reorder) }, - { Name: "Late Constant Propagation" , Pass: new(ConstProp) }, - { Name: "Late Reduction" , Pass: new(Reduce) }, - { Name: "Machine Dependent Lowering" , Pass: new(Lowering) }, - { Name: "Zero Register Substitution" , Pass: new(ZeroReg) }, - { Name: "Write Barrier Insertion" , Pass: new(WriteBarrier) }, - { Name: "ABI-Specific Lowering" , Pass: new(ABILowering) }, - { Name: "Instruction Fusion" , Pass: new(Fusion) }, - { Name: "Instruction Compaction" , Pass: new(Compaction) }, - { Name: "Block Merging" , Pass: new(BlockMerge) }, - { Name: "Critical Edge Splitting" , Pass: new(SplitCritical) }, - { Name: "Phi Propagation" , Pass: new(PhiProp) }, - { Name: "Operand Allocation" , Pass: new(OperandAlloc) }, - { Name: "Constant Rematerialize" , Pass: new(Rematerialize) }, - { Name: "Pre-allocation TDCE" , Pass: new(TDCE) }, - { Name: "Register Allocation" , Pass: new(RegAlloc) }, - { Name: "Stack Liveness Analysis" , Pass: new(StackLiveness) }, - { Name: "Function Layout" , Pass: new(Layout) }, +var Passes = [...]PassDescriptor{ + {Name: "Early Constant Propagation", Pass: new(ConstProp)}, + {Name: "Early Reduction", Pass: new(Reduce)}, + {Name: "Branch Elimination", Pass: new(BranchElim)}, + {Name: "Return Spreading", Pass: new(ReturnSpread)}, + {Name: "Value Reordering", Pass: new(Reorder)}, + {Name: "Late Constant Propagation", Pass: new(ConstProp)}, + {Name: "Late Reduction", Pass: new(Reduce)}, + {Name: "Machine Dependent Lowering", Pass: new(Lowering)}, + {Name: "Zero Register Substitution", Pass: new(ZeroReg)}, + {Name: "Write Barrier Insertion", Pass: new(WriteBarrier)}, + {Name: "ABI-Specific Lowering", Pass: new(ABILowering)}, + {Name: "Instruction Fusion", Pass: new(Fusion)}, + {Name: "Instruction Compaction", Pass: new(Compaction)}, + {Name: "Block Merging", Pass: new(BlockMerge)}, + {Name: "Critical Edge Splitting", Pass: new(SplitCritical)}, + {Name: "Phi Propagation", Pass: new(PhiProp)}, + {Name: "Operand Allocation", Pass: new(OperandAlloc)}, + {Name: "Constant Rematerialize", Pass: new(Rematerialize)}, + {Name: "Pre-allocation TDCE", Pass: new(TDCE)}, + {Name: "Register Allocation", Pass: new(RegAlloc)}, + {Name: "Stack Liveness Analysis", Pass: new(StackLiveness)}, + {Name: "Function Layout", Pass: new(Layout)}, } func toFuncType(fn interface{}) reflect.Type { - if vt := reflect.TypeOf(fn); vt.Kind() != reflect.Func { - panic("ssa: fn must be a function prototype") - } else { - return vt - } + if vt := reflect.TypeOf(fn); vt.Kind() != reflect.Func { + panic("ssa: fn must be a function prototype") + } else { + return vt + } } func executeSSAPasses(cfg *CFG) { - for _, p := range Passes { - p.Pass.Apply(cfg) - } + for _, p := range Passes { + p.Pass.Apply(cfg) + } } func Compile(p hir.Program, fn interface{}) (cfg *CFG) { - cfg = newGraphBuilder().build(p) - cfg.Layout = abi.ABI.LayoutFunc(-1, toFuncType(fn)) - insertPhiNodes(cfg) - renameRegisters(cfg) - executeSSAPasses(cfg) - return + cfg = newGraphBuilder().build(p) + cfg.Layout = abi.ABI.LayoutFunc(-1, toFuncType(fn)) + insertPhiNodes(cfg) + renameRegisters(cfg) + executeSSAPasses(cfg) + return } diff --git a/internal/atm/ssa/constdata.go b/internal/atm/ssa/constdata.go index 1b79d64..5dea0e9 100644 --- a/internal/atm/ssa/constdata.go +++ b/internal/atm/ssa/constdata.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,36 +17,36 @@ package ssa import ( - `fmt` - `unsafe` + "fmt" + "unsafe" ) type _ConstData struct { - i bool - v int64 - c Constness - p unsafe.Pointer + i bool + v int64 + c Constness + p unsafe.Pointer } func (self _ConstData) String() string { - if self.i { - return fmt.Sprintf("(i64) %d", self.v) - } else { - return fmt.Sprintf("(%s ptr) %p", self.c, self.p) - } + if self.i { + return fmt.Sprintf("(i64) %d", self.v) + } else { + return fmt.Sprintf("(%s ptr) %p", self.c, self.p) + } } func constint(v int64) _ConstData { - return _ConstData { - v: v, - i: true, - } + return _ConstData{ + v: v, + i: true, + } } func constptr(p unsafe.Pointer, cc Constness) _ConstData { - return _ConstData { - p: p, - c: cc, - i: false, - } + return _ConstData{ + p: p, + c: cc, + i: false, + } } diff --git a/internal/atm/ssa/dominator.go b/internal/atm/ssa/dominator.go index 105f720..3058b7d 100644 --- a/internal/atm/ssa/dominator.go +++ b/internal/atm/ssa/dominator.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,260 +21,264 @@ package ssa import ( - `sort` + "sort" - `github.com/cloudwego/frugal/internal/rt` - `github.com/oleiade/lane` + "github.com/cloudwego/frugal/internal/rt" + "github.com/oleiade/lane" ) type _LtNode struct { - semi int - node *BasicBlock - dom *_LtNode - label *_LtNode - parent *_LtNode - ancestor *_LtNode - pred []*_LtNode - bucket map[*_LtNode]struct{} + semi int + node *BasicBlock + dom *_LtNode + label *_LtNode + parent *_LtNode + ancestor *_LtNode + pred []*_LtNode + bucket map[*_LtNode]struct{} } type _LengauerTarjan struct { - nodes []*_LtNode - vertex map[int]int + nodes []*_LtNode + vertex map[int]int } func newLengauerTarjan() *_LengauerTarjan { - return &_LengauerTarjan { - vertex: make(map[int]int), - } + return &_LengauerTarjan{ + vertex: make(map[int]int), + } } func (self *_LengauerTarjan) dfs(bb *BasicBlock) { - i := len(self.nodes) - self.vertex[bb.Id] = i - - /* create a new node */ - p := &_LtNode { - semi : i, - node : bb, - bucket : make(map[*_LtNode]struct{}), - } - - /* add to node list */ - p.label = p - self.nodes = append(self.nodes, p) - - /* get it's successors iterator */ - tr := bb.Term - it := tr.Successors() - - /* traverse the successors */ - for it.Next() { - w := it.Block() - idx, ok := self.vertex[w.Id] - - /* not visited yet */ - if !ok { - self.dfs(w) - idx = self.vertex[w.Id] - self.nodes[idx].parent = p - } - - /* add predecessors */ - q := self.nodes[idx] - q.pred = append(q.pred, p) - } + i := len(self.nodes) + self.vertex[bb.Id] = i + + /* create a new node */ + p := &_LtNode{ + semi: i, + node: bb, + bucket: make(map[*_LtNode]struct{}), + } + + /* add to node list */ + p.label = p + self.nodes = append(self.nodes, p) + + /* get it's successors iterator */ + tr := bb.Term + it := tr.Successors() + + /* traverse the successors */ + for it.Next() { + w := it.Block() + idx, ok := self.vertex[w.Id] + + /* not visited yet */ + if !ok { + self.dfs(w) + idx = self.vertex[w.Id] + self.nodes[idx].parent = p + } + + /* add predecessors */ + q := self.nodes[idx] + q.pred = append(q.pred, p) + } } func (self *_LengauerTarjan) eval(p *_LtNode) *_LtNode { - if p.ancestor == nil { - return p - } else { - self.compress(p) - return p.label - } + if p.ancestor == nil { + return p + } else { + self.compress(p) + return p.label + } } func (self *_LengauerTarjan) link(p *_LtNode, q *_LtNode) { - q.ancestor = p + q.ancestor = p } func (self *_LengauerTarjan) relable(p *_LtNode) { - if p.label.semi > p.ancestor.label.semi { - p.label = p.ancestor.label - } + if p.label.semi > p.ancestor.label.semi { + p.label = p.ancestor.label + } } func (self *_LengauerTarjan) compress(p *_LtNode) { - if p.ancestor.ancestor != nil { - self.compress(p.ancestor) - self.relable(p) - p.ancestor = p.ancestor.ancestor - } + if p.ancestor.ancestor != nil { + self.compress(p.ancestor) + self.relable(p) + p.ancestor = p.ancestor.ancestor + } } type _NodeDepth struct { - d int - bb int + d int + bb int } func updateDominatorTree(cfg *CFG) { - rt.MapClear(cfg.DominatedBy) - rt.MapClear(cfg.DominatorOf) - - /* Step 1: Carry out a depth-first search of the problem graph. Number the vertices - * from 1 to n as they are reached during the search. Initialize the variables used - * in succeeding steps. */ - lt := newLengauerTarjan() - lt.dfs(cfg.Root) - - /* perform Step 2 and Step 3 for every node */ - for i := len(lt.nodes) - 1; i > 0; i-- { - p := lt.nodes[i] - q := (*_LtNode)(nil) - - /* Step 2: Compute the semidominators of all vertices by applying Theorem 4. - * Carry out the computation vertex by vertex in decreasing order by number. */ - for _, v := range p.pred { - q = lt.eval(v) - p.semi = minint(p.semi, q.semi) - } - - /* link the ancestor */ - lt.link(p.parent, p) - lt.nodes[p.semi].bucket[p] = struct{}{} - - /* Step 3: Implicitly define the immediate dominator of each vertex by applying Corollary 1 */ - for v := range p.parent.bucket { - if q = lt.eval(v); q.semi < v.semi { - v.dom = q - } else { - v.dom = p.parent - } - } - - /* clear the bucket */ - for v := range p.parent.bucket { - delete(p.parent.bucket, v) - } - } - - /* Step 4: Explicitly define the immediate dominator of each vertex, carrying out the - * computation vertex by vertex in increasing order by number. */ - for _, p := range lt.nodes[1:] { - if p.dom.node.Id != lt.nodes[p.semi].node.Id { - p.dom = p.dom.dom - } - } - - /* map the dominator relationship */ - for _, p := range lt.nodes[1:] { - cfg.DominatedBy[p.node.Id] = p.dom.node - cfg.DominatorOf[p.dom.node.Id] = append(cfg.DominatorOf[p.dom.node.Id], p.node) - } - - /* sort the dominators */ - for _, p := range cfg.DominatorOf { - sort.Slice(p, func(i int, j int) bool { - return p[i].Id < p[j].Id - }) - } + rt.MapClear(cfg.DominatedBy) + rt.MapClear(cfg.DominatorOf) + + /* Step 1: Carry out a depth-first search of the problem graph. Number the vertices + * from 1 to n as they are reached during the search. Initialize the variables used + * in succeeding steps. */ + lt := newLengauerTarjan() + lt.dfs(cfg.Root) + + /* perform Step 2 and Step 3 for every node */ + for i := len(lt.nodes) - 1; i > 0; i-- { + p := lt.nodes[i] + q := (*_LtNode)(nil) + + /* Step 2: Compute the semidominators of all vertices by applying Theorem 4. + * Carry out the computation vertex by vertex in decreasing order by number. */ + for _, v := range p.pred { + q = lt.eval(v) + p.semi = minint(p.semi, q.semi) + } + + /* link the ancestor */ + lt.link(p.parent, p) + lt.nodes[p.semi].bucket[p] = struct{}{} + + /* Step 3: Implicitly define the immediate dominator of each vertex by applying Corollary 1 */ + for v := range p.parent.bucket { + if q = lt.eval(v); q.semi < v.semi { + v.dom = q + } else { + v.dom = p.parent + } + } + + /* clear the bucket */ + for v := range p.parent.bucket { + delete(p.parent.bucket, v) + } + } + + /* Step 4: Explicitly define the immediate dominator of each vertex, carrying out the + * computation vertex by vertex in increasing order by number. */ + for _, p := range lt.nodes[1:] { + if p.dom.node.Id != lt.nodes[p.semi].node.Id { + p.dom = p.dom.dom + } + } + + /* map the dominator relationship */ + for _, p := range lt.nodes[1:] { + cfg.DominatedBy[p.node.Id] = p.dom.node + cfg.DominatorOf[p.dom.node.Id] = append(cfg.DominatorOf[p.dom.node.Id], p.node) + } + + /* sort the dominators */ + for _, p := range cfg.DominatorOf { + sort.Slice(p, func(i int, j int) bool { + return p[i].Id < p[j].Id + }) + } } func updateDominatorDepth(cfg *CFG) { - r := cfg.Root.Id - q := lane.NewQueue() - - /* add the root node */ - q.Enqueue(_NodeDepth { bb: r }) - rt.MapClear(cfg.Depth) - - /* calculate depth for every block */ - for !q.Empty() { - d := q.Dequeue().(_NodeDepth) - cfg.Depth[d.bb] = d.d - - /* add all the dominated nodes */ - for _, p := range cfg.DominatorOf[d.bb] { - q.Enqueue(_NodeDepth { - d : d.d + 1, - bb : p.Id, - }) - } - } + r := cfg.Root.Id + q := lane.NewQueue() + + /* add the root node */ + q.Enqueue(_NodeDepth{bb: r}) + rt.MapClear(cfg.Depth) + + /* calculate depth for every block */ + for !q.Empty() { + d := q.Dequeue().(_NodeDepth) + cfg.Depth[d.bb] = d.d + + /* add all the dominated nodes */ + for _, p := range cfg.DominatorOf[d.bb] { + q.Enqueue(_NodeDepth{ + d: d.d + 1, + bb: p.Id, + }) + } + } } func updateDominatorFrontier(cfg *CFG) { - r := cfg.Root - q := lane.NewQueue() - - /* add the root node */ - q.Enqueue(r) - rt.MapClear(cfg.DominanceFrontier) - - /* calculate dominance frontier for every block */ - for !q.Empty() { - k := q.Dequeue().(*BasicBlock) - addImmediateDominated(cfg.DominatorOf, k, q) - computeDominanceFrontier(cfg.DominatorOf, k, cfg.DominanceFrontier) - } + r := cfg.Root + q := lane.NewQueue() + + /* add the root node */ + q.Enqueue(r) + rt.MapClear(cfg.DominanceFrontier) + + /* calculate dominance frontier for every block */ + for !q.Empty() { + k := q.Dequeue().(*BasicBlock) + addImmediateDominated(cfg.DominatorOf, k, q) + computeDominanceFrontier(cfg.DominatorOf, k, cfg.DominanceFrontier) + } } func isStrictlyDominates(dom map[int][]*BasicBlock, p *BasicBlock, q *BasicBlock) bool { - for _, v := range dom[p.Id] { if v != p && (v == q || isStrictlyDominates(dom, v, q)) { return true } } - return false + for _, v := range dom[p.Id] { + if v != p && (v == q || isStrictlyDominates(dom, v, q)) { + return true + } + } + return false } func addImmediateDominated(dom map[int][]*BasicBlock, node *BasicBlock, q *lane.Queue) { - for _, p := range dom[node.Id] { - q.Enqueue(p) - } + for _, p := range dom[node.Id] { + q.Enqueue(p) + } } func computeDominanceFrontier(dom map[int][]*BasicBlock, node *BasicBlock, dfm map[int][]*BasicBlock) []*BasicBlock { - var it IrSuccessors - var df map[*BasicBlock]struct{} - - /* check for cached values */ - if v, ok := dfm[node.Id]; ok { - return v - } - - /* get the successor iterator */ - it = node.Term.Successors() - df = make(map[*BasicBlock]struct{}) - - /* local(X) = set of successors of X that X does not immediately dominate */ - for it.Next() { - if y := it.Block(); !isStrictlyDominates(dom, node, y) { - df[y] = struct{}{} - } - } - - /* df(X) = union of local(X) and ( union of up(K) for all K that are children of X ) */ - for _, k := range dom[node.Id] { - for _, y := range computeDominanceFrontier(dom, k, dfm) { - if !isStrictlyDominates(dom, node, y) { - df[y] = struct{}{} - } - } - } - - /* convert to slice */ - nb := len(df) - ret := make([]*BasicBlock, 0, nb) - - /* extract all the keys */ - for bb := range df { - ret = append(ret, bb) - } - - /* sort by ID */ - sort.Slice(ret, func(i int, j int) bool { - return ret[i].Id < ret[j].Id - }) - - /* add to cache */ - dfm[node.Id] = ret - return ret + var it IrSuccessors + var df map[*BasicBlock]struct{} + + /* check for cached values */ + if v, ok := dfm[node.Id]; ok { + return v + } + + /* get the successor iterator */ + it = node.Term.Successors() + df = make(map[*BasicBlock]struct{}) + + /* local(X) = set of successors of X that X does not immediately dominate */ + for it.Next() { + if y := it.Block(); !isStrictlyDominates(dom, node, y) { + df[y] = struct{}{} + } + } + + /* df(X) = union of local(X) and ( union of up(K) for all K that are children of X ) */ + for _, k := range dom[node.Id] { + for _, y := range computeDominanceFrontier(dom, k, dfm) { + if !isStrictlyDominates(dom, node, y) { + df[y] = struct{}{} + } + } + } + + /* convert to slice */ + nb := len(df) + ret := make([]*BasicBlock, 0, nb) + + /* extract all the keys */ + for bb := range df { + ret = append(ret, bb) + } + + /* sort by ID */ + sort.Slice(ret, func(i int, j int) bool { + return ret[i].Id < ret[j].Id + }) + + /* add to cache */ + dfm[node.Id] = ret + return ret } diff --git a/internal/atm/ssa/funcdata.go b/internal/atm/ssa/funcdata.go index 9750068..ba804a9 100644 --- a/internal/atm/ssa/funcdata.go +++ b/internal/atm/ssa/funcdata.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,42 +17,42 @@ package ssa import ( - `fmt` - `strings` + "fmt" + "strings" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) type FuncData struct { - Code []byte - Layout *FuncLayout - Liveness map[Pos]SlotSet - StackMap map[uintptr]*rt.StackMap + Code []byte + Layout *FuncLayout + Liveness map[Pos]SlotSet + StackMap map[uintptr]*rt.StackMap } type FuncLayout struct { - Ins []IrNode - Start map[int]int - Block map[int]*BasicBlock + Ins []IrNode + Start map[int]int + Block map[int]*BasicBlock } func (self *FuncLayout) String() string { - ni := len(self.Ins) - ns := len(self.Start) - ss := make([]string, 0, ni + ns) - - /* print every instruction */ - for i, ins := range self.Ins { - if bb, ok := self.Block[i]; !ok { - ss = append(ss, fmt.Sprintf("%06x | %s", i, ins)) - } else { - ss = append(ss, fmt.Sprintf("%06x | bb_%d:", i, bb.Id), fmt.Sprintf("%06x | %s", i, ins)) - } - } - - /* join them together */ - return fmt.Sprintf( - "FuncLayout {\n%s\n}", - strings.Join(ss, "\n"), - ) + ni := len(self.Ins) + ns := len(self.Start) + ss := make([]string, 0, ni+ns) + + /* print every instruction */ + for i, ins := range self.Ins { + if bb, ok := self.Block[i]; !ok { + ss = append(ss, fmt.Sprintf("%06x | %s", i, ins)) + } else { + ss = append(ss, fmt.Sprintf("%06x | bb_%d:", i, bb.Id), fmt.Sprintf("%06x | %s", i, ins)) + } + } + + /* join them together */ + return fmt.Sprintf( + "FuncLayout {\n%s\n}", + strings.Join(ss, "\n"), + ) } diff --git a/internal/atm/ssa/int65.go b/internal/atm/ssa/int65.go index a285c9b..0d73ee5 100644 --- a/internal/atm/ssa/int65.go +++ b/internal/atm/ssa/int65.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,70 +17,70 @@ package ssa import ( - `math` - `math/bits` - `strconv` + "math" + "math/bits" + "strconv" ) var ( - MinInt65 = Int65 { 0, 1 } - MaxInt65 = Int65 { math.MaxUint64, 0 } + MinInt65 = Int65{0, 1} + MaxInt65 = Int65{math.MaxUint64, 0} ) const ( - _MinInt65Str = "-18446744073709551616" + _MinInt65Str = "-18446744073709551616" ) type Int65 struct { - u uint64 - s uint64 + u uint64 + s uint64 } func Int65i(v int64) Int65 { - return Int65 { - u: uint64(v), - s: uint64(v) >> 63, - } + return Int65{ + u: uint64(v), + s: uint64(v) >> 63, + } } func (self Int65) String() string { - if self.s == 0 { - return strconv.FormatUint(self.u, 10) - } else if self.u != 0 { - return "-" + strconv.FormatUint(-self.u, 10) - } else { - return _MinInt65Str - } + if self.s == 0 { + return strconv.FormatUint(self.u, 10) + } else if self.u != 0 { + return "-" + strconv.FormatUint(-self.u, 10) + } else { + return _MinInt65Str + } } func (self Int65) OneLess() (r Int65) { - r.u, r.s = bits.Sub64(self.u, 1, 0) - r.s = (self.s - r.s) & 1 - return + r.u, r.s = bits.Sub64(self.u, 1, 0) + r.s = (self.s - r.s) & 1 + return } func (self Int65) OneMore() (r Int65) { - r.u, r.s = bits.Add64(self.u, 1, 0) - r.s = (self.s + r.s) & 1 - return + r.u, r.s = bits.Add64(self.u, 1, 0) + r.s = (self.s + r.s) & 1 + return } func (self Int65) Compare(other Int65) int { - if self.s == 0 && other.s != 0 { - return 1 - } else if self.s != 0 && other.s == 0 { - return -1 - } else { - return cmpu64(self.u, other.u) - } + if self.s == 0 && other.s != 0 { + return 1 + } else if self.s != 0 && other.s == 0 { + return -1 + } else { + return cmpu64(self.u, other.u) + } } func (self Int65) CompareZero() int { - if self.s != 0 { - return -1 - } else if self.u != 0 { - return 1 - } else { - return 0 - } + if self.s != 0 { + return -1 + } else if self.u != 0 { + return 1 + } else { + return 0 + } } diff --git a/internal/atm/ssa/int65_test.go b/internal/atm/ssa/int65_test.go index 8d82983..d5f18c6 100644 --- a/internal/atm/ssa/int65_test.go +++ b/internal/atm/ssa/int65_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,27 +17,27 @@ package ssa import ( - `testing` + "testing" ) func TestInt65_All(t *testing.T) { - println(" 0 - 1 =", Int65i(0).OneLess().String()) - println(" 0 =", Int65i(0).String()) - println(" 0 + 1 =", Int65i(0).OneMore().String()) - println("-------------------------------------------") - println(" 1 - 1 =", Int65i(1).OneLess().String()) - println(" 1 =", Int65i(1).String()) - println(" 1 + 1 =", Int65i(1).OneMore().String()) - println("-------------------------------------------") - println(" -1 - 1 =", Int65i(-1).OneLess().String()) - println(" -1 =", Int65i(-1).String()) - println(" -1 + 1 =", Int65i(-1).OneMore().String()) - println("-------------------------------------------") - println("MaxInt65 - 1 =", MaxInt65.OneLess().String()) - println("MaxInt65 =", MaxInt65.String()) - println("MaxInt65 + 1 =", MaxInt65.OneMore().String()) - println("-------------------------------------------") - println("MinInt65 - 1 =", MinInt65.OneLess().String()) - println("MinInt65 =", MinInt65.String()) - println("MinInt65 + 1 =", MinInt65.OneMore().String()) + println(" 0 - 1 =", Int65i(0).OneLess().String()) + println(" 0 =", Int65i(0).String()) + println(" 0 + 1 =", Int65i(0).OneMore().String()) + println("-------------------------------------------") + println(" 1 - 1 =", Int65i(1).OneLess().String()) + println(" 1 =", Int65i(1).String()) + println(" 1 + 1 =", Int65i(1).OneMore().String()) + println("-------------------------------------------") + println(" -1 - 1 =", Int65i(-1).OneLess().String()) + println(" -1 =", Int65i(-1).String()) + println(" -1 + 1 =", Int65i(-1).OneMore().String()) + println("-------------------------------------------") + println("MaxInt65 - 1 =", MaxInt65.OneLess().String()) + println("MaxInt65 =", MaxInt65.String()) + println("MaxInt65 + 1 =", MaxInt65.OneMore().String()) + println("-------------------------------------------") + println("MinInt65 - 1 =", MinInt65.OneLess().String()) + println("MinInt65 =", MinInt65.String()) + println("MinInt65 + 1 =", MinInt65.OneMore().String()) } diff --git a/internal/atm/ssa/ir.go b/internal/atm/ssa/ir.go index ad65359..938108a 100644 --- a/internal/atm/ssa/ir.go +++ b/internal/atm/ssa/ir.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,1183 +17,1246 @@ package ssa import ( - `fmt` - `sort` - `strings` - `unsafe` - - `github.com/cloudwego/frugal/internal/atm/abi` - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "fmt" + "sort" + "strings" + "unsafe" + + "github.com/cloudwego/frugal/internal/atm/abi" + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" ) type ( - Reg uint64 - Constness uint8 + Reg uint64 + Constness uint8 Likeliness uint8 ) const ( - _B_ptr = 63 - _B_kind = 60 - _B_name = 52 + _B_ptr = 63 + _B_kind = 60 + _B_name = 52 ) const ( - _M_ptr = 1 - _M_kind = 0x07 - _M_name = 0xff + _M_ptr = 1 + _M_kind = 0x07 + _M_name = 0xff ) const ( - _R_ptr = _M_ptr << _B_ptr - _R_kind = _M_kind << _B_kind - _R_name = _M_name << _B_name - _R_index = (1 << _B_name) - 1 + _R_ptr = _M_ptr << _B_ptr + _R_kind = _M_kind << _B_kind + _R_name = _M_name << _B_name + _R_index = (1 << _B_name) - 1 ) const ( - K_sys = 0 - K_zero = 1 - K_temp = 2 - K_arch = 3 - K_norm = 4 + K_sys = 0 + K_zero = 1 + K_temp = 2 + K_arch = 3 + K_norm = 4 ) const ( - N_size = _M_name + 1 + N_size = _M_name + 1 ) const ( - Rz Reg = (0 << _B_ptr) | (K_zero << _B_kind) - Pn Reg = (1 << _B_ptr) | (K_zero << _B_kind) + Rz Reg = (0 << _B_ptr) | (K_zero << _B_kind) + Pn Reg = (1 << _B_ptr) | (K_zero << _B_kind) ) const ( - Const Constness = iota - Volatile + Const Constness = iota + Volatile ) const ( - Likely Likeliness = iota - Unlikely + Likely Likeliness = iota + Unlikely ) func (self Constness) String() string { - switch self { - case Const : return "const" - case Volatile : return "volatile" - default : return "???" - } + switch self { + case Const: + return "const" + case Volatile: + return "volatile" + default: + return "???" + } } func (self Likeliness) String() string { - switch self { - case Likely : return "likely" - case Unlikely : return "unlikely" - default : return "???" - } + switch self { + case Likely: + return "likely" + case Unlikely: + return "unlikely" + default: + return "???" + } } func mksys(ptr uint64, kind uint64) Reg { - if kind > N_size { - panic(fmt.Sprintf("invalid register kind: %d", kind)) - } else { - return mkreg(ptr, K_sys, kind) - } + if kind > N_size { + panic(fmt.Sprintf("invalid register kind: %d", kind)) + } else { + return mkreg(ptr, K_sys, kind) + } } func mkreg(ptr uint64, kind uint64, name uint64) Reg { - return Reg(((ptr & _M_ptr) << _B_ptr) | ((kind & _M_kind) << _B_kind) | ((name & _M_name) << _B_name)) + return Reg(((ptr & _M_ptr) << _B_ptr) | ((kind & _M_kind) << _B_kind) | ((name & _M_name) << _B_name)) } func Tr(i int) Reg { - if i < 0 || i > N_size { - panic("invalid generic temporary register index") - } else { - return mkreg(0, K_temp, uint64(i)) - } + if i < 0 || i > N_size { + panic("invalid generic temporary register index") + } else { + return mkreg(0, K_temp, uint64(i)) + } } func Pr(i int) Reg { - if i < 0 || i > N_size { - panic("invalid generic temporary register index") - } else { - return mkreg(1, K_temp, uint64(i)) - } + if i < 0 || i > N_size { + panic("invalid generic temporary register index") + } else { + return mkreg(1, K_temp, uint64(i)) + } } func Rv(reg hir.Register) Reg { - switch r := reg.(type) { - case hir.GenericRegister : if r == hir.Rz { return Rz } else { return mksys(0, uint64(r)) } - case hir.PointerRegister : if r == hir.Pn { return Pn } else { return mksys(1, uint64(r)) } - default : panic("unreachable") - } + switch r := reg.(type) { + case hir.GenericRegister: + if r == hir.Rz { + return Rz + } else { + return mksys(0, uint64(r)) + } + case hir.PointerRegister: + if r == hir.Pn { + return Pn + } else { + return mksys(1, uint64(r)) + } + default: + panic("unreachable") + } } func (self Reg) Ptr() bool { - return self & _R_ptr != 0 + return self&_R_ptr != 0 } func (self Reg) Zero() Reg { - return (self & _R_ptr) | (K_zero << _B_kind) + return (self & _R_ptr) | (K_zero << _B_kind) } func (self Reg) Kind() int { - return int((self & _R_kind) >> _B_kind) + return int((self & _R_kind) >> _B_kind) } func (self Reg) Name() int { - return int((self & _R_name) >> _B_name) + return int((self & _R_name) >> _B_name) } func (self Reg) Index() int { - return int(self & _R_index) + return int(self & _R_index) } func (self Reg) String() string { - switch self.Kind() { - default: { - if self.Ptr() { - return fmt.Sprintf("p%d.%d", self.Kind(), self.Index()) - } else { - return fmt.Sprintf("r%d.%d", self.Kind(), self.Index()) - } - } - - /* physical registers */ - case K_arch: { - if i := self.Name(); i >= len(ArchRegs) { - panic(fmt.Sprintf("invalid physical register index: %d", i)) - } else if self.Index() == 0 { - return fmt.Sprintf("%%%s", ArchRegNames[ArchRegs[i]]) - } else if self.Ptr() { - return fmt.Sprintf("p%d:%%%s", self.Index(), ArchRegNames[ArchRegs[i]]) - } else { - return fmt.Sprintf("r%d:%%%s", self.Index(), ArchRegNames[ArchRegs[i]]) - } - } - - /* zero registers */ - case K_zero: { - if self.Ptr() { - return "nil" - } else { - return "zero" - } - } - - /* temp registers */ - case K_temp: { - if self.Ptr() { - return fmt.Sprintf("tp%d.%d", self.Name(), self.Index()) - } else { - return fmt.Sprintf("tr%d.%d", self.Name(), self.Index()) - } - } - - /* SSA normalized registers */ - case K_norm: { - if self.Ptr() { - return fmt.Sprintf("p%d", self.Index()) - } else { - return fmt.Sprintf("r%d", self.Index()) - } - } - } + switch self.Kind() { + default: + { + if self.Ptr() { + return fmt.Sprintf("p%d.%d", self.Kind(), self.Index()) + } else { + return fmt.Sprintf("r%d.%d", self.Kind(), self.Index()) + } + } + + /* physical registers */ + case K_arch: + { + if i := self.Name(); i >= len(ArchRegs) { + panic(fmt.Sprintf("invalid physical register index: %d", i)) + } else if self.Index() == 0 { + return fmt.Sprintf("%%%s", ArchRegNames[ArchRegs[i]]) + } else if self.Ptr() { + return fmt.Sprintf("p%d:%%%s", self.Index(), ArchRegNames[ArchRegs[i]]) + } else { + return fmt.Sprintf("r%d:%%%s", self.Index(), ArchRegNames[ArchRegs[i]]) + } + } + + /* zero registers */ + case K_zero: + { + if self.Ptr() { + return "nil" + } else { + return "zero" + } + } + + /* temp registers */ + case K_temp: + { + if self.Ptr() { + return fmt.Sprintf("tp%d.%d", self.Name(), self.Index()) + } else { + return fmt.Sprintf("tr%d.%d", self.Name(), self.Index()) + } + } + + /* SSA normalized registers */ + case K_norm: + { + if self.Ptr() { + return fmt.Sprintf("p%d", self.Index()) + } else { + return fmt.Sprintf("r%d", self.Index()) + } + } + } } func (self Reg) Derive(i int) Reg { - if self.Kind() == K_zero { - return self - } else { - return (self & (_R_ptr | _R_kind | _R_name)) | Reg(i & _R_index) - } + if self.Kind() == K_zero { + return self + } else { + return (self & (_R_ptr | _R_kind | _R_name)) | Reg(i&_R_index) + } } func (self Reg) Normalize(i int) Reg { - if self.Kind() == K_zero { - return self - } else { - return (self & _R_ptr) | (K_norm << _B_kind) | Reg(i & _R_index) - } + if self.Kind() == K_zero { + return self + } else { + return (self & _R_ptr) | (K_norm << _B_kind) | Reg(i&_R_index) + } } type IrNode interface { - fmt.Stringer - Clone() IrNode - irnode() + fmt.Stringer + Clone() IrNode + irnode() } type IrImpure interface { - IrNode - irimpure() + IrNode + irimpure() } type IrImmovable interface { - IrNode - irimmovable() -} - -func (*IrPhi) irnode() {} -func (*IrSwitch) irnode() {} -func (*IrReturn) irnode() {} -func (*IrNop) irnode() {} -func (*IrBreakpoint) irnode() {} -func (*IrAlias) irnode() {} -func (*IrEntry) irnode() {} -func (*IrLoad) irnode() {} -func (*IrStore) irnode() {} -func (*IrLoadArg) irnode() {} -func (*IrConstInt) irnode() {} -func (*IrConstPtr) irnode() {} -func (*IrLEA) irnode() {} -func (*IrUnaryExpr) irnode() {} -func (*IrBinaryExpr) irnode() {} -func (*IrBitTestSet) irnode() {} -func (*IrCallFunc) irnode() {} -func (*IrCallNative) irnode() {} -func (*IrCallMethod) irnode() {} -func (*IrClobberList) irnode() {} + IrNode + irimmovable() +} + +func (*IrPhi) irnode() {} +func (*IrSwitch) irnode() {} +func (*IrReturn) irnode() {} +func (*IrNop) irnode() {} +func (*IrBreakpoint) irnode() {} +func (*IrAlias) irnode() {} +func (*IrEntry) irnode() {} +func (*IrLoad) irnode() {} +func (*IrStore) irnode() {} +func (*IrLoadArg) irnode() {} +func (*IrConstInt) irnode() {} +func (*IrConstPtr) irnode() {} +func (*IrLEA) irnode() {} +func (*IrUnaryExpr) irnode() {} +func (*IrBinaryExpr) irnode() {} +func (*IrBitTestSet) irnode() {} +func (*IrCallFunc) irnode() {} +func (*IrCallNative) irnode() {} +func (*IrCallMethod) irnode() {} +func (*IrClobberList) irnode() {} func (*IrWriteBarrier) irnode() {} -func (*IrSpill) irnode() {} -func (*IrSlotAlive) irnode() {} - -func (*IrStore) irimpure() {} -func (*IrCallFunc) irimpure() {} -func (*IrCallNative) irimpure() {} -func (*IrCallMethod) irimpure() {} -func (*IrClobberList) irimpure() {} +func (*IrSpill) irnode() {} +func (*IrSlotAlive) irnode() {} + +func (*IrStore) irimpure() {} +func (*IrCallFunc) irimpure() {} +func (*IrCallNative) irimpure() {} +func (*IrCallMethod) irimpure() {} +func (*IrClobberList) irimpure() {} func (*IrWriteBarrier) irimpure() {} -func (*IrSpill) irimpure() {} - -func (*IrLoad) irimmovable() {} -func (*IrStore) irimmovable() {} -func (*IrAlias) irimmovable() {} -func (*IrEntry) irimmovable() {} -func (*IrLoadArg) irimmovable() {} -func (*IrClobberList) irimmovable() {} +func (*IrSpill) irimpure() {} + +func (*IrLoad) irimmovable() {} +func (*IrStore) irimmovable() {} +func (*IrAlias) irimmovable() {} +func (*IrEntry) irimmovable() {} +func (*IrLoadArg) irimmovable() {} +func (*IrClobberList) irimmovable() {} func (*IrWriteBarrier) irimmovable() {} -func (*IrSpill) irimmovable() {} -func (*IrSlotAlive) irimmovable() {} +func (*IrSpill) irimmovable() {} +func (*IrSlotAlive) irimmovable() {} type IrUsages interface { - IrNode - Usages() []*Reg + IrNode + Usages() []*Reg } type IrDefinitions interface { - IrNode - Definitions() []*Reg + IrNode + Definitions() []*Reg } type _PhiSorter struct { - k []int - v []*Reg + k []int + v []*Reg } func (self _PhiSorter) Len() int { - return len(self.k) + return len(self.k) } func (self _PhiSorter) Swap(i int, j int) { - self.k[i], self.k[j] = self.k[j], self.k[i] - self.v[i], self.v[j] = self.v[j], self.v[i] + self.k[i], self.k[j] = self.k[j], self.k[i] + self.v[i], self.v[j] = self.v[j], self.v[i] } func (self _PhiSorter) Less(i int, j int) bool { - return self.k[i] < self.k[j] + return self.k[i] < self.k[j] } type IrPhi struct { - R Reg - V map[*BasicBlock]*Reg + R Reg + V map[*BasicBlock]*Reg } func (self *IrPhi) Clone() IrNode { - ret := new(IrPhi) - ret.V = make(map[*BasicBlock]*Reg, len(self.V)) + ret := new(IrPhi) + ret.V = make(map[*BasicBlock]*Reg, len(self.V)) - /* clone the Phi mappings */ - for b, r := range self.V { - p := *r - ret.V[b] = &p - } + /* clone the Phi mappings */ + for b, r := range self.V { + p := *r + ret.V[b] = &p + } - /* set the dest register */ - ret.R = self.R - return ret + /* set the dest register */ + ret.R = self.R + return ret } func (self *IrPhi) String() string { - nb := len(self.V) - ret := make([]string, 0, nb) - phi := make([]struct { int; Reg }, 0, nb) - - /* add each path */ - for bb, reg := range self.V { - phi = append(phi, struct { int; Reg }{ bb.Id, *reg }) - } - - /* sort by basic block ID */ - sort.Slice(phi, func(i int, j int) bool { - return phi[i].int < phi[j].int - }) - - /* dump as string */ - for _, p := range phi { - ret = append(ret, fmt.Sprintf("bb_%d: %s", p.int, p.Reg)) - } - - /* join them together */ - return fmt.Sprintf( - "%s = φ(%s)", - self.R, - strings.Join(ret, ", "), - ) + nb := len(self.V) + ret := make([]string, 0, nb) + phi := make([]struct { + int + Reg + }, 0, nb) + + /* add each path */ + for bb, reg := range self.V { + phi = append(phi, struct { + int + Reg + }{bb.Id, *reg}) + } + + /* sort by basic block ID */ + sort.Slice(phi, func(i int, j int) bool { + return phi[i].int < phi[j].int + }) + + /* dump as string */ + for _, p := range phi { + ret = append(ret, fmt.Sprintf("bb_%d: %s", p.int, p.Reg)) + } + + /* join them together */ + return fmt.Sprintf( + "%s = φ(%s)", + self.R, + strings.Join(ret, ", "), + ) } func (self *IrPhi) Usages() []*Reg { - k := make([]int, 0, len(self.V)) - v := make([]*Reg, 0, len(self.V)) + k := make([]int, 0, len(self.V)) + v := make([]*Reg, 0, len(self.V)) - /* dump the registers */ - for b, r := range self.V { - v = append(v, r) - k = append(k, b.Id) - } + /* dump the registers */ + for b, r := range self.V { + v = append(v, r) + k = append(k, b.Id) + } - /* sort by basic block ID */ - sort.Sort(_PhiSorter { k, v }) - return v + /* sort by basic block ID */ + sort.Sort(_PhiSorter{k, v}) + return v } func (self *IrPhi) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrBranch struct { - To *BasicBlock - Likeliness Likeliness + To *BasicBlock + Likeliness Likeliness } func IrLikely(bb *BasicBlock) *IrBranch { - return &IrBranch { - To : bb, - Likeliness : Likely, - } + return &IrBranch{ + To: bb, + Likeliness: Likely, + } } func IrUnlikely(bb *BasicBlock) *IrBranch { - return &IrBranch { - To : bb, - Likeliness : Unlikely, - } + return &IrBranch{ + To: bb, + Likeliness: Unlikely, + } } func (self *IrBranch) Clone() *IrBranch { - return &IrBranch { - To : self.To, - Likeliness : self.Likeliness, - } + return &IrBranch{ + To: self.To, + Likeliness: self.Likeliness, + } } func (self *IrBranch) String() string { - return fmt.Sprintf("bb_%d (%s)", self.To.Id, self.Likeliness) + return fmt.Sprintf("bb_%d (%s)", self.To.Id, self.Likeliness) } type IrSuccessors interface { - Next() bool - Block() *BasicBlock - Value() (int32, bool) - Likeliness() Likeliness - UpdateBlock(bb *BasicBlock) + Next() bool + Block() *BasicBlock + Value() (int32, bool) + Likeliness() Likeliness + UpdateBlock(bb *BasicBlock) } type IrTerminator interface { - IrNode - Successors() IrSuccessors - irterminator() + IrNode + Successors() IrSuccessors + irterminator() } func (*IrSwitch) irterminator() {} func (*IrReturn) irterminator() {} type _SwitchTarget struct { - i int32 - b *IrBranch + i int32 + b *IrBranch } type _SwitchSuccessors struct { - i int - t []_SwitchTarget + i int + t []_SwitchTarget } func (self *_SwitchSuccessors) Next() bool { - self.i++ - return self.i < len(self.t) + self.i++ + return self.i < len(self.t) } func (self *_SwitchSuccessors) Block() *BasicBlock { - if self.i >= len(self.t) { - return nil - } else { - return self.t[self.i].b.To - } + if self.i >= len(self.t) { + return nil + } else { + return self.t[self.i].b.To + } } func (self *_SwitchSuccessors) Value() (int32, bool) { - if self.i >= len(self.t) - 1 { - return 0, false - } else { - return self.t[self.i].i, true - } + if self.i >= len(self.t)-1 { + return 0, false + } else { + return self.t[self.i].i, true + } } func (self *_SwitchSuccessors) Likeliness() Likeliness { - if self.i >= len(self.t) { - return Unlikely - } else { - return self.t[self.i].b.Likeliness - } + if self.i >= len(self.t) { + return Unlikely + } else { + return self.t[self.i].b.Likeliness + } } func (self *_SwitchSuccessors) UpdateBlock(to *BasicBlock) { - if self.i >= len(self.t) { - panic("end of iterator") - } else { - self.t[self.i].b.To = to - } + if self.i >= len(self.t) { + panic("end of iterator") + } else { + self.t[self.i].b.To = to + } } type IrSwitch struct { - V Reg - Ln *IrBranch - Br map[int32]*IrBranch + V Reg + Ln *IrBranch + Br map[int32]*IrBranch } func (self *IrSwitch) iter() *_SwitchSuccessors { - n := len(self.Br) - t := make([]_SwitchTarget, 0, n + 1) - - /* add the key and values */ - for i, b := range self.Br { - t = append(t, _SwitchTarget { - i: i, - b: b, - }) - } - - /* add the default branch */ - t = append(t, _SwitchTarget { - i: 0, - b: self.Ln, - }) - - /* sort by switch value */ - sort.Slice(t[:n], func(i int, j int) bool { - return t[i].i < t[j].i - }) - - /* construct the iterator */ - return &_SwitchSuccessors { - t: t, - i: -1, - } + n := len(self.Br) + t := make([]_SwitchTarget, 0, n+1) + + /* add the key and values */ + for i, b := range self.Br { + t = append(t, _SwitchTarget{ + i: i, + b: b, + }) + } + + /* add the default branch */ + t = append(t, _SwitchTarget{ + i: 0, + b: self.Ln, + }) + + /* sort by switch value */ + sort.Slice(t[:n], func(i int, j int) bool { + return t[i].i < t[j].i + }) + + /* construct the iterator */ + return &_SwitchSuccessors{ + t: t, + i: -1, + } } func (self *IrSwitch) Clone() IrNode { - ret := new(IrSwitch) - ret.Br = make(map[int32]*IrBranch, len(ret.Br)) + ret := new(IrSwitch) + ret.Br = make(map[int32]*IrBranch, len(ret.Br)) - /* clone the switch branches */ - for v, b := range self.Br { - ret.Br[v] = b.Clone() - } + /* clone the switch branches */ + for v, b := range self.Br { + ret.Br[v] = b.Clone() + } - /* set the switch register and default branch */ - ret.V = self.V - ret.Ln = self.Ln.Clone() - return ret + /* set the switch register and default branch */ + ret.V = self.V + ret.Ln = self.Ln.Clone() + return ret } func (self *IrSwitch) String() string { - n := len(self.Br) - r := make([]string, 0, n) + n := len(self.Br) + r := make([]string, 0, n) - /* no branches */ - if n == 0 { - return "goto " + self.Ln.String() - } + /* no branches */ + if n == 0 { + return "goto " + self.Ln.String() + } - /* add each case */ - for _, v := range self.iter().t[:n] { - r = append(r, fmt.Sprintf(" %d => %s,", v.i, v.b)) - } + /* add each case */ + for _, v := range self.iter().t[:n] { + r = append(r, fmt.Sprintf(" %d => %s,", v.i, v.b)) + } - /* default branch */ - r = append(r, fmt.Sprintf( - " _ => %s,", - self.Ln, - )) + /* default branch */ + r = append(r, fmt.Sprintf( + " _ => %s,", + self.Ln, + )) - /* join them together */ - return fmt.Sprintf( - "switch %s {\n%s\n}", - self.V, - strings.Join(r, "\n"), - ) + /* join them together */ + return fmt.Sprintf( + "switch %s {\n%s\n}", + self.V, + strings.Join(r, "\n"), + ) } func (self *IrSwitch) Usages() []*Reg { - if len(self.Br) == 0 { - return nil - } else { - return []*Reg { &self.V } - } + if len(self.Br) == 0 { + return nil + } else { + return []*Reg{&self.V} + } } func (self *IrSwitch) Successors() IrSuccessors { - return self.iter() + return self.iter() } type _EmptySuccessor struct{} -func (_EmptySuccessor) Next() bool { return false } -func (_EmptySuccessor) Block() *BasicBlock { return nil } -func (_EmptySuccessor) Value() (int32, bool) { return 0, false } + +func (_EmptySuccessor) Next() bool { return false } +func (_EmptySuccessor) Block() *BasicBlock { return nil } +func (_EmptySuccessor) Value() (int32, bool) { return 0, false } func (_EmptySuccessor) Likeliness() Likeliness { return Unlikely } func (_EmptySuccessor) UpdateBlock(_ *BasicBlock) { panic("empty iterator") } type IrReturn struct { - R []Reg + R []Reg } func (self *IrReturn) Clone() IrNode { - r := new(IrReturn) - r.R = make([]Reg, len(self.R)) - copy(r.R, self.R) - return r + r := new(IrReturn) + r.R = make([]Reg, len(self.R)) + copy(r.R, self.R) + return r } func (self *IrReturn) String() string { - nb := len(self.R) - ret := make([]string, 0, nb) + nb := len(self.R) + ret := make([]string, 0, nb) - /* dump registers */ - for _, r := range self.R { - ret = append(ret, r.String()) - } + /* dump registers */ + for _, r := range self.R { + ret = append(ret, r.String()) + } - /* join them together */ - return fmt.Sprintf( - "ret {%s}", - strings.Join(ret, ", "), - ) + /* join them together */ + return fmt.Sprintf( + "ret {%s}", + strings.Join(ret, ", "), + ) } func (self *IrReturn) Usages() []*Reg { - return regsliceref(self.R) + return regsliceref(self.R) } func (self *IrReturn) Successors() IrSuccessors { - return _EmptySuccessor{} + return _EmptySuccessor{} } type ( - IrNop struct{} - IrBreakpoint struct{} + IrNop struct{} + IrBreakpoint struct{} ) -func (*IrNop) Clone() IrNode { return new(IrNop) } +func (*IrNop) Clone() IrNode { return new(IrNop) } func (*IrBreakpoint) Clone() IrNode { return new(IrBreakpoint) } -func (*IrNop) String() string { return "nop" } +func (*IrNop) String() string { return "nop" } func (*IrBreakpoint) String() string { return "breakpoint" } type IrAlias struct { - R Reg - V Reg + R Reg + V Reg } func (self *IrAlias) Clone() IrNode { - panic(`alias node "` + self.String() + `" is not cloneable`) + panic(`alias node "` + self.String() + `" is not cloneable`) } func (self *IrAlias) String() string { - return fmt.Sprintf("alias %s = %s", self.R, self.V) + return fmt.Sprintf("alias %s = %s", self.R, self.V) } func (self *IrAlias) Usages() []*Reg { - return []*Reg { &self.V } + return []*Reg{&self.V} } func (self *IrAlias) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrEntry struct { - R []Reg + R []Reg } func (self *IrEntry) Clone() IrNode { - panic(`entry node "` + self.String() + `" is not cloneable`) + panic(`entry node "` + self.String() + `" is not cloneable`) } func (self *IrEntry) String() string { - return "entry_point " + regslicerepr(self.R) + return "entry_point " + regslicerepr(self.R) } func (self *IrEntry) Definitions() []*Reg { - return regsliceref(self.R) + return regsliceref(self.R) } type IrLoad struct { - R Reg - Mem Reg - Size uint8 + R Reg + Mem Reg + Size uint8 } func (self *IrLoad) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrLoad) String() string { - if self.R.Ptr() { - return fmt.Sprintf("%s = load.ptr %s", self.R, self.Mem) - } else { - return fmt.Sprintf("%s = load.u%d %s", self.R, self.Size * 8, self.Mem) - } + if self.R.Ptr() { + return fmt.Sprintf("%s = load.ptr %s", self.R, self.Mem) + } else { + return fmt.Sprintf("%s = load.u%d %s", self.R, self.Size*8, self.Mem) + } } func (self *IrLoad) Usages() []*Reg { - return []*Reg { &self.Mem } + return []*Reg{&self.Mem} } func (self *IrLoad) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrStore struct { - R Reg - Mem Reg - Size uint8 + R Reg + Mem Reg + Size uint8 } func (self *IrStore) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrStore) String() string { - return fmt.Sprintf("store.u%d %s -> *%s", self.Size * 8, self.R, self.Mem) + return fmt.Sprintf("store.u%d %s -> *%s", self.Size*8, self.R, self.Mem) } func (self *IrStore) Usages() []*Reg { - return []*Reg { &self.R, &self.Mem } + return []*Reg{&self.R, &self.Mem} } type IrLoadArg struct { - R Reg - I int + R Reg + I int } func (self *IrLoadArg) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrLoadArg) String() string { - if self.R.Ptr() { - return fmt.Sprintf("%s = loadarg.ptr #%d", self.R, self.I) - } else { - return fmt.Sprintf("%s = loadarg.i64 #%d", self.R, self.I) - } + if self.R.Ptr() { + return fmt.Sprintf("%s = loadarg.ptr #%d", self.R, self.I) + } else { + return fmt.Sprintf("%s = loadarg.i64 #%d", self.R, self.I) + } } func (self *IrLoadArg) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrConstInt struct { - R Reg - V int64 + R Reg + V int64 } func (self *IrConstInt) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrConstInt) String() string { - return fmt.Sprintf("%s = const.i64 %d (%#x)", self.R, self.V, self.V) + return fmt.Sprintf("%s = const.i64 %d (%#x)", self.R, self.V, self.V) } func (self *IrConstInt) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrConstPtr struct { - R Reg - P unsafe.Pointer - M Constness + R Reg + P unsafe.Pointer + M Constness } func (self *IrConstPtr) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrConstPtr) String() string { - return fmt.Sprintf("%s = const.ptr (%s)%p [%s]", self.R, self.M, self.P, rt.FuncName(self.P)) + return fmt.Sprintf("%s = const.ptr (%s)%p [%s]", self.R, self.M, self.P, rt.FuncName(self.P)) } func (self *IrConstPtr) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrLEA struct { - R Reg - Mem Reg - Off Reg + R Reg + Mem Reg + Off Reg } func (self *IrLEA) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrLEA) String() string { - return fmt.Sprintf("%s = &(%s)[%s]", self.R, self.Mem, self.Off) + return fmt.Sprintf("%s = &(%s)[%s]", self.R, self.Mem, self.Off) } func (self *IrLEA) Usages() []*Reg { - return []*Reg { &self.Mem, &self.Off } + return []*Reg{&self.Mem, &self.Off} } func (self *IrLEA) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type ( - IrUnaryOp uint8 - IrBinaryOp uint8 + IrUnaryOp uint8 + IrBinaryOp uint8 ) const ( - IrOpNegate IrUnaryOp = iota - IrOpSwap16 - IrOpSwap32 - IrOpSwap64 - IrOpSx32to64 + IrOpNegate IrUnaryOp = iota + IrOpSwap16 + IrOpSwap32 + IrOpSwap64 + IrOpSx32to64 ) const ( - IrOpAdd IrBinaryOp = iota - IrOpSub - IrOpMul - IrOpAnd - IrOpOr - IrOpXor - IrOpShr - IrCmpEq - IrCmpNe - IrCmpLt - IrCmpLtu - IrCmpGeu + IrOpAdd IrBinaryOp = iota + IrOpSub + IrOpMul + IrOpAnd + IrOpOr + IrOpXor + IrOpShr + IrCmpEq + IrCmpNe + IrCmpLt + IrCmpLtu + IrCmpGeu ) func (self IrUnaryOp) String() string { - switch self { - case IrOpNegate : return "negate" - case IrOpSwap16 : return "bswap16" - case IrOpSwap32 : return "bswap32" - case IrOpSwap64 : return "bswap64" - case IrOpSx32to64 : return "sign_extend_32_to_64" - default : panic("unreachable") - } + switch self { + case IrOpNegate: + return "negate" + case IrOpSwap16: + return "bswap16" + case IrOpSwap32: + return "bswap32" + case IrOpSwap64: + return "bswap64" + case IrOpSx32to64: + return "sign_extend_32_to_64" + default: + panic("unreachable") + } } func (self IrBinaryOp) String() string { - switch self { - case IrOpAdd : return "+" - case IrOpSub : return "-" - case IrOpMul : return "*" - case IrOpAnd : return "&" - case IrOpOr : return "|" - case IrOpXor : return "^" - case IrOpShr : return ">>" - case IrCmpEq : return "==" - case IrCmpNe : return "!=" - case IrCmpLt : return "<" - case IrCmpLtu : return "<#" - case IrCmpGeu : return ">=#" - default : panic("unreachable") - } + switch self { + case IrOpAdd: + return "+" + case IrOpSub: + return "-" + case IrOpMul: + return "*" + case IrOpAnd: + return "&" + case IrOpOr: + return "|" + case IrOpXor: + return "^" + case IrOpShr: + return ">>" + case IrCmpEq: + return "==" + case IrCmpNe: + return "!=" + case IrCmpLt: + return "<" + case IrCmpLtu: + return "<#" + case IrCmpGeu: + return ">=#" + default: + panic("unreachable") + } } type IrUnaryExpr struct { - R Reg - V Reg - Op IrUnaryOp + R Reg + V Reg + Op IrUnaryOp } func (self *IrUnaryExpr) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrUnaryExpr) String() string { - return fmt.Sprintf("%s = %s %s", self.R, self.Op, self.V) + return fmt.Sprintf("%s = %s %s", self.R, self.Op, self.V) } func (self *IrUnaryExpr) Usages() []*Reg { - return []*Reg { &self.V } + return []*Reg{&self.V} } func (self *IrUnaryExpr) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrBinaryExpr struct { - R Reg - X Reg - Y Reg - Op IrBinaryOp + R Reg + X Reg + Y Reg + Op IrBinaryOp } func IrCopy(r Reg, v Reg) IrNode { - switch { - case r.Ptr() && v.Ptr() : return &IrLEA { R: r, Mem: v, Off: Rz } - case !r.Ptr() && !v.Ptr() : return &IrBinaryExpr { R: r, X: v, Y: Rz, Op: IrOpAdd } - default : panic("copy between different kind of registers") - } + switch { + case r.Ptr() && v.Ptr(): + return &IrLEA{R: r, Mem: v, Off: Rz} + case !r.Ptr() && !v.Ptr(): + return &IrBinaryExpr{R: r, X: v, Y: Rz, Op: IrOpAdd} + default: + panic("copy between different kind of registers") + } } func IrTryIntoCopy(v IrNode) (Reg, Reg, bool) { - if p, ok := v.(*IrAlias); ok { - return p.R, p.V, true - } else if p, ok := v.(*IrLEA); ok && p.Off == Rz { - return p.R, p.Mem, true - } else if p, ok := v.(*IrBinaryExpr); ok && p.Y == Rz && p.Op == IrOpAdd { - return p.R, p.X, true - } else { - return 0, 0, false - } + if p, ok := v.(*IrAlias); ok { + return p.R, p.V, true + } else if p, ok := v.(*IrLEA); ok && p.Off == Rz { + return p.R, p.Mem, true + } else if p, ok := v.(*IrBinaryExpr); ok && p.Y == Rz && p.Op == IrOpAdd { + return p.R, p.X, true + } else { + return 0, 0, false + } } func (self *IrBinaryExpr) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrBinaryExpr) String() string { - return fmt.Sprintf("%s = %s %s %s", self.R, self.X, self.Op, self.Y) + return fmt.Sprintf("%s = %s %s %s", self.R, self.X, self.Op, self.Y) } func (self *IrBinaryExpr) Usages() []*Reg { - return []*Reg { &self.X, &self.Y } + return []*Reg{&self.X, &self.Y} } func (self *IrBinaryExpr) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrBitTestSet struct { - T Reg - S Reg - X Reg - Y Reg + T Reg + S Reg + X Reg + Y Reg } func (self *IrBitTestSet) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrBitTestSet) String() string { - return fmt.Sprintf("t.%s, s.%s = bts %s, %s", self.T, self.S, self.X, self.Y) + return fmt.Sprintf("t.%s, s.%s = bts %s, %s", self.T, self.S, self.X, self.Y) } func (self *IrBitTestSet) Usages() []*Reg { - return []*Reg { &self.X, &self.Y } + return []*Reg{&self.X, &self.Y} } func (self *IrBitTestSet) Definitions() []*Reg { - return []*Reg { &self.T, &self.S } + return []*Reg{&self.T, &self.S} } type IrCallFunc struct { - R Reg - In []Reg - Out []Reg - Func *abi.FunctionLayout + R Reg + In []Reg + Out []Reg + Func *abi.FunctionLayout } func (self *IrCallFunc) Clone() IrNode { - r := new(IrCallFunc) - r.R = self.R - r.In = make([]Reg, len(self.In)) - r.Out = make([]Reg, len(self.Out)) - r.Func = self.Func - copy(r.In, self.In) - copy(r.Out, self.Out) - return r + r := new(IrCallFunc) + r.R = self.R + r.In = make([]Reg, len(self.In)) + r.Out = make([]Reg, len(self.Out)) + r.Func = self.Func + copy(r.In, self.In) + copy(r.Out, self.Out) + return r } func (self *IrCallFunc) String() string { - if in := regslicerepr(self.In); len(self.Out) == 0 { - return fmt.Sprintf("gcall *%s, {%s}", self.R, in) - } else { - return fmt.Sprintf("%s = gcall *%s, {%s}", regslicerepr(self.Out), self.R, in) - } + if in := regslicerepr(self.In); len(self.Out) == 0 { + return fmt.Sprintf("gcall *%s, {%s}", self.R, in) + } else { + return fmt.Sprintf("%s = gcall *%s, {%s}", regslicerepr(self.Out), self.R, in) + } } func (self *IrCallFunc) Usages() []*Reg { - return append(regsliceref(self.In), &self.R) + return append(regsliceref(self.In), &self.R) } func (self *IrCallFunc) Definitions() []*Reg { - return regsliceref(self.Out) + return regsliceref(self.Out) } type IrCallNative struct { - R Reg - In []Reg - Out Reg + R Reg + In []Reg + Out Reg } func (self *IrCallNative) Clone() IrNode { - r := new(IrCallNative) - r.R = self.R - r.In = make([]Reg, len(self.In)) - r.Out = self.Out - copy(r.In, self.In) - return r + r := new(IrCallNative) + r.R = self.R + r.In = make([]Reg, len(self.In)) + r.Out = self.Out + copy(r.In, self.In) + return r } func (self *IrCallNative) String() string { - if in := regslicerepr(self.In); self.Out.Kind() == K_zero { - return fmt.Sprintf("ccall *%s, {%s}", self.R, in) - } else { - return fmt.Sprintf("%s = ccall *%s, {%s}", self.Out, self.R, in) - } + if in := regslicerepr(self.In); self.Out.Kind() == K_zero { + return fmt.Sprintf("ccall *%s, {%s}", self.R, in) + } else { + return fmt.Sprintf("%s = ccall *%s, {%s}", self.Out, self.R, in) + } } func (self *IrCallNative) Usages() []*Reg { - return append(regsliceref(self.In), &self.R) + return append(regsliceref(self.In), &self.R) } func (self *IrCallNative) Definitions() []*Reg { - if self.Out.Kind() == K_zero { - return nil - } else { - return []*Reg { &self.Out } - } + if self.Out.Kind() == K_zero { + return nil + } else { + return []*Reg{&self.Out} + } } type IrCallMethod struct { - T Reg - V Reg - In []Reg - Out []Reg - Slot int - Func *abi.FunctionLayout + T Reg + V Reg + In []Reg + Out []Reg + Slot int + Func *abi.FunctionLayout } func (self *IrCallMethod) Clone() IrNode { - r := new(IrCallMethod) - r.T = self.T - r.V = self.V - r.In = make([]Reg, len(self.In)) - r.Out = make([]Reg, len(self.Out)) - r.Slot = self.Slot - r.Func = self.Func - copy(r.In, self.In) - copy(r.Out, self.Out) - return r + r := new(IrCallMethod) + r.T = self.T + r.V = self.V + r.In = make([]Reg, len(self.In)) + r.Out = make([]Reg, len(self.Out)) + r.Slot = self.Slot + r.Func = self.Func + copy(r.In, self.In) + copy(r.Out, self.Out) + return r } func (self *IrCallMethod) String() string { - if in := regslicerepr(self.In); len(self.Out) == 0 { - return fmt.Sprintf("icall #%d, (%s:%s), {%s}", self.Slot, self.T, self.V, in) - } else { - return fmt.Sprintf("%s = icall #%d, (%s:%s), {%s}", regslicerepr(self.Out), self.Slot, self.T, self.V, in) - } + if in := regslicerepr(self.In); len(self.Out) == 0 { + return fmt.Sprintf("icall #%d, (%s:%s), {%s}", self.Slot, self.T, self.V, in) + } else { + return fmt.Sprintf("%s = icall #%d, (%s:%s), {%s}", regslicerepr(self.Out), self.Slot, self.T, self.V, in) + } } func (self *IrCallMethod) Usages() []*Reg { - return append(regsliceref(self.In), &self.T, &self.V) + return append(regsliceref(self.In), &self.T, &self.V) } func (self *IrCallMethod) Definitions() []*Reg { - return regsliceref(self.Out) + return regsliceref(self.Out) } type IrClobberList struct { - R []Reg + R []Reg } func IrMarkClobber(r ...Reg) *IrClobberList { - return &IrClobberList { - R: regsliceclone(r), - } + return &IrClobberList{ + R: regsliceclone(r), + } } func (self *IrClobberList) Clone() IrNode { - r := new(IrClobberList) - r.R = make([]Reg, len(self.R)) - copy(r.R, self.R) - return r + r := new(IrClobberList) + r.R = make([]Reg, len(self.R)) + copy(r.R, self.R) + return r } func (self *IrClobberList) String() string { - return "drop " + regslicerepr(self.R) + return "drop " + regslicerepr(self.R) } func (self *IrClobberList) Usages() []*Reg { - return regsliceref(self.R) + return regsliceref(self.R) } type IrWriteBarrier struct { - R Reg - M Reg - Fn Reg - Var Reg + R Reg + M Reg + Fn Reg + Var Reg } func (self *IrWriteBarrier) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrWriteBarrier) String() string { - return fmt.Sprintf("write_barrier (%s:%s), %s -> *%s", self.Var, self.Fn, self.R, self.M) + return fmt.Sprintf("write_barrier (%s:%s), %s -> *%s", self.Var, self.Fn, self.R, self.M) } func (self *IrWriteBarrier) Usages() []*Reg { - return []*Reg { &self.R, &self.M, &self.Var, &self.Fn } + return []*Reg{&self.R, &self.M, &self.Var, &self.Fn} } type ( IrSpillOp uint8 - IrSpillSlot uint64 + IrSpillSlot uint64 ) const ( - IrSpillStore IrSpillOp = iota - IrSpillReload + IrSpillStore IrSpillOp = iota + IrSpillReload ) func mkspillslot(id int, ptr bool) IrSpillSlot { - return IrSpillSlot((id << 1) | bool2int(ptr)) + return IrSpillSlot((id << 1) | bool2int(ptr)) } func (self IrSpillOp) String() string { - switch self { - case IrSpillStore : return "store" - case IrSpillReload : return "reload" - default : panic("invalid spill op") - } + switch self { + case IrSpillStore: + return "store" + case IrSpillReload: + return "reload" + default: + panic("invalid spill op") + } } func (self IrSpillSlot) ID() int { - return int(self >> 1) + return int(self >> 1) } func (self IrSpillSlot) IsPtr() bool { - return int2bool(int(self & 1)) + return int2bool(int(self & 1)) } func (self IrSpillSlot) String() string { - if self.IsPtr() { - return fmt.Sprintf("{slot %d.p}", self.ID()) - } else { - return fmt.Sprintf("{slot %d.i}", self.ID()) - } + if self.IsPtr() { + return fmt.Sprintf("{slot %d.p}", self.ID()) + } else { + return fmt.Sprintf("{slot %d.i}", self.ID()) + } } type IrSpill struct { - R Reg - S IrSpillSlot - Op IrSpillOp + R Reg + S IrSpillSlot + Op IrSpillOp } func IrCreateSpill(reg Reg, id int, op IrSpillOp) IrNode { - return IrCreateSpillEx(reg, reg.Ptr(), id, op) + return IrCreateSpillEx(reg, reg.Ptr(), id, op) } func IrCreateSpillEx(reg Reg, ptr bool, id int, op IrSpillOp) IrNode { - return &IrSpill { - R : reg, - S : mkspillslot(id, ptr), - Op : op, - } + return &IrSpill{ + R: reg, + S: mkspillslot(id, ptr), + Op: op, + } } func (self *IrSpill) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrSpill) String() string { - switch self.Op { - case IrSpillStore : return fmt.Sprintf("spill %s -> %s", self.R, self.S) - case IrSpillReload : return fmt.Sprintf("%s = reload %s", self.R, self.S) - default : panic("invalid spill op") - } + switch self.Op { + case IrSpillStore: + return fmt.Sprintf("spill %s -> %s", self.R, self.S) + case IrSpillReload: + return fmt.Sprintf("%s = reload %s", self.R, self.S) + default: + panic("invalid spill op") + } } func (self *IrSpill) Usages() []*Reg { - switch self.Op { - case IrSpillStore : return []*Reg { &self.R } - case IrSpillReload : return nil - default : panic("invalid spill op") - } + switch self.Op { + case IrSpillStore: + return []*Reg{&self.R} + case IrSpillReload: + return nil + default: + panic("invalid spill op") + } } func (self *IrSpill) Definitions() []*Reg { - switch self.Op { - case IrSpillStore : return nil - case IrSpillReload : return []*Reg { &self.R } - default : panic("invalid spill op") - } + switch self.Op { + case IrSpillStore: + return nil + case IrSpillReload: + return []*Reg{&self.R} + default: + panic("invalid spill op") + } } type IrSlotAlive struct { - S []IrSpillSlot + S []IrSpillSlot } func IrSlotGen(s IrSpillSlot) IrNode { - return &IrSlotAlive { - S: []IrSpillSlot { s }, - } + return &IrSlotAlive{ + S: []IrSpillSlot{s}, + } } func (self *IrSlotAlive) Clone() IrNode { - r := new(IrSlotAlive) - r.S = make([]IrSpillSlot, len(self.S)) - copy(r.S, self.S) - return r + r := new(IrSlotAlive) + r.S = make([]IrSpillSlot, len(self.S)) + copy(r.S, self.S) + return r } func (self *IrSlotAlive) String() string { - nb := len(self.S) - sv := make([]string, 0, nb) - ss := make([]IrSpillSlot, nb) - - /* sort the slots */ - copy(ss, self.S) - sort.Ints(*(*[]int)(unsafe.Pointer(&ss))) - - /* dump the slots */ - for _, v := range ss { - sv = append(sv, v.String()) - } - - /* join them together */ - return fmt.Sprintf( - "mark_alive %s", - strings.Join(sv, ", "), - ) + nb := len(self.S) + sv := make([]string, 0, nb) + ss := make([]IrSpillSlot, nb) + + /* sort the slots */ + copy(ss, self.S) + sort.Ints(*(*[]int)(unsafe.Pointer(&ss))) + + /* dump the slots */ + for _, v := range ss { + sv = append(sv, v.String()) + } + + /* join them together */ + return fmt.Sprintf( + "mark_alive %s", + strings.Join(sv, ", "), + ) } diff --git a/internal/atm/ssa/ir_amd64.go b/internal/atm/ssa/ir_amd64.go index 717a091..192e76d 100644 --- a/internal/atm/ssa/ir_amd64.go +++ b/internal/atm/ssa/ir_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,143 +17,143 @@ package ssa import ( - `fmt` - `unsafe` + "fmt" + "unsafe" - `github.com/cloudwego/iasm/x86_64` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/iasm/x86_64" ) -var ArchRegs = [...]x86_64.Register64 { - x86_64.RAX, - x86_64.RCX, - x86_64.RDX, - x86_64.RBX, - x86_64.RSP, - x86_64.RBP, - x86_64.RSI, - x86_64.RDI, - x86_64.R8, - x86_64.R9, - x86_64.R10, - x86_64.R11, - x86_64.R12, - x86_64.R13, - x86_64.R14, - x86_64.R15, -} - -var ArchRegIds = map[x86_64.Register64]uint64 { - x86_64.RAX : 0, - x86_64.RCX : 1, - x86_64.RDX : 2, - x86_64.RBX : 3, - x86_64.RSP : 4, - x86_64.RBP : 5, - x86_64.RSI : 6, - x86_64.RDI : 7, - x86_64.R8 : 8, - x86_64.R9 : 9, - x86_64.R10 : 10, - x86_64.R11 : 11, - x86_64.R12 : 12, - x86_64.R13 : 13, - x86_64.R14 : 14, - x86_64.R15 : 15, -} - -var ArchRegNames = map[x86_64.Register64]string { - x86_64.RAX : "rax", - x86_64.RCX : "rcx", - x86_64.RDX : "rdx", - x86_64.RBX : "rbx", - x86_64.RSP : "rsp", - x86_64.RBP : "rbp", - x86_64.RSI : "rsi", - x86_64.RDI : "rdi", - x86_64.R8 : "r8", - x86_64.R9 : "r9", - x86_64.R10 : "r10", - x86_64.R11 : "r11", - x86_64.R12 : "r12", - x86_64.R13 : "r13", - x86_64.R14 : "r14", - x86_64.R15 : "r15", -} - -var ArchRegReserved = map[x86_64.Register64]bool { - x86_64.RSP: true, - x86_64.RBP: true, +var ArchRegs = [...]x86_64.Register64{ + x86_64.RAX, + x86_64.RCX, + x86_64.RDX, + x86_64.RBX, + x86_64.RSP, + x86_64.RBP, + x86_64.RSI, + x86_64.RDI, + x86_64.R8, + x86_64.R9, + x86_64.R10, + x86_64.R11, + x86_64.R12, + x86_64.R13, + x86_64.R14, + x86_64.R15, +} + +var ArchRegIds = map[x86_64.Register64]uint64{ + x86_64.RAX: 0, + x86_64.RCX: 1, + x86_64.RDX: 2, + x86_64.RBX: 3, + x86_64.RSP: 4, + x86_64.RBP: 5, + x86_64.RSI: 6, + x86_64.RDI: 7, + x86_64.R8: 8, + x86_64.R9: 9, + x86_64.R10: 10, + x86_64.R11: 11, + x86_64.R12: 12, + x86_64.R13: 13, + x86_64.R14: 14, + x86_64.R15: 15, +} + +var ArchRegNames = map[x86_64.Register64]string{ + x86_64.RAX: "rax", + x86_64.RCX: "rcx", + x86_64.RDX: "rdx", + x86_64.RBX: "rbx", + x86_64.RSP: "rsp", + x86_64.RBP: "rbp", + x86_64.RSI: "rsi", + x86_64.RDI: "rdi", + x86_64.R8: "r8", + x86_64.R9: "r9", + x86_64.R10: "r10", + x86_64.R11: "r11", + x86_64.R12: "r12", + x86_64.R13: "r13", + x86_64.R14: "r14", + x86_64.R15: "r15", +} + +var ArchRegReserved = map[x86_64.Register64]bool{ + x86_64.RSP: true, + x86_64.RBP: true, } func IrSetArch(rr Reg, reg x86_64.Register64) Reg { - if id, ok := ArchRegIds[reg]; !ok { - panic("invalid physical register: " + reg.String()) - } else if rr.Ptr() { - return mkreg(1, K_arch, id).Derive(rr.Index()) - } else { - return mkreg(0, K_arch, id).Derive(rr.Index()) - } + if id, ok := ArchRegIds[reg]; !ok { + panic("invalid physical register: " + reg.String()) + } else if rr.Ptr() { + return mkreg(1, K_arch, id).Derive(rr.Index()) + } else { + return mkreg(0, K_arch, id).Derive(rr.Index()) + } } type Mem struct { - M Reg - I Reg - S uint8 - D int32 + M Reg + I Reg + S uint8 + D int32 } func Ptr(r Reg, d int32) Mem { - return Mem { - M: r, - I: Rz, - S: 1, - D: d, - } + return Mem{ + M: r, + I: Rz, + S: 1, + D: d, + } } func (self Mem) String() string { - if self.I.Kind() == K_zero { - if self.D == 0 { - return fmt.Sprintf("(%s)", self.M) - } else { - return fmt.Sprintf("%d(%s)", self.D, self.M) - } - } else if self.S == 1 { - if self.D == 0 { - return fmt.Sprintf("(%s,%s)", self.M, self.I) - } else { - return fmt.Sprintf("%d(%s,%s)", self.D, self.M, self.I) - } - } else { - if self.D == 0 { - return fmt.Sprintf("(%s,%s,%d)", self.M, self.I, self.S) - } else { - return fmt.Sprintf("%d(%s,%s,%d)", self.D, self.M, self.I, self.S) - } - } + if self.I.Kind() == K_zero { + if self.D == 0 { + return fmt.Sprintf("(%s)", self.M) + } else { + return fmt.Sprintf("%d(%s)", self.D, self.M) + } + } else if self.S == 1 { + if self.D == 0 { + return fmt.Sprintf("(%s,%s)", self.M, self.I) + } else { + return fmt.Sprintf("%d(%s,%s)", self.D, self.M, self.I) + } + } else { + if self.D == 0 { + return fmt.Sprintf("(%s,%s,%d)", self.M, self.I, self.S) + } else { + return fmt.Sprintf("%d(%s,%s,%d)", self.D, self.M, self.I, self.S) + } + } } type IrAMD64_MemOp interface { - MemOp() *Mem + MemOp() *Mem } -func (*IrAMD64_INT) irnode() {} -func (*IrAMD64_LEA) irnode() {} -func (*IrAMD64_NEG) irnode() {} -func (*IrAMD64_BSWAP) irnode() {} +func (*IrAMD64_INT) irnode() {} +func (*IrAMD64_LEA) irnode() {} +func (*IrAMD64_NEG) irnode() {} +func (*IrAMD64_BSWAP) irnode() {} func (*IrAMD64_MOVSLQ) irnode() {} -func (*IrAMD64_MOV_abs) irnode() {} -func (*IrAMD64_MOV_ptr) irnode() {} -func (*IrAMD64_MOV_reg) irnode() {} -func (*IrAMD64_MOV_load) irnode() {} -func (*IrAMD64_MOV_store_r) irnode() {} -func (*IrAMD64_MOV_store_i) irnode() {} -func (*IrAMD64_MOV_store_p) irnode() {} -func (*IrAMD64_MOV_load_be) irnode() {} -func (*IrAMD64_MOV_store_be) irnode() {} -func (*IrAMD64_MOV_load_stack) irnode() {} +func (*IrAMD64_MOV_abs) irnode() {} +func (*IrAMD64_MOV_ptr) irnode() {} +func (*IrAMD64_MOV_reg) irnode() {} +func (*IrAMD64_MOV_load) irnode() {} +func (*IrAMD64_MOV_store_r) irnode() {} +func (*IrAMD64_MOV_store_i) irnode() {} +func (*IrAMD64_MOV_store_p) irnode() {} +func (*IrAMD64_MOV_load_be) irnode() {} +func (*IrAMD64_MOV_store_be) irnode() {} +func (*IrAMD64_MOV_load_stack) irnode() {} func (*IrAMD64_MOV_store_stack) irnode() {} func (*IrAMD64_BinOp_rr) irnode() {} @@ -175,8 +175,8 @@ func (*IrAMD64_CMPQ_mp) irnode() {} func (*IrAMD64_CMPQ_im) irnode() {} func (*IrAMD64_CMPQ_pm) irnode() {} -func (*IrAMD64_CALL_reg) irnode() {} -func (*IrAMD64_CALL_mem) irnode() {} +func (*IrAMD64_CALL_reg) irnode() {} +func (*IrAMD64_CALL_mem) irnode() {} func (*IrAMD64_CALL_gcwb) irnode() {} func (*IrAMD64_RET) irnode() {} @@ -195,24 +195,24 @@ func (*IrAMD64_Jcc_mp) irnode() {} func (*IrAMD64_Jcc_im) irnode() {} func (*IrAMD64_Jcc_pm) irnode() {} -func (*IrAMD64_MOV_store_r) irimpure() {} -func (*IrAMD64_MOV_store_i) irimpure() {} -func (*IrAMD64_MOV_store_p) irimpure() {} -func (*IrAMD64_MOV_store_be) irimpure() {} -func (*IrAMD64_MOV_load_stack) irimpure() {} +func (*IrAMD64_MOV_store_r) irimpure() {} +func (*IrAMD64_MOV_store_i) irimpure() {} +func (*IrAMD64_MOV_store_p) irimpure() {} +func (*IrAMD64_MOV_store_be) irimpure() {} +func (*IrAMD64_MOV_load_stack) irimpure() {} func (*IrAMD64_MOV_store_stack) irimpure() {} -func (*IrAMD64_CALL_reg) irimpure() {} -func (*IrAMD64_CALL_mem) irimpure() {} +func (*IrAMD64_CALL_reg) irimpure() {} +func (*IrAMD64_CALL_mem) irimpure() {} func (*IrAMD64_CALL_gcwb) irimpure() {} -func (*IrAMD64_MOV_load) irimmovable() {} -func (*IrAMD64_MOV_store_r) irimmovable() {} -func (*IrAMD64_MOV_store_i) irimmovable() {} -func (*IrAMD64_MOV_store_p) irimmovable() {} -func (*IrAMD64_MOV_load_be) irimmovable() {} -func (*IrAMD64_MOV_store_be) irimmovable() {} -func (*IrAMD64_MOV_load_stack) irimmovable() {} +func (*IrAMD64_MOV_load) irimmovable() {} +func (*IrAMD64_MOV_store_r) irimmovable() {} +func (*IrAMD64_MOV_store_i) irimmovable() {} +func (*IrAMD64_MOV_store_p) irimmovable() {} +func (*IrAMD64_MOV_load_be) irimmovable() {} +func (*IrAMD64_MOV_store_be) irimmovable() {} +func (*IrAMD64_MOV_load_stack) irimmovable() {} func (*IrAMD64_MOV_store_stack) irimmovable() {} func (*IrAMD64_CMPQ_rm) irimmovable() {} @@ -222,13 +222,13 @@ func (*IrAMD64_CMPQ_mp) irimmovable() {} func (*IrAMD64_CMPQ_im) irimmovable() {} func (*IrAMD64_CMPQ_pm) irimmovable() {} -func (*IrAMD64_CALL_reg) irimmovable() {} -func (*IrAMD64_CALL_mem) irimmovable() {} +func (*IrAMD64_CALL_reg) irimmovable() {} +func (*IrAMD64_CALL_mem) irimmovable() {} func (*IrAMD64_CALL_gcwb) irimmovable() {} -func (*IrAMD64_RET) irterminator() {} -func (*IrAMD64_JMP) irterminator() {} -func (*IrAMD64_JNC) irterminator() {} +func (*IrAMD64_RET) irterminator() {} +func (*IrAMD64_JMP) irterminator() {} +func (*IrAMD64_JNC) irterminator() {} func (*IrAMD64_Jcc_rr) irterminator() {} func (*IrAMD64_Jcc_ri) irterminator() {} @@ -243,1769 +243,1854 @@ func (*IrAMD64_Jcc_im) irterminator() {} func (*IrAMD64_Jcc_pm) irterminator() {} type IrAMD64_INT struct { - I uint8 + I uint8 } func (self *IrAMD64_INT) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_INT) String() string { - switch self.I { - case 1 : return "int1" - case 3 : return "int3" - default : return fmt.Sprintf("int $%d # %#x", self.I, self.I) - } + switch self.I { + case 1: + return "int1" + case 3: + return "int3" + default: + return fmt.Sprintf("int $%d # %#x", self.I, self.I) + } } type IrAMD64_LEA struct { - R Reg - M Mem + R Reg + M Mem } func (self *IrAMD64_LEA) MemOp() *Mem { - return &self.M + return &self.M } func (self *IrAMD64_LEA) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_LEA) String() string { - return fmt.Sprintf("leaq %s, %s", self.M, self.R) + return fmt.Sprintf("leaq %s, %s", self.M, self.R) } func (self *IrAMD64_LEA) Usages() (r []*Reg) { - if self.M.M.Kind() != K_zero { r = append(r, &self.M.M) } - if self.M.I.Kind() != K_zero { r = append(r, &self.M.I) } - return + if self.M.M.Kind() != K_zero { + r = append(r, &self.M.M) + } + if self.M.I.Kind() != K_zero { + r = append(r, &self.M.I) + } + return } func (self *IrAMD64_LEA) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_NEG struct { - R Reg - V Reg + R Reg + V Reg } func (self *IrAMD64_NEG) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_NEG) String() string { - if self.R == self.V { - return fmt.Sprintf("negq %s", self.R) - } else { - return fmt.Sprintf("movq %s, %s; negq %s", self.V, self.R, self.R) - } + if self.R == self.V { + return fmt.Sprintf("negq %s", self.R) + } else { + return fmt.Sprintf("movq %s, %s; negq %s", self.V, self.R, self.R) + } } func (self *IrAMD64_NEG) Usages() []*Reg { - return []*Reg { &self.V } + return []*Reg{&self.V} } func (self *IrAMD64_NEG) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_BSWAP struct { - R Reg - V Reg - N uint8 + R Reg + V Reg + N uint8 } func (self *IrAMD64_BSWAP) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_BSWAP) String() string { - if self.R == self.V { - switch self.N { - case 2 : return fmt.Sprintf("rolw $8, %s", self.R) - case 4 : return fmt.Sprintf("bswapl %s", self.R) - case 8 : return fmt.Sprintf("bswapq %s", self.R) - default : panic("invalid bswap size") - } - } else { - switch self.N { - case 2 : return fmt.Sprintf("movq %s, %s; rolw $8, %s", self.V, self.R, self.R) - case 4 : return fmt.Sprintf("movq %s, %s; bswapl %s", self.V, self.R, self.R) - case 8 : return fmt.Sprintf("movq %s, %s; bswapq %s", self.V, self.R, self.R) - default : panic("invalid bswap size") - } - } + if self.R == self.V { + switch self.N { + case 2: + return fmt.Sprintf("rolw $8, %s", self.R) + case 4: + return fmt.Sprintf("bswapl %s", self.R) + case 8: + return fmt.Sprintf("bswapq %s", self.R) + default: + panic("invalid bswap size") + } + } else { + switch self.N { + case 2: + return fmt.Sprintf("movq %s, %s; rolw $8, %s", self.V, self.R, self.R) + case 4: + return fmt.Sprintf("movq %s, %s; bswapl %s", self.V, self.R, self.R) + case 8: + return fmt.Sprintf("movq %s, %s; bswapq %s", self.V, self.R, self.R) + default: + panic("invalid bswap size") + } + } } func (self *IrAMD64_BSWAP) Usages() []*Reg { - return []*Reg { &self.V } + return []*Reg{&self.V} } func (self *IrAMD64_BSWAP) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_MOVSLQ struct { - R Reg - V Reg + R Reg + V Reg } func (self *IrAMD64_MOVSLQ) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_MOVSLQ) String() string { - return fmt.Sprintf("movslq %s, %s", self.V, self.R) + return fmt.Sprintf("movslq %s, %s", self.V, self.R) } func (self *IrAMD64_MOVSLQ) Usages() []*Reg { - return []*Reg { &self.V } + return []*Reg{&self.V} } func (self *IrAMD64_MOVSLQ) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_MOV_abs struct { - R Reg - V int64 + R Reg + V int64 } func IrArchConstInt(r Reg, v int64) IrNode { - return &IrAMD64_MOV_abs { - R: r, - V: v, - } + return &IrAMD64_MOV_abs{ + R: r, + V: v, + } } func IrArchTryIntoConstInt(v IrNode) (Reg, int64, bool) { - if p, ok := v.(*IrConstInt); ok { - return p.R, p.V, true - } else if p, ok := v.(*IrAMD64_MOV_abs); ok { - return p.R, p.V, true - } else { - return 0, 0, false - } + if p, ok := v.(*IrConstInt); ok { + return p.R, p.V, true + } else if p, ok := v.(*IrAMD64_MOV_abs); ok { + return p.R, p.V, true + } else { + return 0, 0, false + } } func (self *IrAMD64_MOV_abs) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_MOV_abs) String() string { - return fmt.Sprintf("movabsq $%d, %s # %#x", self.V, self.R, self.V) + return fmt.Sprintf("movabsq $%d, %s # %#x", self.V, self.R, self.V) } func (self *IrAMD64_MOV_abs) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_MOV_ptr struct { - R Reg - P unsafe.Pointer + R Reg + P unsafe.Pointer } func IrArchConstPtr(r Reg, p unsafe.Pointer) IrNode { - return &IrAMD64_MOV_ptr { - R: r, - P: p, - } + return &IrAMD64_MOV_ptr{ + R: r, + P: p, + } } func IrArchTryIntoConstPtr(v IrNode) (Reg, unsafe.Pointer, bool) { - if p, ok := v.(*IrConstPtr); ok { - return p.R, p.P, true - } else if p, ok := v.(*IrAMD64_MOV_ptr); ok { - return p.R, p.P, true - } else { - return 0, nil, false - } + if p, ok := v.(*IrConstPtr); ok { + return p.R, p.P, true + } else if p, ok := v.(*IrAMD64_MOV_ptr); ok { + return p.R, p.P, true + } else { + return 0, nil, false + } } func (self *IrAMD64_MOV_ptr) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_MOV_ptr) String() string { - return fmt.Sprintf("movabsq $%p, %s # %s", self.P, self.R, rt.FuncName(self.P)) + return fmt.Sprintf("movabsq $%p, %s # %s", self.P, self.R, rt.FuncName(self.P)) } func (self *IrAMD64_MOV_ptr) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } func IrArchZero(r Reg) IrNode { - if r.Ptr() { - return &IrAMD64_MOV_ptr { R: r } - } else { - return &IrAMD64_MOV_abs { R: r } - } + if r.Ptr() { + return &IrAMD64_MOV_ptr{R: r} + } else { + return &IrAMD64_MOV_abs{R: r} + } } type IrAMD64_MOV_reg struct { - R Reg - V Reg + R Reg + V Reg } func IrArchCopy(r Reg, v Reg) IrNode { - return &IrAMD64_MOV_reg { R: r, V: v } + return &IrAMD64_MOV_reg{R: r, V: v} } func IrArchTryIntoCopy(v IrNode) (Reg, Reg, bool) { - if p, ok := v.(*IrAMD64_MOV_reg); ok { - return p.R, p.V, true - } else { - return IrTryIntoCopy(v) - } + if p, ok := v.(*IrAMD64_MOV_reg); ok { + return p.R, p.V, true + } else { + return IrTryIntoCopy(v) + } } func (self *IrAMD64_MOV_reg) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_MOV_reg) String() string { - return fmt.Sprintf("movq %s, %s", self.V, self.R) + return fmt.Sprintf("movq %s, %s", self.V, self.R) } func (self *IrAMD64_MOV_reg) Usages() []*Reg { - return []*Reg { &self.V } + return []*Reg{&self.V} } func (self *IrAMD64_MOV_reg) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_MOV_load struct { - R Reg - M Mem - N uint8 + R Reg + M Mem + N uint8 } func (self *IrAMD64_MOV_load) MemOp() *Mem { - return &self.M + return &self.M } func (self *IrAMD64_MOV_load) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_MOV_load) String() string { - switch self.N { - case 1 : return fmt.Sprintf("movzbq %s, %s", self.M, self.R) - case 2 : return fmt.Sprintf("movzwq %s, %s", self.M, self.R) - case 4 : return fmt.Sprintf("movl %s, %s", self.M, self.R) - case 8 : return fmt.Sprintf("movq %s, %s", self.M, self.R) - case 16 : return fmt.Sprintf("movdqu %s, %s", self.M, self.R) - default : panic("invalid load size") - } + switch self.N { + case 1: + return fmt.Sprintf("movzbq %s, %s", self.M, self.R) + case 2: + return fmt.Sprintf("movzwq %s, %s", self.M, self.R) + case 4: + return fmt.Sprintf("movl %s, %s", self.M, self.R) + case 8: + return fmt.Sprintf("movq %s, %s", self.M, self.R) + case 16: + return fmt.Sprintf("movdqu %s, %s", self.M, self.R) + default: + panic("invalid load size") + } } func (self *IrAMD64_MOV_load) Usages() (r []*Reg) { - if self.M.M.Kind() != K_zero { r = append(r, &self.M.M) } - if self.M.I.Kind() != K_zero { r = append(r, &self.M.I) } - return + if self.M.M.Kind() != K_zero { + r = append(r, &self.M.M) + } + if self.M.I.Kind() != K_zero { + r = append(r, &self.M.I) + } + return } func (self *IrAMD64_MOV_load) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_MOV_store_r struct { - R Reg - M Mem - N uint8 + R Reg + M Mem + N uint8 } func (self *IrAMD64_MOV_store_r) MemOp() *Mem { - return &self.M + return &self.M } func (self *IrAMD64_MOV_store_r) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_MOV_store_r) String() string { - return fmt.Sprintf("mov%c %s, %s", memsizec(self.N), self.R, self.M) + return fmt.Sprintf("mov%c %s, %s", memsizec(self.N), self.R, self.M) } func (self *IrAMD64_MOV_store_r) Usages() (r []*Reg) { - r = []*Reg { &self.R } - if self.M.M.Kind() != K_zero { r = append(r, &self.M.M) } - if self.M.I.Kind() != K_zero { r = append(r, &self.M.I) } - return + r = []*Reg{&self.R} + if self.M.M.Kind() != K_zero { + r = append(r, &self.M.M) + } + if self.M.I.Kind() != K_zero { + r = append(r, &self.M.I) + } + return } type IrAMD64_MOV_store_i struct { - V int32 - M Mem - N uint8 + V int32 + M Mem + N uint8 } func (self *IrAMD64_MOV_store_i) MemOp() *Mem { - return &self.M + return &self.M } func (self *IrAMD64_MOV_store_i) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_MOV_store_i) String() string { - return fmt.Sprintf("mov%c $%d, %s # %#0*x", memsizec(self.N), self.V, self.M, self.N * 2, self.V) + return fmt.Sprintf("mov%c $%d, %s # %#0*x", memsizec(self.N), self.V, self.M, self.N*2, self.V) } func (self *IrAMD64_MOV_store_i) Usages() (r []*Reg) { - if self.M.M.Kind() != K_zero { r = append(r, &self.M.M) } - if self.M.I.Kind() != K_zero { r = append(r, &self.M.I) } - return + if self.M.M.Kind() != K_zero { + r = append(r, &self.M.M) + } + if self.M.I.Kind() != K_zero { + r = append(r, &self.M.I) + } + return } type IrAMD64_MOV_store_p struct { - P unsafe.Pointer - M Mem + P unsafe.Pointer + M Mem } func (self *IrAMD64_MOV_store_p) MemOp() *Mem { - return &self.M + return &self.M } func (self *IrAMD64_MOV_store_p) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_MOV_store_p) String() string { - return fmt.Sprintf("movq $%p, %s # %s", self.P, self.M, rt.FuncName(self.P)) + return fmt.Sprintf("movq $%p, %s # %s", self.P, self.M, rt.FuncName(self.P)) } func (self *IrAMD64_MOV_store_p) Usages() (r []*Reg) { - if self.M.M.Kind() != K_zero { r = append(r, &self.M.M) } - if self.M.I.Kind() != K_zero { r = append(r, &self.M.I) } - return + if self.M.M.Kind() != K_zero { + r = append(r, &self.M.M) + } + if self.M.I.Kind() != K_zero { + r = append(r, &self.M.I) + } + return } type IrAMD64_MOV_load_be struct { - R Reg - M Mem - N uint8 + R Reg + M Mem + N uint8 } func (self *IrAMD64_MOV_load_be) MemOp() *Mem { - return &self.M + return &self.M } func (self *IrAMD64_MOV_load_be) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_MOV_load_be) String() string { - switch self.N { - case 2 : return fmt.Sprintf("movbew %s, %s; movzwl %s, %s", self.M, self.R, self.R, self.R) - case 4 : return fmt.Sprintf("movbel %s, %s", self.M, self.R) - case 8 : return fmt.Sprintf("movbeq %s, %s", self.M, self.R) - default : panic("invalid load size") - } + switch self.N { + case 2: + return fmt.Sprintf("movbew %s, %s; movzwl %s, %s", self.M, self.R, self.R, self.R) + case 4: + return fmt.Sprintf("movbel %s, %s", self.M, self.R) + case 8: + return fmt.Sprintf("movbeq %s, %s", self.M, self.R) + default: + panic("invalid load size") + } } func (self *IrAMD64_MOV_load_be) Usages() (r []*Reg) { - if self.M.M.Kind() != K_zero { r = append(r, &self.M.M) } - if self.M.I.Kind() != K_zero { r = append(r, &self.M.I) } - return + if self.M.M.Kind() != K_zero { + r = append(r, &self.M.M) + } + if self.M.I.Kind() != K_zero { + r = append(r, &self.M.I) + } + return } func (self *IrAMD64_MOV_load_be) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_MOV_store_be struct { - R Reg - M Mem - N uint8 + R Reg + M Mem + N uint8 } func (self *IrAMD64_MOV_store_be) MemOp() *Mem { - return &self.M + return &self.M } func (self *IrAMD64_MOV_store_be) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_MOV_store_be) String() string { - switch self.N { - case 2 : return fmt.Sprintf("movbew %s, %s", self.R, self.M) - case 4 : return fmt.Sprintf("movbel %s, %s", self.R, self.M) - case 8 : return fmt.Sprintf("movbeq %s, %s", self.R, self.M) - default : panic("invalid store size") - } + switch self.N { + case 2: + return fmt.Sprintf("movbew %s, %s", self.R, self.M) + case 4: + return fmt.Sprintf("movbel %s, %s", self.R, self.M) + case 8: + return fmt.Sprintf("movbeq %s, %s", self.R, self.M) + default: + panic("invalid store size") + } } func (self *IrAMD64_MOV_store_be) Usages() (r []*Reg) { - r = []*Reg { &self.R } - if self.M.M.Kind() != K_zero { r = append(r, &self.M.M) } - if self.M.I.Kind() != K_zero { r = append(r, &self.M.I) } - return + r = []*Reg{&self.R} + if self.M.M.Kind() != K_zero { + r = append(r, &self.M.M) + } + if self.M.I.Kind() != K_zero { + r = append(r, &self.M.I) + } + return } type ( - IrSlotKind uint8 + IrSlotKind uint8 ) const ( - IrSlotArgs IrSlotKind = iota - IrSlotCall - IrSlotLocal + IrSlotArgs IrSlotKind = iota + IrSlotCall + IrSlotLocal ) func (self IrSlotKind) String() string { - switch self { - case IrSlotArgs : return "args" - case IrSlotCall : return "call" - case IrSlotLocal : return "local" - default : return "???" - } + switch self { + case IrSlotArgs: + return "args" + case IrSlotCall: + return "call" + case IrSlotLocal: + return "local" + default: + return "???" + } } type IrAMD64_MOV_load_stack struct { - R Reg - S uintptr - K IrSlotKind + R Reg + S uintptr + K IrSlotKind } func IrArchLoadStack(reg Reg, offs uintptr, kind IrSlotKind) IrNode { - return &IrAMD64_MOV_load_stack { - R: reg, - S: offs, - K: kind, - } + return &IrAMD64_MOV_load_stack{ + R: reg, + S: offs, + K: kind, + } } func (self *IrAMD64_MOV_load_stack) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_MOV_load_stack) String() string { - return fmt.Sprintf("movq %s+%d<>(FP), %s", self.K, self.S, self.R) + return fmt.Sprintf("movq %s+%d<>(FP), %s", self.K, self.S, self.R) } func (self *IrAMD64_MOV_load_stack) Definitions() (r []*Reg) { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_MOV_store_stack struct { - R Reg - S uintptr - K IrSlotKind + R Reg + S uintptr + K IrSlotKind } func IrArchStoreStack(reg Reg, offs uintptr, kind IrSlotKind) IrNode { - return &IrAMD64_MOV_store_stack { - R: reg, - S: offs, - K: kind, - } + return &IrAMD64_MOV_store_stack{ + R: reg, + S: offs, + K: kind, + } } func (self *IrAMD64_MOV_store_stack) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_MOV_store_stack) String() string { - return fmt.Sprintf("movq %s, %s+%d<>(FP)", self.R, self.K, self.S) + return fmt.Sprintf("movq %s, %s+%d<>(FP)", self.R, self.K, self.S) } func (self *IrAMD64_MOV_store_stack) Usages() (r []*Reg) { - return []*Reg { &self.R } + return []*Reg{&self.R} } type ( - IrAMD64_BinOp uint8 - IrAMD64_CmpOp uint8 + IrAMD64_BinOp uint8 + IrAMD64_CmpOp uint8 ) const ( - IrAMD64_BinAdd IrAMD64_BinOp = iota - IrAMD64_BinSub - IrAMD64_BinMul - IrAMD64_BinAnd - IrAMD64_BinOr - IrAMD64_BinXor - IrAMD64_BinShr + IrAMD64_BinAdd IrAMD64_BinOp = iota + IrAMD64_BinSub + IrAMD64_BinMul + IrAMD64_BinAnd + IrAMD64_BinOr + IrAMD64_BinXor + IrAMD64_BinShr ) const ( - IrAMD64_CmpEq IrAMD64_CmpOp = iota - IrAMD64_CmpNe - IrAMD64_CmpLt - IrAMD64_CmpGe - IrAMD64_CmpLtu - IrAMD64_CmpGeu + IrAMD64_CmpEq IrAMD64_CmpOp = iota + IrAMD64_CmpNe + IrAMD64_CmpLt + IrAMD64_CmpGe + IrAMD64_CmpLtu + IrAMD64_CmpGeu ) func (self IrAMD64_BinOp) String() string { - switch self { - case IrAMD64_BinAdd : return "addq" - case IrAMD64_BinSub : return "subq" - case IrAMD64_BinMul : return "imulq" - case IrAMD64_BinAnd : return "andq" - case IrAMD64_BinOr : return "orq" - case IrAMD64_BinXor : return "xorq" - case IrAMD64_BinShr : return "shrq" - default : panic("unreachable") - } + switch self { + case IrAMD64_BinAdd: + return "addq" + case IrAMD64_BinSub: + return "subq" + case IrAMD64_BinMul: + return "imulq" + case IrAMD64_BinAnd: + return "andq" + case IrAMD64_BinOr: + return "orq" + case IrAMD64_BinXor: + return "xorq" + case IrAMD64_BinShr: + return "shrq" + default: + panic("unreachable") + } } func (self IrAMD64_BinOp) IsAdditive() bool { - switch self { - case IrAMD64_BinAdd : fallthrough - case IrAMD64_BinSub : return true - default : return false - } + switch self { + case IrAMD64_BinAdd: + fallthrough + case IrAMD64_BinSub: + return true + default: + return false + } } func (self IrAMD64_BinOp) ScaleFactor() int32 { - switch self { - case IrAMD64_BinAdd : return 1 - case IrAMD64_BinSub : return -1 - default : panic("not an additive operator: " + self.String()) - } + switch self { + case IrAMD64_BinAdd: + return 1 + case IrAMD64_BinSub: + return -1 + default: + panic("not an additive operator: " + self.String()) + } } func (self IrAMD64_CmpOp) String() string { - switch self { - case IrAMD64_CmpEq : return "e" - case IrAMD64_CmpNe : return "ne" - case IrAMD64_CmpLt : return "l" - case IrAMD64_CmpGe : return "ge" - case IrAMD64_CmpLtu : return "b" - case IrAMD64_CmpGeu : return "ae" - default : panic("unreachable") - } + switch self { + case IrAMD64_CmpEq: + return "e" + case IrAMD64_CmpNe: + return "ne" + case IrAMD64_CmpLt: + return "l" + case IrAMD64_CmpGe: + return "ge" + case IrAMD64_CmpLtu: + return "b" + case IrAMD64_CmpGeu: + return "ae" + default: + panic("unreachable") + } } func (self IrAMD64_CmpOp) Negated() IrAMD64_CmpOp { - switch self { - case IrAMD64_CmpEq : return IrAMD64_CmpNe - case IrAMD64_CmpNe : return IrAMD64_CmpEq - case IrAMD64_CmpLt : return IrAMD64_CmpGe - case IrAMD64_CmpGe : return IrAMD64_CmpLt - case IrAMD64_CmpLtu : return IrAMD64_CmpGeu - case IrAMD64_CmpGeu : return IrAMD64_CmpLtu - default : panic("unreachable") - } + switch self { + case IrAMD64_CmpEq: + return IrAMD64_CmpNe + case IrAMD64_CmpNe: + return IrAMD64_CmpEq + case IrAMD64_CmpLt: + return IrAMD64_CmpGe + case IrAMD64_CmpGe: + return IrAMD64_CmpLt + case IrAMD64_CmpLtu: + return IrAMD64_CmpGeu + case IrAMD64_CmpGeu: + return IrAMD64_CmpLtu + default: + panic("unreachable") + } } type IrAMD64_BinOp_rr struct { - R Reg - X Reg - Y Reg - Op IrAMD64_BinOp + R Reg + X Reg + Y Reg + Op IrAMD64_BinOp } func (self *IrAMD64_BinOp_rr) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_BinOp_rr) String() string { - if self.R == self.X { - return fmt.Sprintf("%s %s, %s", self.Op, self.Y, self.X) - } else { - return fmt.Sprintf("movq %s, %s; %s %s, %s", self.X, self.R, self.Op, self.Y, self.R) - } + if self.R == self.X { + return fmt.Sprintf("%s %s, %s", self.Op, self.Y, self.X) + } else { + return fmt.Sprintf("movq %s, %s; %s %s, %s", self.X, self.R, self.Op, self.Y, self.R) + } } func (self *IrAMD64_BinOp_rr) Usages() []*Reg { - return []*Reg { &self.X, &self.Y } + return []*Reg{&self.X, &self.Y} } func (self *IrAMD64_BinOp_rr) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_BinOp_ri struct { - R Reg - X Reg - Y int32 - Op IrAMD64_BinOp + R Reg + X Reg + Y int32 + Op IrAMD64_BinOp } func (self *IrAMD64_BinOp_ri) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_BinOp_ri) String() string { - if self.Op == IrAMD64_BinMul { - return fmt.Sprintf("imulq $%d, %s, %s # %#x", self.Y, self.X, self.R, self.Y) - } else if self.R == self.X { - return fmt.Sprintf("%s $%d, %s # %#x", self.Op, self.Y, self.X, self.Y) - } else { - return fmt.Sprintf("movq %s, %s; %s $%d, %s # %#x", self.X, self.R, self.Op, self.Y, self.R, self.Y) - } + if self.Op == IrAMD64_BinMul { + return fmt.Sprintf("imulq $%d, %s, %s # %#x", self.Y, self.X, self.R, self.Y) + } else if self.R == self.X { + return fmt.Sprintf("%s $%d, %s # %#x", self.Op, self.Y, self.X, self.Y) + } else { + return fmt.Sprintf("movq %s, %s; %s $%d, %s # %#x", self.X, self.R, self.Op, self.Y, self.R, self.Y) + } } func (self *IrAMD64_BinOp_ri) Usages() []*Reg { - return []*Reg { &self.X } + return []*Reg{&self.X} } func (self *IrAMD64_BinOp_ri) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_BinOp_rm struct { - R Reg - X Reg - Y Mem - Op IrAMD64_BinOp + R Reg + X Reg + Y Mem + Op IrAMD64_BinOp } func (self *IrAMD64_BinOp_rm) MemOp() *Mem { - return &self.Y + return &self.Y } func (self *IrAMD64_BinOp_rm) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_BinOp_rm) String() string { - if self.R == self.X { - return fmt.Sprintf("%s %s, %s", self.Op, self.Y, self.X) - } else { - return fmt.Sprintf("movq %s, %s; %s %s, %s", self.X, self.R, self.Op, self.Y, self.R) - } + if self.R == self.X { + return fmt.Sprintf("%s %s, %s", self.Op, self.Y, self.X) + } else { + return fmt.Sprintf("movq %s, %s; %s %s, %s", self.X, self.R, self.Op, self.Y, self.R) + } } func (self *IrAMD64_BinOp_rm) Usages() []*Reg { - if self.Y.I == Rz { - return []*Reg { &self.X, &self.Y.M } - } else { - return []*Reg { &self.X, &self.Y.M, &self.Y.I } - } + if self.Y.I == Rz { + return []*Reg{&self.X, &self.Y.M} + } else { + return []*Reg{&self.X, &self.Y.M, &self.Y.I} + } } func (self *IrAMD64_BinOp_rm) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_BTSQ_rr struct { - T Reg - S Reg - X Reg - Y Reg + T Reg + S Reg + X Reg + Y Reg } func (self *IrAMD64_BTSQ_rr) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_BTSQ_rr) String() string { - if self.T.Kind() == K_zero { - if self.S == self.X { - return fmt.Sprintf("btsq %s, %s", self.Y, self.X) - } else { - return fmt.Sprintf("movq %s, %s; btsq %s, %s", self.X, self.S, self.Y, self.S) - } - } else { - if self.S == self.X { - return fmt.Sprintf("btsq %s, %s; setc %s", self.Y, self.X, self.T) - } else { - return fmt.Sprintf("movq %s, %s; btsq %s, %s; setc %s", self.X, self.S, self.Y, self.S, self.T) - } - } + if self.T.Kind() == K_zero { + if self.S == self.X { + return fmt.Sprintf("btsq %s, %s", self.Y, self.X) + } else { + return fmt.Sprintf("movq %s, %s; btsq %s, %s", self.X, self.S, self.Y, self.S) + } + } else { + if self.S == self.X { + return fmt.Sprintf("btsq %s, %s; setc %s", self.Y, self.X, self.T) + } else { + return fmt.Sprintf("movq %s, %s; btsq %s, %s; setc %s", self.X, self.S, self.Y, self.S, self.T) + } + } } func (self *IrAMD64_BTSQ_rr) Usages() []*Reg { - return []*Reg { &self.X, &self.Y } + return []*Reg{&self.X, &self.Y} } func (self *IrAMD64_BTSQ_rr) Definitions() []*Reg { - return []*Reg { &self.T, &self.S } + return []*Reg{&self.T, &self.S} } type IrAMD64_BTSQ_ri struct { - T Reg - S Reg - X Reg - Y uint8 + T Reg + S Reg + X Reg + Y uint8 } func (self *IrAMD64_BTSQ_ri) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_BTSQ_ri) String() string { - if self.T.Kind() == K_zero { - if self.S == self.X { - return fmt.Sprintf("btsq $%d, %s", self.Y, self.X) - } else { - return fmt.Sprintf("movq %s, %s; btsq $%d, %s", self.X, self.S, self.Y, self.S) - } - } else { - if self.S == self.X { - return fmt.Sprintf("btsq $%d, %s; setc %s", self.Y, self.X, self.T) - } else { - return fmt.Sprintf("movq %s, %s; btsq $%d, %s; setc %s", self.X, self.S, self.Y, self.S, self.T) - } - } + if self.T.Kind() == K_zero { + if self.S == self.X { + return fmt.Sprintf("btsq $%d, %s", self.Y, self.X) + } else { + return fmt.Sprintf("movq %s, %s; btsq $%d, %s", self.X, self.S, self.Y, self.S) + } + } else { + if self.S == self.X { + return fmt.Sprintf("btsq $%d, %s; setc %s", self.Y, self.X, self.T) + } else { + return fmt.Sprintf("movq %s, %s; btsq $%d, %s; setc %s", self.X, self.S, self.Y, self.S, self.T) + } + } } func (self *IrAMD64_BTSQ_ri) Usages() []*Reg { - return []*Reg { &self.X } + return []*Reg{&self.X} } func (self *IrAMD64_BTSQ_ri) Definitions() []*Reg { - return []*Reg { &self.T, &self.S } + return []*Reg{&self.T, &self.S} } type IrAMD64_CMPQ_rr struct { - R Reg - X Reg - Y Reg - Op IrAMD64_CmpOp + R Reg + X Reg + Y Reg + Op IrAMD64_CmpOp } func (self *IrAMD64_CMPQ_rr) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_CMPQ_rr) String() string { - if self.R == Rz { - return fmt.Sprintf("cmpq %s, %s", self.X, self.Y) - } else { - return fmt.Sprintf("cmpq %s, %s; set%s %s", self.X, self.Y, self.Op, self.R) - } + if self.R == Rz { + return fmt.Sprintf("cmpq %s, %s", self.X, self.Y) + } else { + return fmt.Sprintf("cmpq %s, %s; set%s %s", self.X, self.Y, self.Op, self.R) + } } func (self *IrAMD64_CMPQ_rr) Usages() []*Reg { - return []*Reg { &self.X, &self.Y } + return []*Reg{&self.X, &self.Y} } func (self *IrAMD64_CMPQ_rr) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_CMPQ_ri struct { - R Reg - X Reg - Y int32 - Op IrAMD64_CmpOp + R Reg + X Reg + Y int32 + Op IrAMD64_CmpOp } func (self *IrAMD64_CMPQ_ri) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_CMPQ_ri) String() string { - return fmt.Sprintf("cmpq %s, $%d; set%s %s # %#x", self.X, self.Y, self.Op, self.R, self.Y) + return fmt.Sprintf("cmpq %s, $%d; set%s %s # %#x", self.X, self.Y, self.Op, self.R, self.Y) } func (self *IrAMD64_CMPQ_ri) Usages() []*Reg { - return []*Reg { &self.X } + return []*Reg{&self.X} } func (self *IrAMD64_CMPQ_ri) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_CMPQ_rp struct { - R Reg - X Reg - Y unsafe.Pointer - Op IrAMD64_CmpOp + R Reg + X Reg + Y unsafe.Pointer + Op IrAMD64_CmpOp } func (self *IrAMD64_CMPQ_rp) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_CMPQ_rp) String() string { - return fmt.Sprintf("cmpq %s, $%p; set%s %s", self.X, self.Y, self.Op, self.R) + return fmt.Sprintf("cmpq %s, $%p; set%s %s", self.X, self.Y, self.Op, self.R) } func (self *IrAMD64_CMPQ_rp) Usages() []*Reg { - return []*Reg { &self.X } + return []*Reg{&self.X} } func (self *IrAMD64_CMPQ_rp) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_CMPQ_ir struct { - R Reg - X int32 - Y Reg - Op IrAMD64_CmpOp + R Reg + X int32 + Y Reg + Op IrAMD64_CmpOp } func (self *IrAMD64_CMPQ_ir) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_CMPQ_ir) String() string { - return fmt.Sprintf("cmpq $%d, %s; set%s %s # %#x", self.X, self.Y, self.Op, self.R, self.X) + return fmt.Sprintf("cmpq $%d, %s; set%s %s # %#x", self.X, self.Y, self.Op, self.R, self.X) } func (self *IrAMD64_CMPQ_ir) Usages() []*Reg { - return []*Reg { &self.Y } + return []*Reg{&self.Y} } func (self *IrAMD64_CMPQ_ir) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_CMPQ_pr struct { - R Reg - X unsafe.Pointer - Y Reg - Op IrAMD64_CmpOp + R Reg + X unsafe.Pointer + Y Reg + Op IrAMD64_CmpOp } func (self *IrAMD64_CMPQ_pr) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_CMPQ_pr) String() string { - return fmt.Sprintf("cmpq $%p, %s; set%s %s", self.X, self.Y, self.Op, self.R) + return fmt.Sprintf("cmpq $%p, %s; set%s %s", self.X, self.Y, self.Op, self.R) } func (self *IrAMD64_CMPQ_pr) Usages() []*Reg { - return []*Reg { &self.Y } + return []*Reg{&self.Y} } func (self *IrAMD64_CMPQ_pr) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_CMPQ_rm struct { - R Reg - X Reg - Y Mem - N uint8 - Op IrAMD64_CmpOp + R Reg + X Reg + Y Mem + N uint8 + Op IrAMD64_CmpOp } func (self *IrAMD64_CMPQ_rm) MemOp() *Mem { - return &self.Y + return &self.Y } func (self *IrAMD64_CMPQ_rm) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_CMPQ_rm) String() string { - return fmt.Sprintf( - "cmp%c %s, %s; set%s %s", - memsizec(self.N), - self.X, - self.Y, - self.Op, - self.R, - ) + return fmt.Sprintf( + "cmp%c %s, %s; set%s %s", + memsizec(self.N), + self.X, + self.Y, + self.Op, + self.R, + ) } func (self *IrAMD64_CMPQ_rm) Usages() []*Reg { - if self.Y.I == Rz { - return []*Reg { &self.X, &self.Y.M } - } else { - return []*Reg { &self.X, &self.Y.M, &self.Y.I } - } + if self.Y.I == Rz { + return []*Reg{&self.X, &self.Y.M} + } else { + return []*Reg{&self.X, &self.Y.M, &self.Y.I} + } } func (self *IrAMD64_CMPQ_rm) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_CMPQ_mr struct { - R Reg - X Mem - Y Reg - N uint8 - Op IrAMD64_CmpOp + R Reg + X Mem + Y Reg + N uint8 + Op IrAMD64_CmpOp } func (self *IrAMD64_CMPQ_mr) MemOp() *Mem { - return &self.X + return &self.X } func (self *IrAMD64_CMPQ_mr) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_CMPQ_mr) String() string { - return fmt.Sprintf( - "cmp%c %s, %s; set%s %s", - memsizec(self.N), - self.X, - self.Y, - self.Op, - self.R, - ) + return fmt.Sprintf( + "cmp%c %s, %s; set%s %s", + memsizec(self.N), + self.X, + self.Y, + self.Op, + self.R, + ) } func (self *IrAMD64_CMPQ_mr) Usages() []*Reg { - if self.X.I == Rz { - return []*Reg { &self.X.M, &self.Y } - } else { - return []*Reg { &self.X.M, &self.X.I, &self.Y } - } + if self.X.I == Rz { + return []*Reg{&self.X.M, &self.Y} + } else { + return []*Reg{&self.X.M, &self.X.I, &self.Y} + } } func (self *IrAMD64_CMPQ_mr) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_CMPQ_mi struct { - R Reg - X Mem - Y int32 - N uint8 - Op IrAMD64_CmpOp + R Reg + X Mem + Y int32 + N uint8 + Op IrAMD64_CmpOp } func (self *IrAMD64_CMPQ_mi) MemOp() *Mem { - return &self.X + return &self.X } func (self *IrAMD64_CMPQ_mi) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_CMPQ_mi) String() string { - return fmt.Sprintf( - "cmp%c %s, $%d; set%s %s # %#x", - memsizec(self.N), - self.X, - self.Y, - self.Op, - self.R, - self.Y, - ) + return fmt.Sprintf( + "cmp%c %s, $%d; set%s %s # %#x", + memsizec(self.N), + self.X, + self.Y, + self.Op, + self.R, + self.Y, + ) } func (self *IrAMD64_CMPQ_mi) Usages() []*Reg { - if self.X.I == Rz { - return []*Reg { &self.X.M } - } else { - return []*Reg { &self.X.M, &self.X.I } - } + if self.X.I == Rz { + return []*Reg{&self.X.M} + } else { + return []*Reg{&self.X.M, &self.X.I} + } } func (self *IrAMD64_CMPQ_mi) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_CMPQ_mp struct { - R Reg - X Mem - Y unsafe.Pointer - Op IrAMD64_CmpOp + R Reg + X Mem + Y unsafe.Pointer + Op IrAMD64_CmpOp } func (self *IrAMD64_CMPQ_mp) MemOp() *Mem { - return &self.X + return &self.X } func (self *IrAMD64_CMPQ_mp) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_CMPQ_mp) String() string { - return fmt.Sprintf("cmpq %s, $%d; set%s %s # %#x", self.X, self.Y, self.Op, self.R, self.Y) + return fmt.Sprintf("cmpq %s, $%d; set%s %s # %#x", self.X, self.Y, self.Op, self.R, self.Y) } func (self *IrAMD64_CMPQ_mp) Usages() []*Reg { - if self.X.I == Rz { - return []*Reg { &self.X.M } - } else { - return []*Reg { &self.X.M, &self.X.I } - } + if self.X.I == Rz { + return []*Reg{&self.X.M} + } else { + return []*Reg{&self.X.M, &self.X.I} + } } func (self *IrAMD64_CMPQ_mp) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_CMPQ_im struct { - R Reg - X int32 - Y Mem - N uint8 - Op IrAMD64_CmpOp + R Reg + X int32 + Y Mem + N uint8 + Op IrAMD64_CmpOp } func (self *IrAMD64_CMPQ_im) MemOp() *Mem { - return &self.Y + return &self.Y } func (self *IrAMD64_CMPQ_im) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_CMPQ_im) String() string { - return fmt.Sprintf( - "cmp%c $%d, %s; set%s %s # %#x", - memsizec(self.N), - self.X, - self.Y, - self.Op, - self.R, - self.X, - ) + return fmt.Sprintf( + "cmp%c $%d, %s; set%s %s # %#x", + memsizec(self.N), + self.X, + self.Y, + self.Op, + self.R, + self.X, + ) } func (self *IrAMD64_CMPQ_im) Usages() []*Reg { - if self.Y.I == Rz { - return []*Reg { &self.Y.M } - } else { - return []*Reg { &self.Y.M, &self.Y.I } - } + if self.Y.I == Rz { + return []*Reg{&self.Y.M} + } else { + return []*Reg{&self.Y.M, &self.Y.I} + } } func (self *IrAMD64_CMPQ_im) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_CMPQ_pm struct { - R Reg - X unsafe.Pointer - Y Mem - Op IrAMD64_CmpOp + R Reg + X unsafe.Pointer + Y Mem + Op IrAMD64_CmpOp } func (self *IrAMD64_CMPQ_pm) MemOp() *Mem { - return &self.Y + return &self.Y } func (self *IrAMD64_CMPQ_pm) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_CMPQ_pm) String() string { - return fmt.Sprintf("cmpq $%p, %s; set%s %s", self.X, self.Y, self.Op, self.R) + return fmt.Sprintf("cmpq $%p, %s; set%s %s", self.X, self.Y, self.Op, self.R) } func (self *IrAMD64_CMPQ_pm) Usages() []*Reg { - if self.Y.I == Rz { - return []*Reg { &self.Y.M } - } else { - return []*Reg { &self.Y.M, &self.Y.I } - } + if self.Y.I == Rz { + return []*Reg{&self.Y.M} + } else { + return []*Reg{&self.Y.M, &self.Y.I} + } } func (self *IrAMD64_CMPQ_pm) Definitions() []*Reg { - return []*Reg { &self.R } + return []*Reg{&self.R} } type IrAMD64_CALL_reg struct { - Fn Reg - In []Reg - Out []Reg - Clob []Reg + Fn Reg + In []Reg + Out []Reg + Clob []Reg } func (self *IrAMD64_CALL_reg) Clone() IrNode { - r := new(IrAMD64_CALL_reg) - r.Fn = self.Fn - r.In = make([]Reg, len(self.In)) - r.Out = make([]Reg, len(self.Out)) - r.Clob = make([]Reg, len(self.Clob)) - copy(r.In, self.In) - copy(r.Out, self.Out) - copy(r.Clob, self.Clob) - return r + r := new(IrAMD64_CALL_reg) + r.Fn = self.Fn + r.In = make([]Reg, len(self.In)) + r.Out = make([]Reg, len(self.Out)) + r.Clob = make([]Reg, len(self.Clob)) + copy(r.In, self.In) + copy(r.Out, self.Out) + copy(r.Clob, self.Clob) + return r } func (self *IrAMD64_CALL_reg) String() string { - return fmt.Sprintf( - "rcall *%s, {%s}, {%s}", - self.Fn, - regslicerepr(self.In), - regslicerepr(self.Out), - ) + return fmt.Sprintf( + "rcall *%s, {%s}, {%s}", + self.Fn, + regslicerepr(self.In), + regslicerepr(self.Out), + ) } func (self *IrAMD64_CALL_reg) Usages() []*Reg { - return append(regsliceref(self.In), &self.Fn) + return append(regsliceref(self.In), &self.Fn) } func (self *IrAMD64_CALL_reg) Definitions() []*Reg { - return append(regsliceref(self.Out), regsliceref(self.Clob)...) + return append(regsliceref(self.Out), regsliceref(self.Clob)...) } type IrAMD64_CALL_mem struct { - Fn Mem - In []Reg - Out []Reg - Clob []Reg + Fn Mem + In []Reg + Out []Reg + Clob []Reg } func (self *IrAMD64_CALL_mem) MemOp() *Mem { - return &self.Fn + return &self.Fn } func (self *IrAMD64_CALL_mem) Clone() IrNode { - r := new(IrAMD64_CALL_mem) - r.Fn = self.Fn - r.In = make([]Reg, len(self.In)) - r.Out = make([]Reg, len(self.Out)) - r.Clob = make([]Reg, len(self.Clob)) - copy(r.In, self.In) - copy(r.Out, self.Out) - copy(r.Clob, self.Clob) - return r + r := new(IrAMD64_CALL_mem) + r.Fn = self.Fn + r.In = make([]Reg, len(self.In)) + r.Out = make([]Reg, len(self.Out)) + r.Clob = make([]Reg, len(self.Clob)) + copy(r.In, self.In) + copy(r.Out, self.Out) + copy(r.Clob, self.Clob) + return r } func (self *IrAMD64_CALL_mem) String() string { - return fmt.Sprintf( - "mcall *%s, {%s}, {%s}", - self.Fn, - regslicerepr(self.In), - regslicerepr(self.Out), - ) + return fmt.Sprintf( + "mcall *%s, {%s}, {%s}", + self.Fn, + regslicerepr(self.In), + regslicerepr(self.Out), + ) } func (self *IrAMD64_CALL_mem) Usages() []*Reg { - if self.Fn.I == Rz { - return append(regsliceref(self.In), &self.Fn.M) - } else { - return append(regsliceref(self.In), &self.Fn.M, &self.Fn.I) - } + if self.Fn.I == Rz { + return append(regsliceref(self.In), &self.Fn.M) + } else { + return append(regsliceref(self.In), &self.Fn.M, &self.Fn.I) + } } func (self *IrAMD64_CALL_mem) Definitions() []*Reg { - return append(regsliceref(self.Out), regsliceref(self.Clob)...) + return append(regsliceref(self.Out), regsliceref(self.Clob)...) } type IrAMD64_CALL_gcwb struct { - R Reg - M Reg - Fn unsafe.Pointer + R Reg + M Reg + Fn unsafe.Pointer } func (self *IrAMD64_CALL_gcwb) Clone() IrNode { - r := *self - return &r + r := *self + return &r } func (self *IrAMD64_CALL_gcwb) String() string { - return fmt.Sprintf("scall *%p [%s], %s -> (%s)", self.Fn, rt.FuncName(self.Fn), self.R, self.M) + return fmt.Sprintf("scall *%p [%s], %s -> (%s)", self.Fn, rt.FuncName(self.Fn), self.R, self.M) } func (self *IrAMD64_CALL_gcwb) Usages() []*Reg { - return []*Reg { &self.R, &self.M } + return []*Reg{&self.R, &self.M} } type IrAMD64_RET struct { - R []Reg + R []Reg } func IrArchReturn(rr []Reg) IrTerminator { - return &IrAMD64_RET { rr } + return &IrAMD64_RET{rr} } func IrTryIntoArchReturn(p IrNode) ([]Reg, bool) { - if r, ok := p.(*IrAMD64_RET); ok { - return r.R, true - } else { - return nil, false - } + if r, ok := p.(*IrAMD64_RET); ok { + return r.R, true + } else { + return nil, false + } } func (self *IrAMD64_RET) Clone() IrNode { - r := new(IrAMD64_RET) - r.R = make([]Reg, len(self.R)) - copy(r.R, self.R) - return r + r := new(IrAMD64_RET) + r.R = make([]Reg, len(self.R)) + copy(r.R, self.R) + return r } func (self *IrAMD64_RET) String() string { - return "retq" + return "retq" } func (self *IrAMD64_RET) Usages() []*Reg { - return regsliceref(self.R) + return regsliceref(self.R) } func (self *IrAMD64_RET) Successors() IrSuccessors { - return _EmptySuccessor{} + return _EmptySuccessor{} } type IrAMD64_JMP struct { - To *IrBranch + To *IrBranch } func IrArchJump(to *BasicBlock) IrTerminator { - return &IrAMD64_JMP { IrLikely(to) } + return &IrAMD64_JMP{IrLikely(to)} } func (self *IrAMD64_JMP) Clone() IrNode { - return &IrAMD64_JMP { self.To.Clone() } + return &IrAMD64_JMP{self.To.Clone()} } func (self *IrAMD64_JMP) String() string { - return "jmp " + self.To.String() + return "jmp " + self.To.String() } func (self *IrAMD64_JMP) Successors() IrSuccessors { - return &_SwitchSuccessors { - i: -1, - t: []_SwitchTarget {{ b: self.To }}, - } + return &_SwitchSuccessors{ + i: -1, + t: []_SwitchTarget{{b: self.To}}, + } } type IrAMD64_JNC struct { - To *IrBranch - Ln *IrBranch + To *IrBranch + Ln *IrBranch } func (self *IrAMD64_JNC) Clone() IrNode { - return &IrAMD64_JNC { - To: self.To.Clone(), - Ln: self.Ln.Clone(), - } + return &IrAMD64_JNC{ + To: self.To.Clone(), + Ln: self.Ln.Clone(), + } } func (self *IrAMD64_JNC) String() string { - return fmt.Sprintf("jnc %s; jmp %s", self.To, self.Ln) + return fmt.Sprintf("jnc %s; jmp %s", self.To, self.Ln) } func (self *IrAMD64_JNC) Successors() IrSuccessors { - return &_SwitchSuccessors { - i: -1, - t: []_SwitchTarget { - { b: self.Ln, i: 0 }, - { b: self.To }, - }, - } + return &_SwitchSuccessors{ + i: -1, + t: []_SwitchTarget{ + {b: self.Ln, i: 0}, + {b: self.To}, + }, + } } type IrAMD64_Jcc_rr struct { - X Reg - Y Reg - To *IrBranch - Ln *IrBranch - Op IrAMD64_CmpOp + X Reg + Y Reg + To *IrBranch + Ln *IrBranch + Op IrAMD64_CmpOp } func (self *IrAMD64_Jcc_rr) Clone() IrNode { - r := *self - r.To = self.To.Clone() - r.Ln = self.Ln.Clone() - return &r + r := *self + r.To = self.To.Clone() + r.Ln = self.Ln.Clone() + return &r } func (self *IrAMD64_Jcc_rr) String() string { - return fmt.Sprintf( - "cmpq %s, %s; j%s %s; jmp %s", - self.X, - self.Y, - self.Op, - self.To, - self.Ln, - ) + return fmt.Sprintf( + "cmpq %s, %s; j%s %s; jmp %s", + self.X, + self.Y, + self.Op, + self.To, + self.Ln, + ) } func (self *IrAMD64_Jcc_rr) Usages() []*Reg { - return []*Reg { &self.X, &self.Y } + return []*Reg{&self.X, &self.Y} } func (self *IrAMD64_Jcc_rr) Successors() IrSuccessors { - return &_SwitchSuccessors { - i: -1, - t: []_SwitchTarget { - { b: self.Ln, i: 0 }, - { b: self.To }, - }, - } + return &_SwitchSuccessors{ + i: -1, + t: []_SwitchTarget{ + {b: self.Ln, i: 0}, + {b: self.To}, + }, + } } type IrAMD64_Jcc_ri struct { - X Reg - Y int32 - To *IrBranch - Ln *IrBranch - Op IrAMD64_CmpOp + X Reg + Y int32 + To *IrBranch + Ln *IrBranch + Op IrAMD64_CmpOp } func (self *IrAMD64_Jcc_ri) Clone() IrNode { - r := *self - r.To = self.To.Clone() - r.Ln = self.Ln.Clone() - return &r + r := *self + r.To = self.To.Clone() + r.Ln = self.Ln.Clone() + return &r } func (self *IrAMD64_Jcc_ri) String() string { - return fmt.Sprintf( - "cmpq %s, $%d; j%s %s; jmp %s # %#x", - self.X, - self.Y, - self.Op, - self.To, - self.Ln, - self.Y, - ) + return fmt.Sprintf( + "cmpq %s, $%d; j%s %s; jmp %s # %#x", + self.X, + self.Y, + self.Op, + self.To, + self.Ln, + self.Y, + ) } func (self *IrAMD64_Jcc_ri) Usages() []*Reg { - return []*Reg { &self.X } + return []*Reg{&self.X} } func (self *IrAMD64_Jcc_ri) Successors() IrSuccessors { - return &_SwitchSuccessors { - i: -1, - t: []_SwitchTarget { - { b: self.Ln, i: 0 }, - { b: self.To }, - }, - } + return &_SwitchSuccessors{ + i: -1, + t: []_SwitchTarget{ + {b: self.Ln, i: 0}, + {b: self.To}, + }, + } } type IrAMD64_Jcc_rp struct { - X Reg - Y unsafe.Pointer - To *IrBranch - Ln *IrBranch - Op IrAMD64_CmpOp + X Reg + Y unsafe.Pointer + To *IrBranch + Ln *IrBranch + Op IrAMD64_CmpOp } func (self *IrAMD64_Jcc_rp) Clone() IrNode { - r := *self - r.To = self.To.Clone() - r.Ln = self.Ln.Clone() - return &r + r := *self + r.To = self.To.Clone() + r.Ln = self.Ln.Clone() + return &r } func (self *IrAMD64_Jcc_rp) String() string { - return fmt.Sprintf( - "cmpq %s, $%p; j%s %s; jmp %s", - self.X, - self.Y, - self.Op, - self.To, - self.Ln, - ) + return fmt.Sprintf( + "cmpq %s, $%p; j%s %s; jmp %s", + self.X, + self.Y, + self.Op, + self.To, + self.Ln, + ) } func (self *IrAMD64_Jcc_rp) Usages() []*Reg { - return []*Reg { &self.X } + return []*Reg{&self.X} } func (self *IrAMD64_Jcc_rp) Successors() IrSuccessors { - return &_SwitchSuccessors { - i: -1, - t: []_SwitchTarget { - { b: self.Ln, i: 0 }, - { b: self.To }, - }, - } + return &_SwitchSuccessors{ + i: -1, + t: []_SwitchTarget{ + {b: self.Ln, i: 0}, + {b: self.To}, + }, + } } type IrAMD64_Jcc_ir struct { - X int32 - Y Reg - To *IrBranch - Ln *IrBranch - Op IrAMD64_CmpOp + X int32 + Y Reg + To *IrBranch + Ln *IrBranch + Op IrAMD64_CmpOp } func (self *IrAMD64_Jcc_ir) Clone() IrNode { - r := *self - r.To = self.To.Clone() - r.Ln = self.Ln.Clone() - return &r + r := *self + r.To = self.To.Clone() + r.Ln = self.Ln.Clone() + return &r } func (self *IrAMD64_Jcc_ir) String() string { - return fmt.Sprintf( - "cmpq $%d, %s; j%s %s; jmp %s # %#x", - self.X, - self.Y, - self.Op, - self.To, - self.Ln, - self.X, - ) + return fmt.Sprintf( + "cmpq $%d, %s; j%s %s; jmp %s # %#x", + self.X, + self.Y, + self.Op, + self.To, + self.Ln, + self.X, + ) } func (self *IrAMD64_Jcc_ir) Usages() []*Reg { - return []*Reg { &self.Y } + return []*Reg{&self.Y} } func (self *IrAMD64_Jcc_ir) Successors() IrSuccessors { - return &_SwitchSuccessors { - i: -1, - t: []_SwitchTarget { - { b: self.Ln, i: 0 }, - { b: self.To }, - }, - } + return &_SwitchSuccessors{ + i: -1, + t: []_SwitchTarget{ + {b: self.Ln, i: 0}, + {b: self.To}, + }, + } } type IrAMD64_Jcc_pr struct { - X unsafe.Pointer - Y Reg - To *IrBranch - Ln *IrBranch - Op IrAMD64_CmpOp + X unsafe.Pointer + Y Reg + To *IrBranch + Ln *IrBranch + Op IrAMD64_CmpOp } func (self *IrAMD64_Jcc_pr) Clone() IrNode { - r := *self - r.To = self.To.Clone() - r.Ln = self.Ln.Clone() - return &r + r := *self + r.To = self.To.Clone() + r.Ln = self.Ln.Clone() + return &r } func (self *IrAMD64_Jcc_pr) String() string { - return fmt.Sprintf( - "cmpq $%p, %s; j%s %s; jmp %s", - self.X, - self.Y, - self.Op, - self.To, - self.Ln, - ) + return fmt.Sprintf( + "cmpq $%p, %s; j%s %s; jmp %s", + self.X, + self.Y, + self.Op, + self.To, + self.Ln, + ) } func (self *IrAMD64_Jcc_pr) Usages() []*Reg { - return []*Reg { &self.Y } + return []*Reg{&self.Y} } func (self *IrAMD64_Jcc_pr) Successors() IrSuccessors { - return &_SwitchSuccessors { - i: -1, - t: []_SwitchTarget { - { b: self.Ln, i: 0 }, - { b: self.To }, - }, - } + return &_SwitchSuccessors{ + i: -1, + t: []_SwitchTarget{ + {b: self.Ln, i: 0}, + {b: self.To}, + }, + } } type IrAMD64_Jcc_rm struct { - X Reg - Y Mem - N uint8 - To *IrBranch - Ln *IrBranch - Op IrAMD64_CmpOp + X Reg + Y Mem + N uint8 + To *IrBranch + Ln *IrBranch + Op IrAMD64_CmpOp } func (self *IrAMD64_Jcc_rm) MemOp() *Mem { - return &self.Y + return &self.Y } func (self *IrAMD64_Jcc_rm) Clone() IrNode { - r := *self - r.To = self.To.Clone() - r.Ln = self.Ln.Clone() - return &r + r := *self + r.To = self.To.Clone() + r.Ln = self.Ln.Clone() + return &r } func (self *IrAMD64_Jcc_rm) String() string { - return fmt.Sprintf( - "cmp%c %s, %s; j%s %s; jmp %s", - memsizec(self.N), - self.X, - self.Y, - self.Op, - self.To, - self.Ln, - ) + return fmt.Sprintf( + "cmp%c %s, %s; j%s %s; jmp %s", + memsizec(self.N), + self.X, + self.Y, + self.Op, + self.To, + self.Ln, + ) } func (self *IrAMD64_Jcc_rm) Usages() []*Reg { - if self.Y.I == Rz { - return []*Reg { &self.X, &self.Y.M } - } else { - return []*Reg { &self.X, &self.Y.M, &self.Y.I } - } + if self.Y.I == Rz { + return []*Reg{&self.X, &self.Y.M} + } else { + return []*Reg{&self.X, &self.Y.M, &self.Y.I} + } } func (self *IrAMD64_Jcc_rm) Successors() IrSuccessors { - return &_SwitchSuccessors { - i: -1, - t: []_SwitchTarget { - { b: self.Ln, i: 0 }, - { b: self.To }, - }, - } + return &_SwitchSuccessors{ + i: -1, + t: []_SwitchTarget{ + {b: self.Ln, i: 0}, + {b: self.To}, + }, + } } type IrAMD64_Jcc_mr struct { - X Mem - Y Reg - N uint8 - To *IrBranch - Ln *IrBranch - Op IrAMD64_CmpOp + X Mem + Y Reg + N uint8 + To *IrBranch + Ln *IrBranch + Op IrAMD64_CmpOp } func (self *IrAMD64_Jcc_mr) MemOp() *Mem { - return &self.X + return &self.X } func (self *IrAMD64_Jcc_mr) Clone() IrNode { - r := *self - r.To = self.To.Clone() - r.Ln = self.Ln.Clone() - return &r + r := *self + r.To = self.To.Clone() + r.Ln = self.Ln.Clone() + return &r } func (self *IrAMD64_Jcc_mr) String() string { - return fmt.Sprintf( - "cmp%c %s, %s; j%s %s; jmp %s", - memsizec(self.N), - self.X, - self.Y, - self.Op, - self.To, - self.Ln, - ) + return fmt.Sprintf( + "cmp%c %s, %s; j%s %s; jmp %s", + memsizec(self.N), + self.X, + self.Y, + self.Op, + self.To, + self.Ln, + ) } func (self *IrAMD64_Jcc_mr) Usages() []*Reg { - if self.X.I == Rz { - return []*Reg { &self.X.M, &self.Y } - } else { - return []*Reg { &self.X.M, &self.X.I, &self.Y } - } + if self.X.I == Rz { + return []*Reg{&self.X.M, &self.Y} + } else { + return []*Reg{&self.X.M, &self.X.I, &self.Y} + } } func (self *IrAMD64_Jcc_mr) Successors() IrSuccessors { - return &_SwitchSuccessors { - i: -1, - t: []_SwitchTarget { - { b: self.Ln, i: 0 }, - { b: self.To }, - }, - } + return &_SwitchSuccessors{ + i: -1, + t: []_SwitchTarget{ + {b: self.Ln, i: 0}, + {b: self.To}, + }, + } } type IrAMD64_Jcc_mi struct { - X Mem - Y int32 - N uint8 - To *IrBranch - Ln *IrBranch - Op IrAMD64_CmpOp + X Mem + Y int32 + N uint8 + To *IrBranch + Ln *IrBranch + Op IrAMD64_CmpOp } func (self *IrAMD64_Jcc_mi) MemOp() *Mem { - return &self.X + return &self.X } func (self *IrAMD64_Jcc_mi) Clone() IrNode { - r := *self - r.To = self.To.Clone() - r.Ln = self.Ln.Clone() - return &r + r := *self + r.To = self.To.Clone() + r.Ln = self.Ln.Clone() + return &r } func (self *IrAMD64_Jcc_mi) String() string { - return fmt.Sprintf( - "cmp%c %s, $%d; j%s %s; jmp %s # %#x", - memsizec(self.N), - self.X, - self.Y, - self.Op, - self.To, - self.Ln, - self.Y, - ) + return fmt.Sprintf( + "cmp%c %s, $%d; j%s %s; jmp %s # %#x", + memsizec(self.N), + self.X, + self.Y, + self.Op, + self.To, + self.Ln, + self.Y, + ) } func (self *IrAMD64_Jcc_mi) Usages() []*Reg { - if self.X.I == Rz { - return []*Reg { &self.X.M } - } else { - return []*Reg { &self.X.M, &self.X.I } - } + if self.X.I == Rz { + return []*Reg{&self.X.M} + } else { + return []*Reg{&self.X.M, &self.X.I} + } } func (self *IrAMD64_Jcc_mi) Successors() IrSuccessors { - return &_SwitchSuccessors { - i: -1, - t: []_SwitchTarget { - { b: self.Ln, i: 0 }, - { b: self.To }, - }, - } + return &_SwitchSuccessors{ + i: -1, + t: []_SwitchTarget{ + {b: self.Ln, i: 0}, + {b: self.To}, + }, + } } type IrAMD64_Jcc_mp struct { - X Mem - Y unsafe.Pointer - To *IrBranch - Ln *IrBranch - Op IrAMD64_CmpOp + X Mem + Y unsafe.Pointer + To *IrBranch + Ln *IrBranch + Op IrAMD64_CmpOp } func (self *IrAMD64_Jcc_mp) MemOp() *Mem { - return &self.X + return &self.X } func (self *IrAMD64_Jcc_mp) Clone() IrNode { - r := *self - r.To = self.To.Clone() - r.Ln = self.Ln.Clone() - return &r + r := *self + r.To = self.To.Clone() + r.Ln = self.Ln.Clone() + return &r } func (self *IrAMD64_Jcc_mp) String() string { - return fmt.Sprintf( - "cmpq %s, $%d; j%s %s; jmp %s # %#x", - self.X, - self.Y, - self.Op, - self.To, - self.Ln, - self.Y, - ) + return fmt.Sprintf( + "cmpq %s, $%d; j%s %s; jmp %s # %#x", + self.X, + self.Y, + self.Op, + self.To, + self.Ln, + self.Y, + ) } func (self *IrAMD64_Jcc_mp) Usages() []*Reg { - if self.X.I == Rz { - return []*Reg { &self.X.M } - } else { - return []*Reg { &self.X.M, &self.X.I } - } + if self.X.I == Rz { + return []*Reg{&self.X.M} + } else { + return []*Reg{&self.X.M, &self.X.I} + } } func (self *IrAMD64_Jcc_mp) Successors() IrSuccessors { - return &_SwitchSuccessors { - i: -1, - t: []_SwitchTarget { - { b: self.Ln, i: 0 }, - { b: self.To }, - }, - } + return &_SwitchSuccessors{ + i: -1, + t: []_SwitchTarget{ + {b: self.Ln, i: 0}, + {b: self.To}, + }, + } } type IrAMD64_Jcc_im struct { - X int32 - Y Mem - N uint8 - To *IrBranch - Ln *IrBranch - Op IrAMD64_CmpOp + X int32 + Y Mem + N uint8 + To *IrBranch + Ln *IrBranch + Op IrAMD64_CmpOp } func (self *IrAMD64_Jcc_im) MemOp() *Mem { - return &self.Y + return &self.Y } func (self *IrAMD64_Jcc_im) Clone() IrNode { - r := *self - r.To = self.To.Clone() - r.Ln = self.Ln.Clone() - return &r + r := *self + r.To = self.To.Clone() + r.Ln = self.Ln.Clone() + return &r } func (self *IrAMD64_Jcc_im) String() string { - return fmt.Sprintf( - "cmp%c $%d, %s; j%s %s; jmp %s # %#x", - memsizec(self.N), - self.X, - self.Y, - self.Op, - self.To, - self.Ln, - self.X, - ) + return fmt.Sprintf( + "cmp%c $%d, %s; j%s %s; jmp %s # %#x", + memsizec(self.N), + self.X, + self.Y, + self.Op, + self.To, + self.Ln, + self.X, + ) } func (self *IrAMD64_Jcc_im) Usages() []*Reg { - if self.Y.I == Rz { - return []*Reg { &self.Y.M } - } else { - return []*Reg { &self.Y.M, &self.Y.I } - } + if self.Y.I == Rz { + return []*Reg{&self.Y.M} + } else { + return []*Reg{&self.Y.M, &self.Y.I} + } } func (self *IrAMD64_Jcc_im) Successors() IrSuccessors { - return &_SwitchSuccessors { - i: -1, - t: []_SwitchTarget { - { b: self.Ln, i: 0 }, - { b: self.To }, - }, - } + return &_SwitchSuccessors{ + i: -1, + t: []_SwitchTarget{ + {b: self.Ln, i: 0}, + {b: self.To}, + }, + } } type IrAMD64_Jcc_pm struct { - X unsafe.Pointer - Y Mem - To *IrBranch - Ln *IrBranch - Op IrAMD64_CmpOp + X unsafe.Pointer + Y Mem + To *IrBranch + Ln *IrBranch + Op IrAMD64_CmpOp } func (self *IrAMD64_Jcc_pm) MemOp() *Mem { - return &self.Y + return &self.Y } func (self *IrAMD64_Jcc_pm) Clone() IrNode { - r := *self - r.To = self.To.Clone() - r.Ln = self.Ln.Clone() - return &r + r := *self + r.To = self.To.Clone() + r.Ln = self.Ln.Clone() + return &r } func (self *IrAMD64_Jcc_pm) String() string { - return fmt.Sprintf( - "cmpq $%p, %s; j%s %s; jmp %s", - self.X, - self.Y, - self.Op, - self.To, - self.Ln, - ) + return fmt.Sprintf( + "cmpq $%p, %s; j%s %s; jmp %s", + self.X, + self.Y, + self.Op, + self.To, + self.Ln, + ) } func (self *IrAMD64_Jcc_pm) Usages() []*Reg { - if self.Y.I == Rz { - return []*Reg { &self.Y.M } - } else { - return []*Reg { &self.Y.M, &self.Y.I } - } + if self.Y.I == Rz { + return []*Reg{&self.Y.M} + } else { + return []*Reg{&self.Y.M, &self.Y.I} + } } func (self *IrAMD64_Jcc_pm) Successors() IrSuccessors { - return &_SwitchSuccessors { - i: -1, - t: []_SwitchTarget { - { b: self.Ln, i: 0 }, - { b: self.To }, - }, - } + return &_SwitchSuccessors{ + i: -1, + t: []_SwitchTarget{ + {b: self.Ln, i: 0}, + {b: self.To}, + }, + } } diff --git a/internal/atm/ssa/pass_abispec.go b/internal/atm/ssa/pass_abispec.go index 5ae849f..77b2593 100644 --- a/internal/atm/ssa/pass_abispec.go +++ b/internal/atm/ssa/pass_abispec.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,98 +17,103 @@ package ssa import ( - `fmt` + "fmt" ) // ABILowering lowers ABI-specific instructions to machine specific instructions. type ABILowering struct{} func (self ABILowering) Apply(cfg *CFG) { - rr := make([]Reg, len(cfg.Layout.Args)) - rb := rr[:0] + rr := make([]Reg, len(cfg.Layout.Args)) + rb := rr[:0] - /* map each argument load to register alias or stack load */ - for i, v := range cfg.Root.Ins { - if iv, ok := v.(*IrLoadArg); !ok { - break - } else if iv.I < 0 || iv.I >= len(cfg.Layout.Args) { - panic("abi: argument load out of bound: " + v.String()) - } else if rr[iv.I] != 0 { - panic("abi: second load of the same argument: " + v.String()) - } else if a := cfg.Layout.Args[iv.I]; a.InRegister { - rr[iv.I] = IrSetArch(Rz, a.Reg) - cfg.Root.Ins[i] = &IrAlias { R: iv.R, V: rr[iv.I] } - } else { - rr[iv.I] = Rz - cfg.Root.Ins[i] = IrArchLoadStack(iv.R, a.Mem, IrSlotArgs) - } - } + /* map each argument load to register alias or stack load */ + for i, v := range cfg.Root.Ins { + if iv, ok := v.(*IrLoadArg); !ok { + break + } else if iv.I < 0 || iv.I >= len(cfg.Layout.Args) { + panic("abi: argument load out of bound: " + v.String()) + } else if rr[iv.I] != 0 { + panic("abi: second load of the same argument: " + v.String()) + } else if a := cfg.Layout.Args[iv.I]; a.InRegister { + rr[iv.I] = IrSetArch(Rz, a.Reg) + cfg.Root.Ins[i] = &IrAlias{R: iv.R, V: rr[iv.I]} + } else { + rr[iv.I] = Rz + cfg.Root.Ins[i] = IrArchLoadStack(iv.R, a.Mem, IrSlotArgs) + } + } - /* extract all the registers */ - for _, r := range rr { - if r != 0 && r != Rz { - rb = append(rb, r) - } - } + /* extract all the registers */ + for _, r := range rr { + if r != 0 && r != Rz { + rb = append(rb, r) + } + } - /* insert an entry point node to hold all the register arguments */ - if len(rb) != 0 { - cfg.Root.Ins = append( - []IrNode { &IrEntry { rb } }, - cfg.Root.Ins... - ) - } + /* insert an entry point node to hold all the register arguments */ + if len(rb) != 0 { + cfg.Root.Ins = append( + []IrNode{&IrEntry{rb}}, + cfg.Root.Ins..., + ) + } - /* lower the entire program */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - ins := bb.Ins - bb.Ins = make([]IrNode, 0, len(ins)) + /* lower the entire program */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + ins := bb.Ins + bb.Ins = make([]IrNode, 0, len(ins)) - /* scan every instruction */ - for _, v := range ins { - switch p := v.(type) { - case *IrLoadArg : panic(fmt.Sprintf("abi: argument load in the middle of CFG: bb_%d: %s", bb.Id, v)) - case *IrCallFunc : self.abiCallFunc(cfg, bb, p) - case *IrCallNative : self.abiCallNative(cfg, bb, p) - case *IrCallMethod : self.abiCallMethod(cfg, bb, p) - default : bb.Ins = append(bb.Ins, p) - } - } + /* scan every instruction */ + for _, v := range ins { + switch p := v.(type) { + case *IrLoadArg: + panic(fmt.Sprintf("abi: argument load in the middle of CFG: bb_%d: %s", bb.Id, v)) + case *IrCallFunc: + self.abiCallFunc(cfg, bb, p) + case *IrCallNative: + self.abiCallNative(cfg, bb, p) + case *IrCallMethod: + self.abiCallMethod(cfg, bb, p) + default: + bb.Ins = append(bb.Ins, p) + } + } - /* scan the terminator */ - if p, ok := bb.Term.(*IrReturn); ok { - rr = make([]Reg, len(p.R)) - rb = rr[:0] + /* scan the terminator */ + if p, ok := bb.Term.(*IrReturn); ok { + rr = make([]Reg, len(p.R)) + rb = rr[:0] - /* check for return value count */ - if len(p.R) != len(cfg.Layout.Rets) { - panic("abi: return value store size mismatch: " + p.String()) - } + /* check for return value count */ + if len(p.R) != len(cfg.Layout.Rets) { + panic("abi: return value store size mismatch: " + p.String()) + } - /* copy return values */ - for i, rv := range p.R { - if r := cfg.Layout.Rets[i]; r.InRegister { - rr[i] = IrSetArch(cfg.CreateRegister(rv.Ptr()), r.Reg) - bb.Ins = append(bb.Ins, IrArchCopy(rr[i], rv)) - } else { - rr[i] = Rz - bb.Ins = append(bb.Ins, IrArchStoreStack(rv, r.Mem, IrSlotArgs)) - } - } + /* copy return values */ + for i, rv := range p.R { + if r := cfg.Layout.Rets[i]; r.InRegister { + rr[i] = IrSetArch(cfg.CreateRegister(rv.Ptr()), r.Reg) + bb.Ins = append(bb.Ins, IrArchCopy(rr[i], rv)) + } else { + rr[i] = Rz + bb.Ins = append(bb.Ins, IrArchStoreStack(rv, r.Mem, IrSlotArgs)) + } + } - /* extract all the registers */ - for _, r := range rr { - if r != 0 && r != Rz { - rb = append(rb, r) - } - } + /* extract all the registers */ + for _, r := range rr { + if r != 0 && r != Rz { + rb = append(rb, r) + } + } - /* replace the terminator */ - if len(rb) != 0 { - bb.Term = IrArchReturn(rb) - } else { - bb.Term = IrArchReturn(nil) - } - } - }) + /* replace the terminator */ + if len(rb) != 0 { + bb.Term = IrArchReturn(rb) + } else { + bb.Term = IrArchReturn(nil) + } + } + }) } diff --git a/internal/atm/ssa/pass_abispec_amd64.go b/internal/atm/ssa/pass_abispec_amd64.go index 17281f3..1723795 100644 --- a/internal/atm/ssa/pass_abispec_amd64.go +++ b/internal/atm/ssa/pass_abispec_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,253 +17,253 @@ package ssa import ( - `github.com/cloudwego/iasm/x86_64` - `github.com/cloudwego/frugal/internal/atm/abi` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/abi" + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/iasm/x86_64" ) -var _AbiClobbersC = []x86_64.Register64 { - x86_64.RAX, - x86_64.RCX, - x86_64.RDX, - x86_64.RSI, - x86_64.RDI, - x86_64.R8, - x86_64.R9, - x86_64.R10, - x86_64.R11, +var _AbiClobbersC = []x86_64.Register64{ + x86_64.RAX, + x86_64.RCX, + x86_64.RDX, + x86_64.RSI, + x86_64.RDI, + x86_64.R8, + x86_64.R9, + x86_64.R10, + x86_64.R11, } -var _AbiClobbersGo = []x86_64.Register64 { - x86_64.RAX, - x86_64.RCX, - x86_64.RDX, - x86_64.RBX, - x86_64.RSI, - x86_64.RDI, - x86_64.R8, - x86_64.R9, - x86_64.R10, - x86_64.R11, - x86_64.R12, - x86_64.R13, - x86_64.R14, - x86_64.R15, +var _AbiClobbersGo = []x86_64.Register64{ + x86_64.RAX, + x86_64.RCX, + x86_64.RDX, + x86_64.RBX, + x86_64.RSI, + x86_64.RDI, + x86_64.R8, + x86_64.R9, + x86_64.R10, + x86_64.R11, + x86_64.R12, + x86_64.R13, + x86_64.R14, + x86_64.R15, } -var _NativeArgsOrder = [...]x86_64.Register64 { - x86_64.RDI, - x86_64.RSI, - x86_64.RDX, - x86_64.RCX, - x86_64.R8, - x86_64.R9, +var _NativeArgsOrder = [...]x86_64.Register64{ + x86_64.RDI, + x86_64.RSI, + x86_64.RDX, + x86_64.RCX, + x86_64.R8, + x86_64.R9, } func (ABILowering) abiCallFunc(cfg *CFG, bb *BasicBlock, p *IrCallFunc) { - argc := len(p.In) - retc := len(p.Out) - - /* check argument & return value count */ - if argc != len(p.Func.Args) || retc != len(p.Func.Rets) { - panic("abi: gcall argument count mismatch: " + p.String()) - } - - /* register buffer */ - argv := make([]Reg, 0, argc) - retv := make([]Reg, 0, retc) - clob := make([]Reg, 0, len(_AbiClobbersGo)) - rmap := make(map[x86_64.Register64]bool, len(_AbiClobbersGo)) - - /* add all arch registers */ - for _, r := range _AbiClobbersGo { - rmap[r] = true - } - - /* store each argument */ - for i, r := range p.In { - if v := p.Func.Args[i]; !v.InRegister { - mm := v.Mem - bb.Ins = append(bb.Ins, IrArchStoreStack(r, mm, IrSlotCall)) - } else { - rr := IrSetArch(cfg.CreateRegister(r.Ptr()), v.Reg) - bb.Ins, argv = append(bb.Ins, IrArchCopy(rr, r)), append(argv, rr) - } - } - - /* convert each return register */ - for i, r := range p.Out { - if v := p.Func.Rets[i]; v.InRegister && r.Kind() != K_zero { - retv = append(retv, IrSetArch(cfg.CreateRegister(r.Ptr()), v.Reg)) - delete(rmap, v.Reg) - } - } - - /* exclude return values from clobbering list (they are implied) */ - for _, r := range _AbiClobbersGo { - if rmap[r] { - clob = append(clob, IrSetArch(cfg.CreateRegister(false), r)) - } - } - - /* add the call instruction */ - bb.Ins = append(bb.Ins, &IrAMD64_CALL_reg { - Fn : p.R, - In : argv, - Out : retv, - Clob : clob, - }) - - /* declare clobber list if any */ - if len(clob) != 0 { - bb.Ins = append(bb.Ins, IrMarkClobber(clob...)) - } - - /* load each return value */ - for i, r := range p.Out { - if r.Kind() != K_zero { - if v := p.Func.Rets[i]; v.InRegister { - bb.Ins = append(bb.Ins, IrArchCopy(r, retv[i])) - } else { - bb.Ins = append(bb.Ins, IrArchLoadStack(r, v.Mem, IrSlotCall)) - } - } - } + argc := len(p.In) + retc := len(p.Out) + + /* check argument & return value count */ + if argc != len(p.Func.Args) || retc != len(p.Func.Rets) { + panic("abi: gcall argument count mismatch: " + p.String()) + } + + /* register buffer */ + argv := make([]Reg, 0, argc) + retv := make([]Reg, 0, retc) + clob := make([]Reg, 0, len(_AbiClobbersGo)) + rmap := make(map[x86_64.Register64]bool, len(_AbiClobbersGo)) + + /* add all arch registers */ + for _, r := range _AbiClobbersGo { + rmap[r] = true + } + + /* store each argument */ + for i, r := range p.In { + if v := p.Func.Args[i]; !v.InRegister { + mm := v.Mem + bb.Ins = append(bb.Ins, IrArchStoreStack(r, mm, IrSlotCall)) + } else { + rr := IrSetArch(cfg.CreateRegister(r.Ptr()), v.Reg) + bb.Ins, argv = append(bb.Ins, IrArchCopy(rr, r)), append(argv, rr) + } + } + + /* convert each return register */ + for i, r := range p.Out { + if v := p.Func.Rets[i]; v.InRegister && r.Kind() != K_zero { + retv = append(retv, IrSetArch(cfg.CreateRegister(r.Ptr()), v.Reg)) + delete(rmap, v.Reg) + } + } + + /* exclude return values from clobbering list (they are implied) */ + for _, r := range _AbiClobbersGo { + if rmap[r] { + clob = append(clob, IrSetArch(cfg.CreateRegister(false), r)) + } + } + + /* add the call instruction */ + bb.Ins = append(bb.Ins, &IrAMD64_CALL_reg{ + Fn: p.R, + In: argv, + Out: retv, + Clob: clob, + }) + + /* declare clobber list if any */ + if len(clob) != 0 { + bb.Ins = append(bb.Ins, IrMarkClobber(clob...)) + } + + /* load each return value */ + for i, r := range p.Out { + if r.Kind() != K_zero { + if v := p.Func.Rets[i]; v.InRegister { + bb.Ins = append(bb.Ins, IrArchCopy(r, retv[i])) + } else { + bb.Ins = append(bb.Ins, IrArchLoadStack(r, v.Mem, IrSlotCall)) + } + } + } } func (ABILowering) abiCallNative(cfg *CFG, bb *BasicBlock, p *IrCallNative) { - retv := Rz - argc := len(p.In) - - /* check for argument count */ - if argc > len(_NativeArgsOrder) { - panic("abi: too many native arguments: " + p.String()) - } - - /* register buffers */ - argv := make([]Reg, 0, argc) - clob := make([]Reg, 0, len(_AbiClobbersC)) - rmap := make(map[x86_64.Register64]bool, len(_AbiClobbersC)) - - /* add all arch registers */ - for _, r := range _AbiClobbersC { - rmap[r] = true - } - - /* convert each argument */ - for i, r := range p.In { - argv = append(argv, IrSetArch(cfg.CreateRegister(r.Ptr()), _NativeArgsOrder[i])) - bb.Ins = append(bb.Ins, IrArchCopy(argv[i], r)) - } - - /* allocate register for return value if needed */ - if p.Out.Kind() != K_zero { - retv = IrSetArch(cfg.CreateRegister(p.Out.Ptr()), x86_64.RAX) - delete(rmap, x86_64.RAX) - } - - /* exclude return values from clobbering list (they are implied) */ - for _, r := range _AbiClobbersC { - if rmap[r] { - clob = append(clob, IrSetArch(cfg.CreateRegister(false), r)) - } - } - - /* add the call instruction */ - bb.Ins = append(bb.Ins, &IrAMD64_CALL_reg { - Fn : p.R, - In : argv, - Out : []Reg { retv }, - Clob : clob, - }) - - /* declare clobber list if any */ - if len(clob) != 0 { - bb.Ins = append(bb.Ins, IrMarkClobber(clob...)) - } - - /* copy the return value if needed */ - if p.Out.Kind() != K_zero { - bb.Ins = append(bb.Ins, IrArchCopy(p.Out, retv)) - } + retv := Rz + argc := len(p.In) + + /* check for argument count */ + if argc > len(_NativeArgsOrder) { + panic("abi: too many native arguments: " + p.String()) + } + + /* register buffers */ + argv := make([]Reg, 0, argc) + clob := make([]Reg, 0, len(_AbiClobbersC)) + rmap := make(map[x86_64.Register64]bool, len(_AbiClobbersC)) + + /* add all arch registers */ + for _, r := range _AbiClobbersC { + rmap[r] = true + } + + /* convert each argument */ + for i, r := range p.In { + argv = append(argv, IrSetArch(cfg.CreateRegister(r.Ptr()), _NativeArgsOrder[i])) + bb.Ins = append(bb.Ins, IrArchCopy(argv[i], r)) + } + + /* allocate register for return value if needed */ + if p.Out.Kind() != K_zero { + retv = IrSetArch(cfg.CreateRegister(p.Out.Ptr()), x86_64.RAX) + delete(rmap, x86_64.RAX) + } + + /* exclude return values from clobbering list (they are implied) */ + for _, r := range _AbiClobbersC { + if rmap[r] { + clob = append(clob, IrSetArch(cfg.CreateRegister(false), r)) + } + } + + /* add the call instruction */ + bb.Ins = append(bb.Ins, &IrAMD64_CALL_reg{ + Fn: p.R, + In: argv, + Out: []Reg{retv}, + Clob: clob, + }) + + /* declare clobber list if any */ + if len(clob) != 0 { + bb.Ins = append(bb.Ins, IrMarkClobber(clob...)) + } + + /* copy the return value if needed */ + if p.Out.Kind() != K_zero { + bb.Ins = append(bb.Ins, IrArchCopy(p.Out, retv)) + } } func (ABILowering) abiCallMethod(cfg *CFG, bb *BasicBlock, p *IrCallMethod) { - argc := len(p.In) + 1 - retc := len(p.Out) - - /* check argument & return value count */ - if argc != len(p.Func.Args) || retc != len(p.Func.Rets) { - panic("abi: icall argument count mismatch: " + p.String()) - } - - /* register buffer */ - argv := make([]Reg, 0, argc) - retv := make([]Reg, 0, retc) - clob := make([]Reg, 0, len(_AbiClobbersGo)) - rmap := make(map[x86_64.Register64]bool, len(_AbiClobbersGo)) - - /* add all arch registers */ - for _, r := range _AbiClobbersGo { - rmap[r] = true - } - - /* store the receiver */ - if rx := p.Func.Args[0]; !rx.InRegister { - mm := p.Func.Args[0].Mem - bb.Ins = append(bb.Ins, IrArchStoreStack(p.V, mm, IrSlotCall)) - } else { - rr := IrSetArch(cfg.CreateRegister(p.V.Ptr()), rx.Reg) - bb.Ins, argv = append(bb.Ins, IrArchCopy(rr, p.V)), append(argv, rr) - } - - /* store each argument */ - for i, r := range p.In { - if v := p.Func.Args[i+1]; !v.InRegister { - mm := v.Mem - bb.Ins = append(bb.Ins, IrArchStoreStack(r, mm, IrSlotCall)) - } else { - rr := IrSetArch(cfg.CreateRegister(r.Ptr()), v.Reg) - bb.Ins, argv = append(bb.Ins, IrArchCopy(rr, r)), append(argv, rr) - } - } - - /* convert each return register */ - for i, r := range p.Out { - if v := p.Func.Rets[i]; v.InRegister && r.Kind() != K_zero { - retv = append(retv, IrSetArch(cfg.CreateRegister(r.Ptr()), v.Reg)) - delete(rmap, v.Reg) - } - } - - /* exclude return values from clobbering list (they are implied) */ - for _, r := range _AbiClobbersGo { - if rmap[r] { - clob = append(clob, IrSetArch(cfg.CreateRegister(false), r)) - } - } - - /* add the call instruction */ - bb.Ins = append(bb.Ins, &IrAMD64_CALL_mem { - Fn : Ptr(p.T, int32(rt.GoItabFuncBase) + int32(p.Slot) * abi.PtrSize), - In : argv, - Out : retv, - Clob : clob, - }) - - /* declare clobber list if any */ - if len(clob) != 0 { - bb.Ins = append(bb.Ins, IrMarkClobber(clob...)) - } - - /* load each return value */ - for i, r := range p.Out { - if r.Kind() != K_zero { - if v := p.Func.Rets[i]; v.InRegister { - bb.Ins = append(bb.Ins, IrArchCopy(r, retv[i])) - } else { - bb.Ins = append(bb.Ins, IrArchLoadStack(r, v.Mem, IrSlotCall)) - } - } - } -} \ No newline at end of file + argc := len(p.In) + 1 + retc := len(p.Out) + + /* check argument & return value count */ + if argc != len(p.Func.Args) || retc != len(p.Func.Rets) { + panic("abi: icall argument count mismatch: " + p.String()) + } + + /* register buffer */ + argv := make([]Reg, 0, argc) + retv := make([]Reg, 0, retc) + clob := make([]Reg, 0, len(_AbiClobbersGo)) + rmap := make(map[x86_64.Register64]bool, len(_AbiClobbersGo)) + + /* add all arch registers */ + for _, r := range _AbiClobbersGo { + rmap[r] = true + } + + /* store the receiver */ + if rx := p.Func.Args[0]; !rx.InRegister { + mm := p.Func.Args[0].Mem + bb.Ins = append(bb.Ins, IrArchStoreStack(p.V, mm, IrSlotCall)) + } else { + rr := IrSetArch(cfg.CreateRegister(p.V.Ptr()), rx.Reg) + bb.Ins, argv = append(bb.Ins, IrArchCopy(rr, p.V)), append(argv, rr) + } + + /* store each argument */ + for i, r := range p.In { + if v := p.Func.Args[i+1]; !v.InRegister { + mm := v.Mem + bb.Ins = append(bb.Ins, IrArchStoreStack(r, mm, IrSlotCall)) + } else { + rr := IrSetArch(cfg.CreateRegister(r.Ptr()), v.Reg) + bb.Ins, argv = append(bb.Ins, IrArchCopy(rr, r)), append(argv, rr) + } + } + + /* convert each return register */ + for i, r := range p.Out { + if v := p.Func.Rets[i]; v.InRegister && r.Kind() != K_zero { + retv = append(retv, IrSetArch(cfg.CreateRegister(r.Ptr()), v.Reg)) + delete(rmap, v.Reg) + } + } + + /* exclude return values from clobbering list (they are implied) */ + for _, r := range _AbiClobbersGo { + if rmap[r] { + clob = append(clob, IrSetArch(cfg.CreateRegister(false), r)) + } + } + + /* add the call instruction */ + bb.Ins = append(bb.Ins, &IrAMD64_CALL_mem{ + Fn: Ptr(p.T, int32(rt.GoItabFuncBase)+int32(p.Slot)*abi.PtrSize), + In: argv, + Out: retv, + Clob: clob, + }) + + /* declare clobber list if any */ + if len(clob) != 0 { + bb.Ins = append(bb.Ins, IrMarkClobber(clob...)) + } + + /* load each return value */ + for i, r := range p.Out { + if r.Kind() != K_zero { + if v := p.Func.Rets[i]; v.InRegister { + bb.Ins = append(bb.Ins, IrArchCopy(r, retv[i])) + } else { + bb.Ins = append(bb.Ins, IrArchLoadStack(r, v.Mem, IrSlotCall)) + } + } + } +} diff --git a/internal/atm/ssa/pass_blockmerge.go b/internal/atm/ssa/pass_blockmerge.go index 8db7dd3..3a48913 100644 --- a/internal/atm/ssa/pass_blockmerge.go +++ b/internal/atm/ssa/pass_blockmerge.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,65 +21,64 @@ package ssa type BlockMerge struct{} func (BlockMerge) Apply(cfg *CFG) { - for { - var rt bool - var nx *BasicBlock - var it IrSuccessors - var tr IrTerminator + for { + var rt bool + var nx *BasicBlock + var it IrSuccessors + var tr IrTerminator - /* check every block */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - tr = bb.Term - it = tr.Successors() + /* check every block */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + tr = bb.Term + it = tr.Successors() - /* it must have successors */ - if !it.Next() { - return - } + /* it must have successors */ + if !it.Next() { + return + } - /* it must have exactly 1 successor, and the successor must have exactly 1 predecessor */ - if nx = it.Block(); it.Next() || len(nx.Pred) != 1 { - return - } + /* it must have exactly 1 successor, and the successor must have exactly 1 predecessor */ + if nx = it.Block(); it.Next() || len(nx.Pred) != 1 { + return + } - /* merge the two blocks */ - rt = true - bb.Ins = append(bb.Ins, nx.Ins...) - bb.Term = nx.Term + /* merge the two blocks */ + rt = true + bb.Ins = append(bb.Ins, nx.Ins...) + bb.Term = nx.Term - /* must not have Phi nodes */ - if len(nx.Phi) != 0 { - panic("invalid Phi node found in intermediate blocks") - } + /* must not have Phi nodes */ + if len(nx.Phi) != 0 { + panic("invalid Phi node found in intermediate blocks") + } - /* get the successor iterator */ - tr = nx.Term - it = tr.Successors() + /* get the successor iterator */ + tr = nx.Term + it = tr.Successors() - /* update all predecessors references */ - for it.Next() { - rb := it.Block() - pp := rb.Pred + /* update all predecessors references */ + for it.Next() { + rb := it.Block() + pp := rb.Pred - /* update predecessor list */ - for i, p := range pp { - if p == nx { - pp[i] = bb - } - } + /* update predecessor list */ + for i, p := range pp { + if p == nx { + pp[i] = bb + } + } - /* update in Phi nodes */ - for _, v := range rb.Phi { - v.V[bb] = v.V[nx] - delete(v.V, nx) - } - } - }) + /* update in Phi nodes */ + for _, v := range rb.Phi { + v.V[bb] = v.V[nx] + delete(v.V, nx) + } + } + }) - /* rebuild the dominator tree, and retry if needed */ - if cfg.Rebuild(); !rt { - break - } - } + /* rebuild the dominator tree, and retry if needed */ + if cfg.Rebuild(); !rt { + break + } + } } - diff --git a/internal/atm/ssa/pass_branchelim.go b/internal/atm/ssa/pass_branchelim.go index 307b57b..b0c1606 100644 --- a/internal/atm/ssa/pass_branchelim.go +++ b/internal/atm/ssa/pass_branchelim.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,592 +17,640 @@ package ssa import ( - `fmt` - `sort` - `strings` + "fmt" + "sort" + "strings" - `github.com/oleiade/lane` + "github.com/oleiade/lane" ) type _Term interface { - fmt.Stringer - term() + fmt.Stringer + term() } type ( - _TrRel uint8 - _RegTerm Reg - _ValueTerm Int65 + _TrRel uint8 + _RegTerm Reg + _ValueTerm Int65 ) -func (_Stmt) term() {} -func (_RegTerm) term() {} +func (_Stmt) term() {} +func (_RegTerm) term() {} func (_ValueTerm) term() {} -func (self _RegTerm) String() string { return Reg(self).String() } +func (self _RegTerm) String() string { return Reg(self).String() } func (self _ValueTerm) String() string { return Int65(self).String() } const ( - _R_eq _TrRel = iota - _R_ne - _R_lt - _R_ltu - _R_ge - _R_geu + _R_eq _TrRel = iota + _R_ne + _R_lt + _R_ltu + _R_ge + _R_geu ) func (self _TrRel) String() string { - switch self { - case _R_eq : return "==" - case _R_ne : return "!=" - case _R_lt : return "<" - case _R_ltu : return "<#" - case _R_ge : return ">=" - case _R_geu : return ">=#" - default : panic("unreachable") - } + switch self { + case _R_eq: + return "==" + case _R_ne: + return "!=" + case _R_lt: + return "<" + case _R_ltu: + return "<#" + case _R_ge: + return ">=" + case _R_geu: + return ">=#" + default: + panic("unreachable") + } } type _Edge struct { - bb *BasicBlock - to *BasicBlock + bb *BasicBlock + to *BasicBlock } func (self _Edge) String() string { - return fmt.Sprintf("bb_%d => bb_%d", self.bb.Id, self.to.Id) + return fmt.Sprintf("bb_%d => bb_%d", self.bb.Id, self.to.Id) } type _Stmt struct { - lhs Reg - rhs _Term - rel _TrRel + lhs Reg + rhs _Term + rel _TrRel } func (self _Stmt) String() string { - return fmt.Sprintf("%s %s %s", self.lhs, self.rel, self.rhs) + return fmt.Sprintf("%s %s %s", self.lhs, self.rel, self.rhs) } func (self _Stmt) negated() _Stmt { - switch self.rel { - case _R_eq : return _Stmt { self.lhs, self.rhs, _R_ne } - case _R_ne : return _Stmt { self.lhs, self.rhs, _R_eq } - case _R_lt : return _Stmt { self.lhs, self.rhs, _R_ge } - case _R_ltu : return _Stmt { self.lhs, self.rhs, _R_geu } - case _R_ge : return _Stmt { self.lhs, self.rhs, _R_lt } - case _R_geu : return _Stmt { self.lhs, self.rhs, _R_ltu } - default : panic("unreachable") - } + switch self.rel { + case _R_eq: + return _Stmt{self.lhs, self.rhs, _R_ne} + case _R_ne: + return _Stmt{self.lhs, self.rhs, _R_eq} + case _R_lt: + return _Stmt{self.lhs, self.rhs, _R_ge} + case _R_ltu: + return _Stmt{self.lhs, self.rhs, _R_geu} + case _R_ge: + return _Stmt{self.lhs, self.rhs, _R_lt} + case _R_geu: + return _Stmt{self.lhs, self.rhs, _R_ltu} + default: + panic("unreachable") + } } func (self _Stmt) condition(cond bool) _Stmt { - if cond { - return self - } else { - return self.negated() - } + if cond { + return self + } else { + return self.negated() + } } type _Range struct { - rr []Int65 + rr []Int65 } func newRange(lower Int65, upper Int65) *_Range { - return &_Range { - rr: []Int65 { lower, upper }, - } + return &_Range{ + rr: []Int65{lower, upper}, + } } func (self *_Range) lower() Int65 { - if len(self.rr) == 0 { - panic("empty range") - } else { - return self.rr[0] - } + if len(self.rr) == 0 { + panic("empty range") + } else { + return self.rr[0] + } } func (self *_Range) upper() Int65 { - if n := len(self.rr); n == 0 { - panic("empty range") - } else { - return self.rr[n - 1] - } + if n := len(self.rr); n == 0 { + panic("empty range") + } else { + return self.rr[n-1] + } } func (self *_Range) truth() (bool, bool) { - var lower Int65 - var upper Int65 - - /* empty range */ - if len(self.rr) == 0 { - return false, false - } - - /* fast path: there is only one range */ - if len(self.rr) == 2 { - if self.rr[0].CompareZero() == 0 && self.rr[1].CompareZero() == 0 { - return false, true - } else if self.rr[0].CompareZero() > 0 || self.rr[1].CompareZero() < 0 { - return true, true - } else { - return false, false - } - } - - /* check if any range contains the zero */ - for i := 0; i < len(self.rr); i += 2 { - lower = self.rr[i] - upper = self.rr[i + 1] - - /* the range contains zero, the truth cannot be determained */ - if lower.CompareZero() <= 0 && upper.CompareZero() >= 0 { - return false, false - } - } - - /* no, the range can be interpreted as true */ - return true, true + var lower Int65 + var upper Int65 + + /* empty range */ + if len(self.rr) == 0 { + return false, false + } + + /* fast path: there is only one range */ + if len(self.rr) == 2 { + if self.rr[0].CompareZero() == 0 && self.rr[1].CompareZero() == 0 { + return false, true + } else if self.rr[0].CompareZero() > 0 || self.rr[1].CompareZero() < 0 { + return true, true + } else { + return false, false + } + } + + /* check if any range contains the zero */ + for i := 0; i < len(self.rr); i += 2 { + lower = self.rr[i] + upper = self.rr[i+1] + + /* the range contains zero, the truth cannot be determained */ + if lower.CompareZero() <= 0 && upper.CompareZero() >= 0 { + return false, false + } + } + + /* no, the range can be interpreted as true */ + return true, true } func (self *_Range) remove(lower Int65, upper Int65) { - for i := 0; i < len(self.rr); i += 2 { - l := self.rr[i] - u := self.rr[i + 1] - - /* not intersecting */ - if lower.Compare(u) > 0 { break } - if upper.Compare(l) < 0 { continue } - - /* splicing */ - if lower.Compare(l) > 0 && upper.Compare(u) < 0 { - next := []Int65 { l, lower.OneLess(), upper.OneMore(), u } - self.rr = append(self.rr[:i], append(next, self.rr[i + 2:]...)...) - i += 2 - break - } - - /* remove the upper half */ - if lower.Compare(l) > 0 { - self.rr[i + 1] = lower.OneLess() - continue - } - - /* remove the lower half */ - if upper.Compare(u) < 0 { - self.rr[i] = upper.OneMore() - break - } - - /* remove the entire range */ - copy(self.rr[i:], self.rr[i + 2:]) - self.rr = self.rr[:len(self.rr) - 2] - i -= 2 - } + for i := 0; i < len(self.rr); i += 2 { + l := self.rr[i] + u := self.rr[i+1] + + /* not intersecting */ + if lower.Compare(u) > 0 { + break + } + if upper.Compare(l) < 0 { + continue + } + + /* splicing */ + if lower.Compare(l) > 0 && upper.Compare(u) < 0 { + next := []Int65{l, lower.OneLess(), upper.OneMore(), u} + self.rr = append(self.rr[:i], append(next, self.rr[i+2:]...)...) + i += 2 + break + } + + /* remove the upper half */ + if lower.Compare(l) > 0 { + self.rr[i+1] = lower.OneLess() + continue + } + + /* remove the lower half */ + if upper.Compare(u) < 0 { + self.rr[i] = upper.OneMore() + break + } + + /* remove the entire range */ + copy(self.rr[i:], self.rr[i+2:]) + self.rr = self.rr[:len(self.rr)-2] + i -= 2 + } } func (self *_Range) intersect(lower Int65, upper Int65) { - if lower != MinInt65 { self.remove(MinInt65, lower.OneLess()) } - if upper != MaxInt65 { self.remove(upper.OneMore(), MaxInt65) } + if lower != MinInt65 { + self.remove(MinInt65, lower.OneLess()) + } + if upper != MaxInt65 { + self.remove(upper.OneMore(), MaxInt65) + } } func (self *_Range) removeRange(r *_Range) { - for i := 0; i < len(r.rr); i += 2 { - self.remove(r.rr[i], r.rr[i + 1]) - } + for i := 0; i < len(r.rr); i += 2 { + self.remove(r.rr[i], r.rr[i+1]) + } } func (self *_Range) intersectRange(r *_Range) { - for i := 0; i < len(r.rr); i += 2 { - self.intersect(r.rr[i], r.rr[i + 1]) - } + for i := 0; i < len(r.rr); i += 2 { + self.intersect(r.rr[i], r.rr[i+1]) + } } func (self *_Range) String() string { - nb := len(self.rr) - rb := make([]string, nb / 2) - - /* empty ranges */ - if nb == 0 { - return "{ (empty) }" - } - - /* dump every range */ - for i := 0; i < nb; i += 2 { - l := self.rr[i] - u := self.rr[i + 1] - s := new(strings.Builder) - - /* lower bounds */ - if s.WriteRune('['); l == MinInt65 { - s.WriteString("-∞") - } else { - s.WriteString(fmt.Sprint(l)) - } - - /* upper bounds */ - if s.WriteString(", "); u == MaxInt65 { - s.WriteString("+∞") - } else { - s.WriteString(fmt.Sprint(u)) - } - - /* build the range */ - s.WriteRune(']') - rb[i / 2] = s.String() - } - - /* join them together */ - return fmt.Sprintf( - "{ %s }", - strings.Join(rb, " ∪ "), - ) + nb := len(self.rr) + rb := make([]string, nb/2) + + /* empty ranges */ + if nb == 0 { + return "{ (empty) }" + } + + /* dump every range */ + for i := 0; i < nb; i += 2 { + l := self.rr[i] + u := self.rr[i+1] + s := new(strings.Builder) + + /* lower bounds */ + if s.WriteRune('['); l == MinInt65 { + s.WriteString("-∞") + } else { + s.WriteString(fmt.Sprint(l)) + } + + /* upper bounds */ + if s.WriteString(", "); u == MaxInt65 { + s.WriteString("+∞") + } else { + s.WriteString(fmt.Sprint(u)) + } + + /* build the range */ + s.WriteRune(']') + rb[i/2] = s.String() + } + + /* join them together */ + return fmt.Sprintf( + "{ %s }", + strings.Join(rb, " ∪ "), + ) } type _Ranges struct { - rr map[Reg]*_Range + rr map[Reg]*_Range } func newRanges(nb int) (r _Ranges) { - r.rr = make(map[Reg]*_Range, nb) - r.rr[Rz] = newRange(Int65{}, Int65{}) - return + r.rr = make(map[Reg]*_Range, nb) + r.rr[Rz] = newRange(Int65{}, Int65{}) + return } func (self _Ranges) of(reg Reg) (r *_Range) { - var ok bool - var rr *_Range - - /* check for existing range */ - if rr, ok = self.rr[reg]; ok { - return rr - } - - /* create a new one if needed */ - rr = newRange(MinInt65, MaxInt65) - self.rr[reg] = rr - return rr + var ok bool + var rr *_Range + + /* check for existing range */ + if rr, ok = self.rr[reg]; ok { + return rr + } + + /* create a new one if needed */ + rr = newRange(MinInt65, MaxInt65) + self.rr[reg] = rr + return rr } func (self _Ranges) at(reg Reg) (r *_Range, ok bool) { - r, ok = self.rr[reg] - return + r, ok = self.rr[reg] + return } type _Proof struct { - cp []int - st []_Stmt + cp []int + st []_Stmt } func (self *_Proof) define(lhs Reg, rhs _Term, rel _TrRel) { - self.st = append(self.st, _Stmt { lhs, rhs, rel }) + self.st = append(self.st, _Stmt{lhs, rhs, rel}) } func (self *_Proof) assume(ref Reg, lhs Reg, rhs _Term, rel _TrRel) { - self.st = append(self.st, _Stmt { - lhs: ref, - rel: _R_eq, - rhs: _Stmt { lhs, rhs, rel }, - }) + self.st = append(self.st, _Stmt{ + lhs: ref, + rel: _R_eq, + rhs: _Stmt{lhs, rhs, rel}, + }) } func (self *_Proof) restore() { - p := len(self.cp) - 1 - self.st, self.cp = self.st[:self.cp[p]], self.cp[:p] + p := len(self.cp) - 1 + self.st, self.cp = self.st[:self.cp[p]], self.cp[:p] } func (self *_Proof) checkpoint() { - self.cp = append(self.cp, len(self.st)) + self.cp = append(self.cp, len(self.st)) } func (self *_Proof) isContradiction(st _Stmt) (ret bool) { - self.checkpoint() - self.st = append(self.st, st) - ret = !self.verifyCorrectness() - self.restore() - return + self.checkpoint() + self.st = append(self.st, st) + ret = !self.verifyCorrectness() + self.restore() + return } func (self *_Proof) verifyCorrectness() bool { - rt := true - rr := newRanges(len(self.st)) - sp := make([]_Stmt, 0, len(self.st)) - st := append([]_Stmt(nil), self.st...) - - /* calculate ranges for every variable */ - for rt { - rt = false - sp, st = st, sp[:0] - - /* update all the ranges */ - for _, v := range sp { - var f bool - var p _ValueTerm - - /* must be a value term */ - if p, f = v.rhs.(_ValueTerm); !f { - continue - } - - /* evaluate the range */ - switch x := Int65(p); v.rel { - default: { - panic("unreachable") - } - - /* simple ranges */ - case _R_ne: rr.of(v.lhs).remove(x, x) - case _R_eq: rr.of(v.lhs).intersect(x, x) - case _R_ge: rr.of(v.lhs).intersect(x, MaxInt65) - - /* signed less-than */ - case _R_lt: { - if x == MinInt65 { - return false - } else { - rr.of(v.lhs).intersect(MinInt65, x.OneLess()) - } - } - - /* unsigned greater-than-or-equal-to */ - case _R_geu: { - if x.CompareZero() < 0 { - panic(fmt.Sprintf("unsigned comparison to a negative value %s", x)) - } else { - rr.of(v.lhs).intersect(x, MaxInt65) - } - } - - /* unsigned less-than */ - case _R_ltu: { - if x.CompareZero() <= 0 { - panic(fmt.Sprintf("unsigned comparison to a non-positive value %s", x)) - } else { - rr.of(v.lhs).intersect(Int65{}, x.OneLess()) - } - } - } - } - - /* expand all the definations */ - for _, v := range sp { - if p, ok := v.rhs.(_Stmt); ok { - if r, rk := rr.at(v.lhs); rk { - if t, tk := r.truth(); tk { - rt = true - st = append(st, p.condition(t)) - } - } - } - } - - /* evaluate all the registers */ - for _, v := range sp { - var f bool - var x Int65 - var r *_Range - var t _RegTerm - - /* must be a register term with a valid range */ - if t, f = v.rhs.(_RegTerm) ; !f { continue } - if r, f = rr.at(Reg(t)) ; !f { continue } - - /* empty range, already found contradictions */ - if len(r.rr) == 0 { - return false - } - - /* update the ranges */ - switch v.rel { - default: { - panic("unreachable") - } - - /* equality and inequality */ - case _R_ne: rr.of(v.lhs).removeRange(r) - case _R_eq: rr.of(v.lhs).intersectRange(r) - - /* signed less-than */ - case _R_lt: { - rt = true - st = append(st, _Stmt { v.lhs, _ValueTerm(r.upper()), _R_lt }) - } - - /* signed greater-than */ - case _R_ge: { - rt = true - st = append(st, _Stmt { v.lhs, _ValueTerm(r.lower()), _R_ge }) - } - - /* unsigned less-than */ - case _R_ltu: { - if x, rt = r.upper(), true; x.CompareZero() > 0 { - st = append(st, _Stmt { v.lhs, _ValueTerm(x), _R_ltu }) - } else { - return false - } - } - - /* unsigned greater-than-or-equal-to */ - case _R_geu: { - if x, rt = r.lower(), true; x.CompareZero() >= 0 { - st = append(st, _Stmt { v.lhs, _ValueTerm(x), _R_geu }) - } else { - st = append(st, _Stmt { v.lhs, _ValueTerm(Int65{}), _R_geu }) - } - } - } - } - } - - /* the statements are valid iff there are no empty ranges */ - for _, r := range rr.rr { - if len(r.rr) == 0 { - return false - } - } - - /* all checked fine */ - return true + rt := true + rr := newRanges(len(self.st)) + sp := make([]_Stmt, 0, len(self.st)) + st := append([]_Stmt(nil), self.st...) + + /* calculate ranges for every variable */ + for rt { + rt = false + sp, st = st, sp[:0] + + /* update all the ranges */ + for _, v := range sp { + var f bool + var p _ValueTerm + + /* must be a value term */ + if p, f = v.rhs.(_ValueTerm); !f { + continue + } + + /* evaluate the range */ + switch x := Int65(p); v.rel { + default: + { + panic("unreachable") + } + + /* simple ranges */ + case _R_ne: + rr.of(v.lhs).remove(x, x) + case _R_eq: + rr.of(v.lhs).intersect(x, x) + case _R_ge: + rr.of(v.lhs).intersect(x, MaxInt65) + + /* signed less-than */ + case _R_lt: + { + if x == MinInt65 { + return false + } else { + rr.of(v.lhs).intersect(MinInt65, x.OneLess()) + } + } + + /* unsigned greater-than-or-equal-to */ + case _R_geu: + { + if x.CompareZero() < 0 { + panic(fmt.Sprintf("unsigned comparison to a negative value %s", x)) + } else { + rr.of(v.lhs).intersect(x, MaxInt65) + } + } + + /* unsigned less-than */ + case _R_ltu: + { + if x.CompareZero() <= 0 { + panic(fmt.Sprintf("unsigned comparison to a non-positive value %s", x)) + } else { + rr.of(v.lhs).intersect(Int65{}, x.OneLess()) + } + } + } + } + + /* expand all the definitions */ + for _, v := range sp { + if p, ok := v.rhs.(_Stmt); ok { + if r, rk := rr.at(v.lhs); rk { + if t, tk := r.truth(); tk { + rt = true + st = append(st, p.condition(t)) + } + } + } + } + + /* evaluate all the registers */ + for _, v := range sp { + var f bool + var x Int65 + var r *_Range + var t _RegTerm + + /* must be a register term with a valid range */ + if t, f = v.rhs.(_RegTerm); !f { + continue + } + if r, f = rr.at(Reg(t)); !f { + continue + } + + /* empty range, already found contradictions */ + if len(r.rr) == 0 { + return false + } + + /* update the ranges */ + switch v.rel { + default: + { + panic("unreachable") + } + + /* equality and inequality */ + case _R_ne: + rr.of(v.lhs).removeRange(r) + case _R_eq: + rr.of(v.lhs).intersectRange(r) + + /* signed less-than */ + case _R_lt: + { + rt = true + st = append(st, _Stmt{v.lhs, _ValueTerm(r.upper()), _R_lt}) + } + + /* signed greater-than */ + case _R_ge: + { + rt = true + st = append(st, _Stmt{v.lhs, _ValueTerm(r.lower()), _R_ge}) + } + + /* unsigned less-than */ + case _R_ltu: + { + if x, rt = r.upper(), true; x.CompareZero() > 0 { + st = append(st, _Stmt{v.lhs, _ValueTerm(x), _R_ltu}) + } else { + return false + } + } + + /* unsigned greater-than-or-equal-to */ + case _R_geu: + { + if x, rt = r.lower(), true; x.CompareZero() >= 0 { + st = append(st, _Stmt{v.lhs, _ValueTerm(x), _R_geu}) + } else { + st = append(st, _Stmt{v.lhs, _ValueTerm(Int65{}), _R_geu}) + } + } + } + } + } + + /* the statements are valid iff there are no empty ranges */ + for _, r := range rr.rr { + if len(r.rr) == 0 { + return false + } + } + + /* all checked fine */ + return true } // BranchElim removes branches that can be proved unreachable. type BranchElim struct{} func (self BranchElim) dfs(cfg *CFG, bb *BasicBlock, ps *_Proof) { - var ok bool - var sw *IrSwitch - - /* add facts for this basic block */ - for _, v := range bb.Ins { - switch p := v.(type) { - default: { - break - } - - /* integer constant */ - case *IrConstInt: { - ps.define(p.R, _ValueTerm(Int65i(p.V)), _R_eq) - } - - /* binary operators */ - case *IrBinaryExpr: { - switch p.Op { - case IrCmpEq : ps.assume(p.R, p.X, _RegTerm(p.Y), _R_eq) - case IrCmpNe : ps.assume(p.R, p.X, _RegTerm(p.Y), _R_ne) - case IrCmpLt : ps.assume(p.R, p.X, _RegTerm(p.Y), _R_lt) - case IrCmpLtu : ps.assume(p.R, p.X, _RegTerm(p.Y), _R_ltu) - case IrCmpGeu : ps.assume(p.R, p.X, _RegTerm(p.Y), _R_geu) - } - } - } - } - - /* only care about switches */ - if sw, ok = bb.Term.(*IrSwitch); !ok { - return - } - - /* edges to be removed */ - rem := lane.NewQueue() - del := make(map[_Edge]bool) - val := make([]int32, 0, len(sw.Br)) - - /* prove every branch */ - for v, p := range sw.Br { - if val = append(val, v); ps.isContradiction(_Stmt { sw.V, _ValueTerm(Int65i(int64(v))), _R_eq }) { - delete(sw.Br, v) - rem.Enqueue(_Edge { bb, p.To }) - } - } - - /* create a save-point */ - ps.checkpoint() - sort.Slice(val, func(i int, j int) bool { return val[i] < val[j] }) - - /* add all the negated conditions */ - for _, i := range val { - ps.define(sw.V, _ValueTerm(Int65i(int64(i))), _R_ne) - } - - /* prove the default branch */ - reachable := ps.verifyCorrectness() - ps.restore() - - /* check for reachability */ - if !reachable { - if rem.Enqueue(_Edge { bb, sw.Ln.To }); len(sw.Br) != 1 { - sw.Ln = IrUnlikely(cfg.CreateUnreachable(bb)) - } else { - sw.Ln, sw.Br = sw.Br[val[0]], make(map[int32]*IrBranch) - } - } - - /* clear register reference if needed */ - if len(sw.Br) == 0 { - sw.V = Rz - } - - /* adjust all the edges */ - for !rem.Empty() { - e := rem.Pop().(_Edge) - del[e] = true - - /* adjust Phi nodes in the target block */ - for _, v := range e.to.Phi { - delete(v.V, e.bb) - } - - /* remove predecessors from the target block */ - for i, p := range e.to.Pred { - if p == e.bb { - e.to.Pred = append(e.to.Pred[:i], e.to.Pred[i + 1:]...) - break - } - } - - /* remove the entire block if no more entry edges left */ - if len(e.to.Pred) == 0 { - for it := e.to.Term.Successors(); it.Next(); { - rem.Enqueue(_Edge { - bb: e.to, - to: it.Block(), - }) - } - } - } - - /* DFS the dominator tree */ - for _, p := range cfg.DominatorOf[bb.Id] { - var f bool - var v _ValueTerm - - /* no need to recurse into unreachable branches */ - if del[_Edge { bb, p }] { - continue - } - - /* find the branch value */ - for i, b := range sw.Br { - if b.To == p { - f, v = true, _ValueTerm(Int65i(int64(i))) - break - } - } - - /* it is not a direct successor, just pass all the facts down */ - if !f { - self.dfs(cfg, p, ps) - continue - } - - /* add the fact and recurse into the node */ - ps.checkpoint() - ps.define(sw.V, v, _R_eq) - self.dfs(cfg, p, ps) - ps.restore() - } + var ok bool + var sw *IrSwitch + + /* add facts for this basic block */ + for _, v := range bb.Ins { + switch p := v.(type) { + default: + { + break + } + + /* integer constant */ + case *IrConstInt: + { + ps.define(p.R, _ValueTerm(Int65i(p.V)), _R_eq) + } + + /* binary operators */ + case *IrBinaryExpr: + { + switch p.Op { + case IrCmpEq: + ps.assume(p.R, p.X, _RegTerm(p.Y), _R_eq) + case IrCmpNe: + ps.assume(p.R, p.X, _RegTerm(p.Y), _R_ne) + case IrCmpLt: + ps.assume(p.R, p.X, _RegTerm(p.Y), _R_lt) + case IrCmpLtu: + ps.assume(p.R, p.X, _RegTerm(p.Y), _R_ltu) + case IrCmpGeu: + ps.assume(p.R, p.X, _RegTerm(p.Y), _R_geu) + } + } + } + } + + /* only care about switches */ + if sw, ok = bb.Term.(*IrSwitch); !ok { + return + } + + /* edges to be removed */ + rem := lane.NewQueue() + del := make(map[_Edge]bool) + val := make([]int32, 0, len(sw.Br)) + + /* prove every branch */ + for v, p := range sw.Br { + if val = append(val, v); ps.isContradiction(_Stmt{sw.V, _ValueTerm(Int65i(int64(v))), _R_eq}) { + delete(sw.Br, v) + rem.Enqueue(_Edge{bb, p.To}) + } + } + + /* create a save-point */ + ps.checkpoint() + sort.Slice(val, func(i int, j int) bool { return val[i] < val[j] }) + + /* add all the negated conditions */ + for _, i := range val { + ps.define(sw.V, _ValueTerm(Int65i(int64(i))), _R_ne) + } + + /* prove the default branch */ + reachable := ps.verifyCorrectness() + ps.restore() + + /* check for reachability */ + if !reachable { + if rem.Enqueue(_Edge{bb, sw.Ln.To}); len(sw.Br) != 1 { + sw.Ln = IrUnlikely(cfg.CreateUnreachable(bb)) + } else { + sw.Ln, sw.Br = sw.Br[val[0]], make(map[int32]*IrBranch) + } + } + + /* clear register reference if needed */ + if len(sw.Br) == 0 { + sw.V = Rz + } + + /* adjust all the edges */ + for !rem.Empty() { + e := rem.Pop().(_Edge) + del[e] = true + + /* adjust Phi nodes in the target block */ + for _, v := range e.to.Phi { + delete(v.V, e.bb) + } + + /* remove predecessors from the target block */ + for i, p := range e.to.Pred { + if p == e.bb { + e.to.Pred = append(e.to.Pred[:i], e.to.Pred[i+1:]...) + break + } + } + + /* remove the entire block if no more entry edges left */ + if len(e.to.Pred) == 0 { + for it := e.to.Term.Successors(); it.Next(); { + rem.Enqueue(_Edge{ + bb: e.to, + to: it.Block(), + }) + } + } + } + + /* DFS the dominator tree */ + for _, p := range cfg.DominatorOf[bb.Id] { + var f bool + var v _ValueTerm + + /* no need to recurse into unreachable branches */ + if del[_Edge{bb, p}] { + continue + } + + /* find the branch value */ + for i, b := range sw.Br { + if b.To == p { + f, v = true, _ValueTerm(Int65i(int64(i))) + break + } + } + + /* it is not a direct successor, just pass all the facts down */ + if !f { + self.dfs(cfg, p, ps) + continue + } + + /* add the fact and recurse into the node */ + ps.checkpoint() + ps.define(sw.V, v, _R_eq) + self.dfs(cfg, p, ps) + ps.restore() + } } func (self BranchElim) Apply(cfg *CFG) { - self.dfs(cfg, cfg.Root, new(_Proof)) - cfg.Rebuild() + self.dfs(cfg, cfg.Root, new(_Proof)) + cfg.Rebuild() } diff --git a/internal/atm/ssa/pass_compact_amd64.go b/internal/atm/ssa/pass_compact_amd64.go index e9038d4..19bca22 100644 --- a/internal/atm/ssa/pass_compact_amd64.go +++ b/internal/atm/ssa/pass_compact_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,256 +17,277 @@ package ssa import ( - `github.com/cloudwego/frugal/internal/atm/abi` + "github.com/cloudwego/frugal/internal/atm/abi" ) type _MemAddr struct { - m Mem - n uint8 + m Mem + n uint8 } func memaddr(m Mem, n uint8) _MemAddr { - return _MemAddr { m, n } + return _MemAddr{m, n} } type _MemTree struct { - next *_MemTree - mems map[Mem]Reg + next *_MemTree + mems map[Mem]Reg } func (self *_MemTree) add(mm Mem, rr Reg) { - if _, ok := self.mems[mm]; ok { - panic("memtree: memory operand conflict: " + mm.String()) - } else { - self.mems[mm] = rr - } + if _, ok := self.mems[mm]; ok { + panic("memtree: memory operand conflict: " + mm.String()) + } else { + self.mems[mm] = rr + } } func (self *_MemTree) find(mm Mem) (Reg, bool) { - if rr, ok := self.mems[mm]; ok { - return rr, true - } else if self.next == nil { - return 0, false - } else { - return self.next.find(mm) - } + if rr, ok := self.mems[mm]; ok { + return rr, true + } else if self.next == nil { + return 0, false + } else { + return self.next.find(mm) + } } func (self *_MemTree) derive() *_MemTree { - return &_MemTree { - next: self, - mems: make(map[Mem]Reg), - } + return &_MemTree{ + next: self, + mems: make(map[Mem]Reg), + } } // Compaction is like Fusion, but it performs the reverse action to reduce redundant operations. type Compaction struct{} func (self Compaction) dfs(cfg *CFG, bb *BasicBlock, mems *_MemTree, next *bool) { - id := bb.Id - vv := make(map[_MemAddr]Reg) - - /* check every instructions, reuse memory addresses as much as possible */ - for i, v := range bb.Ins { - switch p := v.(type) { - default: { - break - } - - /* load effective address */ - case *IrAMD64_LEA: { - mems.add(p.M, p.R) - } - - /* lea {mem}, %r0; movx {mem}, %r1 --> lea {mem}, %r0; movx (%r0), %r1 */ - case *IrAMD64_MOV_load: { - if m, ok := mems.find(p.M); ok { - p.M, *next = Ptr(m, 0), true - } else if _, ok = vv[memaddr(p.M, p.N)]; !ok { - vv[memaddr(p.M, p.N)] = p.R - } - } - - /* movx {mem}, %r0; movbex {mem}, %r1 --> movx {mem}, %r0; bswapx %r0, %r1 - * leax {mem}, %r0; movbex {mem}, %r1 --> leax {mem}, %r0; movbex (%r0), %r1 */ - case *IrAMD64_MOV_load_be: { - if m, ok := mems.find(p.M); ok { - p.M, *next = Ptr(m, 0), true - } else if r, ok := vv[memaddr(p.M, p.N)]; ok { - bb.Ins[i], *next = &IrAMD64_BSWAP { R: p.R, V: r, N: p.N }, true - } - } - - /* lea {mem}, %r0; movx %r1, {mem} --> lea {mem}, %r0; movx %r1, (%r0) */ - case *IrAMD64_MOV_store_r: { - if m, ok := mems.find(p.M); ok { - p.M, *next = Ptr(m, 0), true - delete(vv, memaddr(p.M, p.N)) - } - } - - /* lea {mem}, %r0; movx {imm}, {mem} --> lea {mem}, %r0; movx {imm}, (%r0) */ - case *IrAMD64_MOV_store_i: { - if m, ok := mems.find(p.M); ok { - p.M, *next = Ptr(m, 0), true - delete(vv, memaddr(p.M, p.N)) - } - } - - /* lea {mem}, %r0; movx {ptr}, {mem} --> lea {mem}, %r0; movx {ptr}, (%r0) */ - case *IrAMD64_MOV_store_p: { - if m, ok := mems.find(p.M); ok { - p.M, *next = Ptr(m, 0), true - delete(vv, memaddr(p.M, abi.PtrSize)) - } - } - - /* lea {mem}, %r0; movbex %r1, {mem} --> lea {mem}, %r0; movbex %r1, (%r0) */ - case *IrAMD64_MOV_store_be: { - if m, ok := mems.find(p.M); ok { - p.M, *next = Ptr(m, 0), true - delete(vv, memaddr(p.M, p.N)) - } - } - - /* movx {mem}, %r0; cmpx %r1, {mem} --> movx {mem}, %r0; cmpx %r1, %r0 - * leaq {mem}, %r0; cmpx %r1, {mem} --> leaq {mem}, %r0; cmpx %r1, (%r0) */ - case *IrAMD64_CMPQ_rm: { - if m, ok := mems.find(p.Y); ok { - p.Y, *next = Ptr(m, 0), true - } else if v, ok := vv[memaddr(p.Y, p.N)]; ok { - bb.Ins[i], *next = &IrAMD64_CMPQ_rr { X: p.X, Y: v, Op: p.Op }, true - } - } - - /* movx {mem}, %r0; cmpx {mem}, %r1 --> movx {mem}, %r0; cmpx %r0, %r1 - * leaq {mem}, %r0; cmpx {mem}, %r1 --> leaq {mem}, %r0; cmpx (%r0), %r1 */ - case *IrAMD64_CMPQ_mr: { - if m, ok := mems.find(p.X); ok { - p.X, *next = Ptr(m, 0), true - } else if r, ok := vv[memaddr(p.X, p.N)]; ok { - bb.Ins[i], *next = &IrAMD64_CMPQ_rr { X: r, Y: p.Y, Op: p.Op }, true - } - } - - /* movx {mem}, %r0; cmpx {mem}, {imm} --> movx {mem}, %r0; cmpx %r0, {imm} - * leaq {mem}, %r0; cmpx {mem}, {imm} --> leaq {mem}, %r0; cmpx (%r0), {imm} */ - case *IrAMD64_CMPQ_mi: { - if m, ok := mems.find(p.X); ok { - p.X, *next = Ptr(m, 0), true - } else if r, ok := vv[memaddr(p.X, p.N)]; ok { - bb.Ins[i], *next = &IrAMD64_CMPQ_ri { X: r, Y: p.Y, Op: p.Op }, true - } - } - - /* movx {mem}, %r0; cmpx {mem}, {ptr} --> movx {mem}, %r0; cmpx %r0, {ptr} - * leaq {mem}, %r0; cmpx {mem}, {ptr} --> leaq {mem}, %r0; cmpx (%r0), {ptr} */ - case *IrAMD64_CMPQ_mp: { - if m, ok := mems.find(p.X); ok { - p.X, *next = Ptr(m, 0), true - } else if r, ok := vv[memaddr(p.X, abi.PtrSize)]; ok { - bb.Ins[i], *next = &IrAMD64_CMPQ_rp { X: r, Y: p.Y, Op: p.Op }, true - } - } - - /* movx {mem}, %r0; cmpx {imm}, {mem} --> movx {mem}, %r0; cmpx {imm}, %r0 - * leaq {mem}, %r0; cmpx {imm}, {mem} --> leaq {mem}, %r0; cmpx {imm}, (%r0) */ - case *IrAMD64_CMPQ_im: { - if m, ok := mems.find(p.Y); ok { - p.Y, *next = Ptr(m, 0), true - } else if r, ok := vv[memaddr(p.Y, p.N)]; ok { - bb.Ins[i], *next = &IrAMD64_CMPQ_ir { X: p.X, Y: r, Op: p.Op }, true - } - } - - /* movx {mem}, %r0; cmpx {ptr}, {mem} --> movx {mem}, %r0; cmpx %r0, {ptr} - * leaq {mem}, %r0; cmpx {ptr}, {mem} --> leaq {mem}, %r0; cmpx (%r0), {ptr} */ - case *IrAMD64_CMPQ_pm: { - if m, ok := mems.find(p.Y); ok { - p.Y, *next = Ptr(m, 0), true - } else if r, ok := vv[memaddr(p.Y, abi.PtrSize)]; ok { - bb.Ins[i], *next = &IrAMD64_CMPQ_pr { X: p.X, Y: r, Op: p.Op }, true - } - } - } - } - - /* check for terminators */ - switch p := bb.Term.(type) { - default: { - break - } - - /* movx {mem}, %r0; cmpx %r1, {mem}; jcc --> movx {mem}, %r0; cmpx %r1, %r0; jcc - * leaq {mem}, %r0; cmpx %r1, {mem}; jcc --> leaq {mem}, %r0; cmpx %r1, (%r0); jcc */ - case *IrAMD64_Jcc_rm: { - if m, ok := mems.find(p.Y); ok { - p.Y, *next = Ptr(m, 0), true - } else if r, ok := vv[memaddr(p.Y, p.N)]; ok { - bb.Term, *next = &IrAMD64_Jcc_rr { X: p.X, Y: r, To: p.To, Ln: p.Ln, Op: p.Op }, true - } - } - - /* movx {mem}, %r0; cmpx {mem}, %r1; jcc --> movx {mem}, %r0; cmpx %r0, %r1; jcc - * leaq {mem}, %r0; cmpx {mem}, %r1; jcc --> leaq {mem}, %r0; cmpx (%r0), %r1; jcc */ - case *IrAMD64_Jcc_mr: { - if m, ok := mems.find(p.X); ok { - p.X, *next = Ptr(m, 0), true - } else if r, ok := vv[memaddr(p.X, p.N)]; ok { - bb.Term, *next = &IrAMD64_Jcc_rr { X: r, Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op }, true - } - } - - /* movx {mem}, %r0; cmpx {mem}, {imm}; jcc --> movx {mem}, %r0; cmpx %r0, {imm}; jcc - * leaq {mem}, %r0; cmpx {mem}, {imm}; jcc --> leaq {mem}, %r0; cmpx (%r0), {imm}; jcc */ - case *IrAMD64_Jcc_mi: { - if m, ok := mems.find(p.X); ok { - p.X, *next = Ptr(m, 0), true - } else if r, ok := vv[memaddr(p.X, p.N)]; ok { - bb.Term, *next = &IrAMD64_Jcc_ri { X: r, Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op }, true - } - } - - /* movx {mem}, %r0; cmpx {mem}, {ptr}; jcc --> movx {mem}, %r0; cmpx %r0, {ptr}; jcc - * leaq {mem}, %r0; cmpx {mem}, {ptr}; jcc --> leaq {mem}, %r0; cmpx (%r0), {ptr}; jcc */ - case *IrAMD64_Jcc_mp: { - if m, ok := mems.find(p.X); ok { - p.X, *next = Ptr(m, 0), true - } else if r, ok := vv[memaddr(p.X, abi.PtrSize)]; ok { - bb.Term, *next = &IrAMD64_Jcc_rp { X: r, Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op }, true - } - } - - /* movx {mem}, %r0; cmpx {imm}, {mem}; jcc --> movx {mem}, %r0; cmpx {imm}, %r0; jcc - * leaq {mem}, %r0; cmpx {imm}, {mem}; jcc --> leaq {mem}, %r0; cmpx {imm}, (%r0); jcc */ - case *IrAMD64_Jcc_im: { - if m, ok := mems.find(p.Y); ok { - p.Y, *next = Ptr(m, 0), true - } else if r, ok := vv[memaddr(p.Y, p.N)]; ok { - bb.Term, *next = &IrAMD64_Jcc_ir { X: p.X, Y: r, To: p.To, Ln: p.Ln, Op: p.Op }, true - } - } - - /* movx {mem}, %r0; cmpx {ptr}, {mem}; jcc --> movx {mem}, %r0; cmpx %r0, {ptr}; jcc - * leaq {mem}, %r0; cmpx {ptr}, {mem}; jcc --> leaq {mem}, %r0; cmpx (%r0), {ptr}; jcc */ - case *IrAMD64_Jcc_pm: { - if m, ok := mems.find(p.Y); ok { - p.Y, *next = Ptr(m, 0), true - } else if r, ok := vv[memaddr(p.Y, abi.PtrSize)]; ok { - bb.Term, *next = &IrAMD64_Jcc_pr { X: p.X, Y: r, To: p.To, Ln: p.Ln, Op: p.Op }, true - } - } - } - - /* DFS the dominator tree */ - for _, p := range cfg.DominatorOf[id] { - self.dfs(cfg, p, mems.derive(), next) - } + id := bb.Id + vv := make(map[_MemAddr]Reg) + + /* check every instructions, reuse memory addresses as much as possible */ + for i, v := range bb.Ins { + switch p := v.(type) { + default: + { + break + } + + /* load effective address */ + case *IrAMD64_LEA: + { + mems.add(p.M, p.R) + } + + /* lea {mem}, %r0; movx {mem}, %r1 --> lea {mem}, %r0; movx (%r0), %r1 */ + case *IrAMD64_MOV_load: + { + if m, ok := mems.find(p.M); ok { + p.M, *next = Ptr(m, 0), true + } else if _, ok = vv[memaddr(p.M, p.N)]; !ok { + vv[memaddr(p.M, p.N)] = p.R + } + } + + /* movx {mem}, %r0; movbex {mem}, %r1 --> movx {mem}, %r0; bswapx %r0, %r1 + * leax {mem}, %r0; movbex {mem}, %r1 --> leax {mem}, %r0; movbex (%r0), %r1 */ + case *IrAMD64_MOV_load_be: + { + if m, ok := mems.find(p.M); ok { + p.M, *next = Ptr(m, 0), true + } else if r, ok := vv[memaddr(p.M, p.N)]; ok { + bb.Ins[i], *next = &IrAMD64_BSWAP{R: p.R, V: r, N: p.N}, true + } + } + + /* lea {mem}, %r0; movx %r1, {mem} --> lea {mem}, %r0; movx %r1, (%r0) */ + case *IrAMD64_MOV_store_r: + { + if m, ok := mems.find(p.M); ok { + p.M, *next = Ptr(m, 0), true + delete(vv, memaddr(p.M, p.N)) + } + } + + /* lea {mem}, %r0; movx {imm}, {mem} --> lea {mem}, %r0; movx {imm}, (%r0) */ + case *IrAMD64_MOV_store_i: + { + if m, ok := mems.find(p.M); ok { + p.M, *next = Ptr(m, 0), true + delete(vv, memaddr(p.M, p.N)) + } + } + + /* lea {mem}, %r0; movx {ptr}, {mem} --> lea {mem}, %r0; movx {ptr}, (%r0) */ + case *IrAMD64_MOV_store_p: + { + if m, ok := mems.find(p.M); ok { + p.M, *next = Ptr(m, 0), true + delete(vv, memaddr(p.M, abi.PtrSize)) + } + } + + /* lea {mem}, %r0; movbex %r1, {mem} --> lea {mem}, %r0; movbex %r1, (%r0) */ + case *IrAMD64_MOV_store_be: + { + if m, ok := mems.find(p.M); ok { + p.M, *next = Ptr(m, 0), true + delete(vv, memaddr(p.M, p.N)) + } + } + + /* movx {mem}, %r0; cmpx %r1, {mem} --> movx {mem}, %r0; cmpx %r1, %r0 + * leaq {mem}, %r0; cmpx %r1, {mem} --> leaq {mem}, %r0; cmpx %r1, (%r0) */ + case *IrAMD64_CMPQ_rm: + { + if m, ok := mems.find(p.Y); ok { + p.Y, *next = Ptr(m, 0), true + } else if v, ok := vv[memaddr(p.Y, p.N)]; ok { + bb.Ins[i], *next = &IrAMD64_CMPQ_rr{X: p.X, Y: v, Op: p.Op}, true + } + } + + /* movx {mem}, %r0; cmpx {mem}, %r1 --> movx {mem}, %r0; cmpx %r0, %r1 + * leaq {mem}, %r0; cmpx {mem}, %r1 --> leaq {mem}, %r0; cmpx (%r0), %r1 */ + case *IrAMD64_CMPQ_mr: + { + if m, ok := mems.find(p.X); ok { + p.X, *next = Ptr(m, 0), true + } else if r, ok := vv[memaddr(p.X, p.N)]; ok { + bb.Ins[i], *next = &IrAMD64_CMPQ_rr{X: r, Y: p.Y, Op: p.Op}, true + } + } + + /* movx {mem}, %r0; cmpx {mem}, {imm} --> movx {mem}, %r0; cmpx %r0, {imm} + * leaq {mem}, %r0; cmpx {mem}, {imm} --> leaq {mem}, %r0; cmpx (%r0), {imm} */ + case *IrAMD64_CMPQ_mi: + { + if m, ok := mems.find(p.X); ok { + p.X, *next = Ptr(m, 0), true + } else if r, ok := vv[memaddr(p.X, p.N)]; ok { + bb.Ins[i], *next = &IrAMD64_CMPQ_ri{X: r, Y: p.Y, Op: p.Op}, true + } + } + + /* movx {mem}, %r0; cmpx {mem}, {ptr} --> movx {mem}, %r0; cmpx %r0, {ptr} + * leaq {mem}, %r0; cmpx {mem}, {ptr} --> leaq {mem}, %r0; cmpx (%r0), {ptr} */ + case *IrAMD64_CMPQ_mp: + { + if m, ok := mems.find(p.X); ok { + p.X, *next = Ptr(m, 0), true + } else if r, ok := vv[memaddr(p.X, abi.PtrSize)]; ok { + bb.Ins[i], *next = &IrAMD64_CMPQ_rp{X: r, Y: p.Y, Op: p.Op}, true + } + } + + /* movx {mem}, %r0; cmpx {imm}, {mem} --> movx {mem}, %r0; cmpx {imm}, %r0 + * leaq {mem}, %r0; cmpx {imm}, {mem} --> leaq {mem}, %r0; cmpx {imm}, (%r0) */ + case *IrAMD64_CMPQ_im: + { + if m, ok := mems.find(p.Y); ok { + p.Y, *next = Ptr(m, 0), true + } else if r, ok := vv[memaddr(p.Y, p.N)]; ok { + bb.Ins[i], *next = &IrAMD64_CMPQ_ir{X: p.X, Y: r, Op: p.Op}, true + } + } + + /* movx {mem}, %r0; cmpx {ptr}, {mem} --> movx {mem}, %r0; cmpx %r0, {ptr} + * leaq {mem}, %r0; cmpx {ptr}, {mem} --> leaq {mem}, %r0; cmpx (%r0), {ptr} */ + case *IrAMD64_CMPQ_pm: + { + if m, ok := mems.find(p.Y); ok { + p.Y, *next = Ptr(m, 0), true + } else if r, ok := vv[memaddr(p.Y, abi.PtrSize)]; ok { + bb.Ins[i], *next = &IrAMD64_CMPQ_pr{X: p.X, Y: r, Op: p.Op}, true + } + } + } + } + + /* check for terminators */ + switch p := bb.Term.(type) { + default: + { + break + } + + /* movx {mem}, %r0; cmpx %r1, {mem}; jcc --> movx {mem}, %r0; cmpx %r1, %r0; jcc + * leaq {mem}, %r0; cmpx %r1, {mem}; jcc --> leaq {mem}, %r0; cmpx %r1, (%r0); jcc */ + case *IrAMD64_Jcc_rm: + { + if m, ok := mems.find(p.Y); ok { + p.Y, *next = Ptr(m, 0), true + } else if r, ok := vv[memaddr(p.Y, p.N)]; ok { + bb.Term, *next = &IrAMD64_Jcc_rr{X: p.X, Y: r, To: p.To, Ln: p.Ln, Op: p.Op}, true + } + } + + /* movx {mem}, %r0; cmpx {mem}, %r1; jcc --> movx {mem}, %r0; cmpx %r0, %r1; jcc + * leaq {mem}, %r0; cmpx {mem}, %r1; jcc --> leaq {mem}, %r0; cmpx (%r0), %r1; jcc */ + case *IrAMD64_Jcc_mr: + { + if m, ok := mems.find(p.X); ok { + p.X, *next = Ptr(m, 0), true + } else if r, ok := vv[memaddr(p.X, p.N)]; ok { + bb.Term, *next = &IrAMD64_Jcc_rr{X: r, Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op}, true + } + } + + /* movx {mem}, %r0; cmpx {mem}, {imm}; jcc --> movx {mem}, %r0; cmpx %r0, {imm}; jcc + * leaq {mem}, %r0; cmpx {mem}, {imm}; jcc --> leaq {mem}, %r0; cmpx (%r0), {imm}; jcc */ + case *IrAMD64_Jcc_mi: + { + if m, ok := mems.find(p.X); ok { + p.X, *next = Ptr(m, 0), true + } else if r, ok := vv[memaddr(p.X, p.N)]; ok { + bb.Term, *next = &IrAMD64_Jcc_ri{X: r, Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op}, true + } + } + + /* movx {mem}, %r0; cmpx {mem}, {ptr}; jcc --> movx {mem}, %r0; cmpx %r0, {ptr}; jcc + * leaq {mem}, %r0; cmpx {mem}, {ptr}; jcc --> leaq {mem}, %r0; cmpx (%r0), {ptr}; jcc */ + case *IrAMD64_Jcc_mp: + { + if m, ok := mems.find(p.X); ok { + p.X, *next = Ptr(m, 0), true + } else if r, ok := vv[memaddr(p.X, abi.PtrSize)]; ok { + bb.Term, *next = &IrAMD64_Jcc_rp{X: r, Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op}, true + } + } + + /* movx {mem}, %r0; cmpx {imm}, {mem}; jcc --> movx {mem}, %r0; cmpx {imm}, %r0; jcc + * leaq {mem}, %r0; cmpx {imm}, {mem}; jcc --> leaq {mem}, %r0; cmpx {imm}, (%r0); jcc */ + case *IrAMD64_Jcc_im: + { + if m, ok := mems.find(p.Y); ok { + p.Y, *next = Ptr(m, 0), true + } else if r, ok := vv[memaddr(p.Y, p.N)]; ok { + bb.Term, *next = &IrAMD64_Jcc_ir{X: p.X, Y: r, To: p.To, Ln: p.Ln, Op: p.Op}, true + } + } + + /* movx {mem}, %r0; cmpx {ptr}, {mem}; jcc --> movx {mem}, %r0; cmpx %r0, {ptr}; jcc + * leaq {mem}, %r0; cmpx {ptr}, {mem}; jcc --> leaq {mem}, %r0; cmpx (%r0), {ptr}; jcc */ + case *IrAMD64_Jcc_pm: + { + if m, ok := mems.find(p.Y); ok { + p.Y, *next = Ptr(m, 0), true + } else if r, ok := vv[memaddr(p.Y, abi.PtrSize)]; ok { + bb.Term, *next = &IrAMD64_Jcc_pr{X: p.X, Y: r, To: p.To, Ln: p.Ln, Op: p.Op}, true + } + } + } + + /* DFS the dominator tree */ + for _, p := range cfg.DominatorOf[id] { + self.dfs(cfg, p, mems.derive(), next) + } } func (self Compaction) Apply(cfg *CFG) { - for next := true; next; { - next = false - self.dfs(cfg, cfg.Root, (*_MemTree).derive(nil), &next) - } + for next := true; next; { + next = false + self.dfs(cfg, cfg.Root, (*_MemTree).derive(nil), &next) + } } diff --git a/internal/atm/ssa/pass_comsubexpr.go b/internal/atm/ssa/pass_comsubexpr.go index 724bf10..5178c3a 100644 --- a/internal/atm/ssa/pass_comsubexpr.go +++ b/internal/atm/ssa/pass_comsubexpr.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,159 +17,168 @@ package ssa import ( - `fmt` + "fmt" ) type _Vid interface { - IrDefinitions - vid() []string + IrDefinitions + vid() []string } func (self *IrLoadArg) vid() []string { - return []string { - fmt.Sprintf("#%d", self.I), - } + return []string{ + fmt.Sprintf("#%d", self.I), + } } func (self *IrConstInt) vid() []string { - return []string { - fmt.Sprintf("$%d", self.V), - } + return []string{ + fmt.Sprintf("$%d", self.V), + } } func (self *IrConstPtr) vid() []string { - return []string { - fmt.Sprintf("$%p", self.P), - } + return []string{ + fmt.Sprintf("$%p", self.P), + } } func (self *IrLEA) vid() []string { - return []string { - fmt.Sprintf("(& %s %s)", self.Mem, self.Off), - } + return []string{ + fmt.Sprintf("(& %s %s)", self.Mem, self.Off), + } } func (self *IrUnaryExpr) vid() []string { - return []string { - fmt.Sprintf("(%s %s)", self.Op, self.V), - } + return []string{ + fmt.Sprintf("(%s %s)", self.Op, self.V), + } } func (self *IrBinaryExpr) vid() []string { - x := self.X - y := self.Y - - /* commutative operations, sort the operands */ - switch self.Op { - case IrOpAdd : fallthrough - case IrOpMul : fallthrough - case IrOpAnd : fallthrough - case IrOpOr : fallthrough - case IrOpXor : fallthrough - case IrCmpEq : fallthrough - case IrCmpNe : if x > y { x, y = y, x } - } - - /* build the value ID */ - return []string { - fmt.Sprintf("(%s %s %s)", self.Op, x, y), - } + x := self.X + y := self.Y + + /* commutative operations, sort the operands */ + switch self.Op { + case IrOpAdd: + fallthrough + case IrOpMul: + fallthrough + case IrOpAnd: + fallthrough + case IrOpOr: + fallthrough + case IrOpXor: + fallthrough + case IrCmpEq: + fallthrough + case IrCmpNe: + if x > y { + x, y = y, x + } + } + + /* build the value ID */ + return []string{ + fmt.Sprintf("(%s %s %s)", self.Op, x, y), + } } func (self *IrBitTestSet) vid() []string { - return []string { - fmt.Sprintf("(&# %s %s)", self.X, self.Y), - fmt.Sprintf("(|# %s %s)", self.X, self.Y), - } + return []string{ + fmt.Sprintf("(&# %s %s)", self.X, self.Y), + fmt.Sprintf("(|# %s %s)", self.X, self.Y), + } } type _VidMap struct { - p *_VidMap - m map[string]Reg + p *_VidMap + m map[string]Reg } func (self *_VidMap) derive() *_VidMap { - return &_VidMap { - p: self, - m: make(map[string]Reg), - } + return &_VidMap{ + p: self, + m: make(map[string]Reg), + } } func (self *_VidMap) lookup(vid string) (Reg, bool) { - if r, ok := self.m[vid]; ok { - return r, true - } else if self.p == nil { - return 0, false - } else { - return self.p.lookup(vid) - } + if r, ok := self.m[vid]; ok { + return r, true + } else if self.p == nil { + return 0, false + } else { + return self.p.lookup(vid) + } } func (self *_VidMap) define(vid string, reg Reg) { - self.m[vid] = reg + self.m[vid] = reg } -// CSE performs the Common Sub-expression Elimintation optimization. +// CSE performs the Common Sub-expression Elimination optimization. type CSE struct{} func (self CSE) dfs(cfg *CFG, bb *BasicBlock, vm *_VidMap) { - ins := bb.Ins - vals := vm.derive() - - /* scan every instructions */ - for i, v := range ins { - var r Reg - var d _Vid - var ok bool - - /* check if the instruction have VIDs */ - if d, ok = v.(_Vid); !ok { - continue - } - - /* calculate the VIDs */ - repc := i - vids := d.vid() - defs := d.Definitions() - - /* replace each VID with a copy instruction */ - for j, vid := range vids { - s := defs[j] - r, ok = vals.lookup(vid) - - /* skip zero registers */ - if s.Kind() == K_zero { - continue - } - - /* add to definations if not found */ - if !ok { - vals.define(vid, *s) - continue - } - - /* allocate one slot for the new instruction */ - repc++ - bb.Ins = append(bb.Ins, nil) - copy(bb.Ins[repc + 1:], bb.Ins[repc:]) - - /* insert a new copy instruction */ - bb.Ins[repc] = IrCopy(*s, r) - *s = s.Zero() - } - - /* all the definations are been replaced */ - if repc == i + len(defs) { - bb.Ins = append(bb.Ins[:i], bb.Ins[i + 1:]...) - } - } - - /* DFS the dominator tree */ - for _, v := range cfg.DominatorOf[bb.Id] { - self.dfs(cfg, v, vals) - } + ins := bb.Ins + vals := vm.derive() + + /* scan every instruction */ + for i, v := range ins { + var r Reg + var d _Vid + var ok bool + + /* check if the instruction have VIDs */ + if d, ok = v.(_Vid); !ok { + continue + } + + /* calculate the VIDs */ + repc := i + vids := d.vid() + defs := d.Definitions() + + /* replace each VID with a copy instruction */ + for j, vid := range vids { + s := defs[j] + r, ok = vals.lookup(vid) + + /* skip zero registers */ + if s.Kind() == K_zero { + continue + } + + /* add to definitions if not found */ + if !ok { + vals.define(vid, *s) + continue + } + + /* allocate one slot for the new instruction */ + repc++ + bb.Ins = append(bb.Ins, nil) + copy(bb.Ins[repc+1:], bb.Ins[repc:]) + + /* insert a new copy instruction */ + bb.Ins[repc] = IrCopy(*s, r) + *s = s.Zero() + } + + /* all the definitions are been replaced */ + if repc == i+len(defs) { + bb.Ins = append(bb.Ins[:i], bb.Ins[i+1:]...) + } + } + + /* DFS the dominator tree */ + for _, v := range cfg.DominatorOf[bb.Id] { + self.dfs(cfg, v, vals) + } } func (self CSE) Apply(cfg *CFG) { - self.dfs(cfg, cfg.Root, nil) + self.dfs(cfg, cfg.Root, nil) } diff --git a/internal/atm/ssa/pass_constprop.go b/internal/atm/ssa/pass_constprop.go index 22b4e0f..8dcf6b9 100644 --- a/internal/atm/ssa/pass_constprop.go +++ b/internal/atm/ssa/pass_constprop.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,264 +17,316 @@ package ssa import ( - `fmt` - `math/bits` - `unsafe` + "fmt" + "math/bits" + "unsafe" - `github.com/cloudwego/frugal/internal/atm/abi` + "github.com/cloudwego/frugal/internal/atm/abi" ) // ConstProp propagates constant through the expression tree. type ConstProp struct{} func (ConstProp) unary(v int64, op IrUnaryOp) int64 { - switch op { - case IrOpNegate : return -v - case IrOpSwap16 : return int64(bits.ReverseBytes16(uint16(v))) - case IrOpSwap32 : return int64(bits.ReverseBytes32(uint32(v))) - case IrOpSwap64 : return int64(bits.ReverseBytes64(uint64(v))) - case IrOpSx32to64 : return int64(int32(v)) - default : panic(fmt.Sprintf("constprop: invalid unary operator: %d", op)) - } + switch op { + case IrOpNegate: + return -v + case IrOpSwap16: + return int64(bits.ReverseBytes16(uint16(v))) + case IrOpSwap32: + return int64(bits.ReverseBytes32(uint32(v))) + case IrOpSwap64: + return int64(bits.ReverseBytes64(uint64(v))) + case IrOpSx32to64: + return int64(int32(v)) + default: + panic(fmt.Sprintf("constprop: invalid unary operator: %d", op)) + } } func (ConstProp) binary(x int64, y int64, op IrBinaryOp) int64 { - switch op { - case IrOpAdd : return x + y - case IrOpSub : return x - y - case IrOpMul : return x * y - case IrOpAnd : return x & y - case IrOpOr : return x | y - case IrOpXor : return x ^ y - case IrOpShr : return x >> y - case IrCmpEq : if x == y { return 1 } else { return 0 } - case IrCmpNe : if x != y { return 1 } else { return 0 } - case IrCmpLt : if x < y { return 1 } else { return 0 } - case IrCmpLtu : if uint64(x) < uint64(y) { return 1 } else { return 0 } - case IrCmpGeu : if uint64(x) >= uint64(y) { return 1 } else { return 0 } - default : panic(fmt.Sprintf("constprop: invalid binary operator: %d", op)) - } + switch op { + case IrOpAdd: + return x + y + case IrOpSub: + return x - y + case IrOpMul: + return x * y + case IrOpAnd: + return x & y + case IrOpOr: + return x | y + case IrOpXor: + return x ^ y + case IrOpShr: + return x >> y + case IrCmpEq: + if x == y { + return 1 + } else { + return 0 + } + case IrCmpNe: + if x != y { + return 1 + } else { + return 0 + } + case IrCmpLt: + if x < y { + return 1 + } else { + return 0 + } + case IrCmpLtu: + if uint64(x) < uint64(y) { + return 1 + } else { + return 0 + } + case IrCmpGeu: + if uint64(x) >= uint64(y) { + return 1 + } else { + return 0 + } + default: + panic(fmt.Sprintf("constprop: invalid binary operator: %d", op)) + } } func (ConstProp) testandset(x int64, y int64) (int64, int64) { - if bv := int64(1 << y); x & bv == 0 { - return 0, x | bv - } else { - return 1, x | bv - } + if bv := int64(1 << y); x&bv == 0 { + return 0, x | bv + } else { + return 1, x | bv + } } func (self ConstProp) Apply(cfg *CFG) { - done := false - consts := make(map[Reg]_ConstData) - - /* constant zero registers */ - consts[Rz] = constint(0) - consts[Pn] = constptr(nil, Const) - - /* const adder */ - addconst := func(r Reg, v _ConstData) { - if _, ok := consts[r]; !ok { - done = false - consts[r] = v - } - } - - /* evaluate const expression until no modifications were made */ - for !done { - done = true - for _, bb := range cfg.PostOrder().Reversed() { - phi := make([]*IrPhi, 0, len(bb.Phi)) - ins := make([]IrNode, 0, len(bb.Ins)) - isconst := false - - /* check every Phi node */ - for _, p := range bb.Phi { - var first bool - var cdata _ConstData - - /* assume it's a const */ - first = true - isconst = true - - /* a Phi node is a const iff all it's arguments are the same const */ - for _, r := range p.V { - if *r != p.R { - if cc, ok := consts[*r]; !ok { - isconst = false - break - } else if first { - cdata = cc - first = false - } else if cdata != cc { - isconst = false - break - } - } - } - - /* not constant, keep it as is */ - if !isconst { - phi = append(phi, p) - continue - } - - /* registers declared by Phi nodes can never be zero registers */ - if p.R.Kind() == K_zero { - panic("constprop: assignment to zero registers in Phi node: " + p.String()) - } - - /* replace the Phi node with a Const node */ - if cdata.i { - ins = append(ins, &IrConstInt { R: p.R, V: cdata.v }) - } else { - ins = append(ins, &IrConstPtr { R: p.R, P: cdata.p, M: cdata.c }) - } - - /* mark as constant */ - done = false - consts[p.R] = cdata - } - - /* check every instructions */ - for _, v := range bb.Ins { - switch p := v.(type) { - default: { - ins = append(ins, p) - } - - /* value loads */ - case *IrLoad: { - var ok bool - var iv int64 - var cc _ConstData - var pv unsafe.Pointer - - /* must be constant memory address */ - if cc, ok = consts[p.Mem]; !ok { - ins = append(ins, p) - break - } - - /* address must be a pointer */ - if cc.i { - panic(fmt.Sprintf("constprop: value load on integer value %#x: %s", cc.v, p)) - } - - /* do not load a volatile pointer */ - if cc.c != Const { - ins = append(ins, p) - break - } - - /* constant pointer */ - if p.R.Ptr() { - if p.Size != abi.PtrSize { - panic("constprop: pointer load of invalid size: " + p.String()) - } else { - pv = *(*unsafe.Pointer)(cc.p) - ins = append(ins, &IrConstPtr { R: p.R, P: pv, M: Volatile }) - addconst(p.R, constptr(pv, Volatile)) - break - } - } - - /* read the integer */ - switch p.Size { - case 1 : iv = int64(*(*uint8)(cc.p)) - case 2 : iv = int64(*(*uint16)(cc.p)) - case 4 : iv = int64(*(*uint32)(cc.p)) - case 8 : iv = int64(*(*uint64)(cc.p)) - default : panic("constprop: integer load of invalid size: " + p.String()) - } - - /* replace the instruction */ - ins = append(ins, &IrConstInt { R: p.R, V: iv }) - addconst(p.R, constint(iv)) - } - - /* integer constant */ - case *IrConstInt: { - ins = append(ins, p) - addconst(p.R, constint(p.V)) - } - - /* pointer constant */ - case *IrConstPtr: { - ins = append(ins, p) - addconst(p.R, constptr(p.P, p.M)) - } - - /* pointer arithmetics */ - case *IrLEA: { - if mem, ok := consts[p.Mem]; !ok { - ins = append(ins, p) - } else if off, ok := consts[p.Off]; !ok { - ins = append(ins, p) - } else if mem.i { - panic(fmt.Sprintf("constprop: pointer operation on integer value %#x: %s", mem.v, p)) - } else if !off.i { - panic(fmt.Sprintf("constprop: pointer operation with pointer offset %p: %s", off.p, p)) - } else { - r := addptr(mem.p, off.v) - ins = append(ins, &IrConstPtr { R: p.R, P: r, M: Volatile }) - addconst(p.R, constptr(r, mem.c)) - } - } - - /* unary expressions */ - case *IrUnaryExpr: { - if cc, ok := consts[p.V]; !ok { - ins = append(ins, p) - } else if !cc.i { - panic(fmt.Sprintf("constprop: integer operation on pointer value %p: %s", cc.p, p)) - } else { - r := self.unary(cc.v, p.Op) - ins = append(ins, &IrConstInt { R: p.R, V: r }) - addconst(p.R, constint(r)) - } - } - - /* binary expressions */ - case *IrBinaryExpr: { - if x, ok := consts[p.X]; !ok { - ins = append(ins, p) - } else if y, ok := consts[p.Y]; !ok { - ins = append(ins, p) - } else if !x.i { - panic(fmt.Sprintf("constprop: integer operation on pointer value %p: %s", x.p, p)) - } else if !y.i { - panic(fmt.Sprintf("constprop: integer operation on pointer value %p: %s", y.p, p)) - } else { - r := self.binary(x.v, y.v, p.Op) - ins = append(ins, &IrConstInt { R: p.R, V: r }) - addconst(p.R, constint(r)) - } - } - - /* bit test and set operation */ - case *IrBitTestSet: { - if x, ok := consts[p.X]; !ok { - ins = append(ins, p) - } else if y, ok := consts[p.Y]; !ok { - ins = append(ins, p) - } else if !x.i { - panic(fmt.Sprintf("constprop: integer operation on pointer value %p: %s", x.p, p)) - } else if !y.i { - panic(fmt.Sprintf("constprop: integer operation on pointer value %p: %s", y.p, p)) - } else { - t, s := self.testandset(x.v, y.v) - ins = append(ins, &IrConstInt { R: p.T, V: t }, &IrConstInt { R: p.S, V: s }) - addconst(p.T, constint(t)) - addconst(p.S, constint(s)) - } - } - } - } - - /* rebuild the basic block */ - bb.Phi = phi - bb.Ins = ins - } - } + done := false + consts := make(map[Reg]_ConstData) + + /* constant zero registers */ + consts[Rz] = constint(0) + consts[Pn] = constptr(nil, Const) + + /* const adder */ + addconst := func(r Reg, v _ConstData) { + if _, ok := consts[r]; !ok { + done = false + consts[r] = v + } + } + + /* evaluate const expression until no modifications were made */ + for !done { + done = true + for _, bb := range cfg.PostOrder().Reversed() { + phi := make([]*IrPhi, 0, len(bb.Phi)) + ins := make([]IrNode, 0, len(bb.Ins)) + isconst := false + + /* check every Phi node */ + for _, p := range bb.Phi { + var first bool + var cdata _ConstData + + /* assume it's a const */ + first = true + isconst = true + + /* a Phi node is a const iff all it's arguments are the same const */ + for _, r := range p.V { + if *r != p.R { + if cc, ok := consts[*r]; !ok { + isconst = false + break + } else if first { + cdata = cc + first = false + } else if cdata != cc { + isconst = false + break + } + } + } + + /* not constant, keep it as is */ + if !isconst { + phi = append(phi, p) + continue + } + + /* registers declared by Phi nodes can never be zero registers */ + if p.R.Kind() == K_zero { + panic("constprop: assignment to zero registers in Phi node: " + p.String()) + } + + /* replace the Phi node with a Const node */ + if cdata.i { + ins = append(ins, &IrConstInt{R: p.R, V: cdata.v}) + } else { + ins = append(ins, &IrConstPtr{R: p.R, P: cdata.p, M: cdata.c}) + } + + /* mark as constant */ + done = false + consts[p.R] = cdata + } + + /* check every instructions */ + for _, v := range bb.Ins { + switch p := v.(type) { + default: + { + ins = append(ins, p) + } + + /* value loads */ + case *IrLoad: + { + var ok bool + var iv int64 + var cc _ConstData + var pv unsafe.Pointer + + /* must be constant memory address */ + if cc, ok = consts[p.Mem]; !ok { + ins = append(ins, p) + break + } + + /* address must be a pointer */ + if cc.i { + panic(fmt.Sprintf("constprop: value load on integer value %#x: %s", cc.v, p)) + } + + /* do not load a volatile pointer */ + if cc.c != Const { + ins = append(ins, p) + break + } + + /* constant pointer */ + if p.R.Ptr() { + if p.Size != abi.PtrSize { + panic("constprop: pointer load of invalid size: " + p.String()) + } else { + pv = *(*unsafe.Pointer)(cc.p) + ins = append(ins, &IrConstPtr{R: p.R, P: pv, M: Volatile}) + addconst(p.R, constptr(pv, Volatile)) + break + } + } + + /* read the integer */ + switch p.Size { + case 1: + iv = int64(*(*uint8)(cc.p)) + case 2: + iv = int64(*(*uint16)(cc.p)) + case 4: + iv = int64(*(*uint32)(cc.p)) + case 8: + iv = int64(*(*uint64)(cc.p)) + default: + panic("constprop: integer load of invalid size: " + p.String()) + } + + /* replace the instruction */ + ins = append(ins, &IrConstInt{R: p.R, V: iv}) + addconst(p.R, constint(iv)) + } + + /* integer constant */ + case *IrConstInt: + { + ins = append(ins, p) + addconst(p.R, constint(p.V)) + } + + /* pointer constant */ + case *IrConstPtr: + { + ins = append(ins, p) + addconst(p.R, constptr(p.P, p.M)) + } + + /* pointer arithmetics */ + case *IrLEA: + { + if mem, ok := consts[p.Mem]; !ok { + ins = append(ins, p) + } else if off, ok := consts[p.Off]; !ok { + ins = append(ins, p) + } else if mem.i { + panic(fmt.Sprintf("constprop: pointer operation on integer value %#x: %s", mem.v, p)) + } else if !off.i { + panic(fmt.Sprintf("constprop: pointer operation with pointer offset %p: %s", off.p, p)) + } else { + r := addptr(mem.p, off.v) + ins = append(ins, &IrConstPtr{R: p.R, P: r, M: Volatile}) + addconst(p.R, constptr(r, mem.c)) + } + } + + /* unary expressions */ + case *IrUnaryExpr: + { + if cc, ok := consts[p.V]; !ok { + ins = append(ins, p) + } else if !cc.i { + panic(fmt.Sprintf("constprop: integer operation on pointer value %p: %s", cc.p, p)) + } else { + r := self.unary(cc.v, p.Op) + ins = append(ins, &IrConstInt{R: p.R, V: r}) + addconst(p.R, constint(r)) + } + } + + /* binary expressions */ + case *IrBinaryExpr: + { + if x, ok := consts[p.X]; !ok { + ins = append(ins, p) + } else if y, ok := consts[p.Y]; !ok { + ins = append(ins, p) + } else if !x.i { + panic(fmt.Sprintf("constprop: integer operation on pointer value %p: %s", x.p, p)) + } else if !y.i { + panic(fmt.Sprintf("constprop: integer operation on pointer value %p: %s", y.p, p)) + } else { + r := self.binary(x.v, y.v, p.Op) + ins = append(ins, &IrConstInt{R: p.R, V: r}) + addconst(p.R, constint(r)) + } + } + + /* bit test and set operation */ + case *IrBitTestSet: + { + if x, ok := consts[p.X]; !ok { + ins = append(ins, p) + } else if y, ok := consts[p.Y]; !ok { + ins = append(ins, p) + } else if !x.i { + panic(fmt.Sprintf("constprop: integer operation on pointer value %p: %s", x.p, p)) + } else if !y.i { + panic(fmt.Sprintf("constprop: integer operation on pointer value %p: %s", y.p, p)) + } else { + t, s := self.testandset(x.v, y.v) + ins = append(ins, &IrConstInt{R: p.T, V: t}, &IrConstInt{R: p.S, V: s}) + addconst(p.T, constint(t)) + addconst(p.S, constint(s)) + } + } + } + } + + /* rebuild the basic block */ + bb.Phi = phi + bb.Ins = ins + } + } } diff --git a/internal/atm/ssa/pass_copyelim.go b/internal/atm/ssa/pass_copyelim.go index 196a0a9..a173f17 100644 --- a/internal/atm/ssa/pass_copyelim.go +++ b/internal/atm/ssa/pass_copyelim.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,119 +20,132 @@ package ssa type CopyElim struct{} func (CopyElim) Apply(cfg *CFG) { - regs := make(map[Reg]Reg) - consts := make(map[Reg]_ConstData) - - /* constant zero registers */ - consts[Rz] = constint(0) - consts[Pn] = constptr(nil, Const) - - /* register replacement func */ - replacereg := func(rr *Reg) { - var rv Reg - var ok bool - var cc _ConstData - - /* substitute registers */ - for { - if rv, ok = regs[*rr]; ok { - *rr = rv - } else { - break - } - } - - /* substitute zero registers */ - if cc, ok = consts[*rr]; ok { - if cc.i && cc.v == 0 { - *rr = Rz - } else if !cc.i && cc.p == nil { - *rr = Pn - } - } - } - - /* Phase 1: Find all the constants */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - for _, v := range bb.Ins { - switch p := v.(type) { - case *IrConstInt: consts[p.R] = constint(p.V) - case *IrConstPtr: consts[p.R] = constptr(p.P, p.M) - } - } - }) - - /* Phase 2: Identify all the identity operations */ - for _, bb := range cfg.PostOrder().Reversed() { - for _, v := range bb.Ins { - switch p := v.(type) { - default: { - continue - } - - /* pointer arithmetic */ - case *IrLEA: { - if cc, ok := consts[p.Off]; ok && cc.i && cc.v == 0 { - regs[p.R] = p.Mem - } - } - - /* integer arithmetic */ - case *IrBinaryExpr: { - var c bool - var i int64 - var x _ConstData - var y _ConstData - - /* calculate mathematical identities */ - switch p.Op { - case IrOpAdd : i = 0 - case IrOpSub : i = 0 - case IrOpMul : i = 1 - case IrOpAnd : i = -1 - case IrOpOr : i = 0 - case IrOpXor : i = 0 - case IrOpShr : i = 0 - default : continue - } - - /* check for identity operations */ - if x, c = consts[p.X]; c && x.i && x.v == i { - regs[p.R] = p.Y - } else if y, c = consts[p.Y]; c && y.i && y.v == i { - regs[p.R] = p.X - } - } - } - } - } - - /* Phase 3: Replace all the register references */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - var ok bool - var use IrUsages - - /* replace in Phi nodes */ - for _, v := range bb.Phi { - for _, u := range v.Usages() { - replacereg(u) - } - } - - /* replace in instructions */ - for _, v := range bb.Ins { - if use, ok = v.(IrUsages); ok { - for _, u := range use.Usages() { - replacereg(u) - } - } - } - - /* replace in terminators */ - if use, ok = bb.Term.(IrUsages); ok { - for _, u := range use.Usages() { - replacereg(u) - } - } - }) + regs := make(map[Reg]Reg) + consts := make(map[Reg]_ConstData) + + /* constant zero registers */ + consts[Rz] = constint(0) + consts[Pn] = constptr(nil, Const) + + /* register replacement func */ + replacereg := func(rr *Reg) { + var rv Reg + var ok bool + var cc _ConstData + + /* substitute registers */ + for { + if rv, ok = regs[*rr]; ok { + *rr = rv + } else { + break + } + } + + /* substitute zero registers */ + if cc, ok = consts[*rr]; ok { + if cc.i && cc.v == 0 { + *rr = Rz + } else if !cc.i && cc.p == nil { + *rr = Pn + } + } + } + + /* Phase 1: Find all the constants */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + for _, v := range bb.Ins { + switch p := v.(type) { + case *IrConstInt: + consts[p.R] = constint(p.V) + case *IrConstPtr: + consts[p.R] = constptr(p.P, p.M) + } + } + }) + + /* Phase 2: Identify all the identity operations */ + for _, bb := range cfg.PostOrder().Reversed() { + for _, v := range bb.Ins { + switch p := v.(type) { + default: + { + continue + } + + /* pointer arithmetic */ + case *IrLEA: + { + if cc, ok := consts[p.Off]; ok && cc.i && cc.v == 0 { + regs[p.R] = p.Mem + } + } + + /* integer arithmetic */ + case *IrBinaryExpr: + { + var c bool + var i int64 + var x _ConstData + var y _ConstData + + /* calculate mathematical identities */ + switch p.Op { + case IrOpAdd: + i = 0 + case IrOpSub: + i = 0 + case IrOpMul: + i = 1 + case IrOpAnd: + i = -1 + case IrOpOr: + i = 0 + case IrOpXor: + i = 0 + case IrOpShr: + i = 0 + default: + continue + } + + /* check for identity operations */ + if x, c = consts[p.X]; c && x.i && x.v == i { + regs[p.R] = p.Y + } else if y, c = consts[p.Y]; c && y.i && y.v == i { + regs[p.R] = p.X + } + } + } + } + } + + /* Phase 3: Replace all the register references */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + var ok bool + var use IrUsages + + /* replace in Phi nodes */ + for _, v := range bb.Phi { + for _, u := range v.Usages() { + replacereg(u) + } + } + + /* replace in instructions */ + for _, v := range bb.Ins { + if use, ok = v.(IrUsages); ok { + for _, u := range use.Usages() { + replacereg(u) + } + } + } + + /* replace in terminators */ + if use, ok = bb.Term.(IrUsages); ok { + for _, u := range use.Usages() { + replacereg(u) + } + } + }) } diff --git a/internal/atm/ssa/pass_fusion_amd64.go b/internal/atm/ssa/pass_fusion_amd64.go index 4135f33..96a92ee 100644 --- a/internal/atm/ssa/pass_fusion_amd64.go +++ b/internal/atm/ssa/pass_fusion_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,366 +17,394 @@ package ssa import ( - `github.com/cloudwego/frugal/internal/atm/abi` - `github.com/cloudwego/frugal/internal/cpu` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/abi" + "github.com/cloudwego/frugal/internal/cpu" + "github.com/cloudwego/frugal/internal/rt" ) // Fusion fuses simple instructions into more complex one, to reduce the instruction count. type Fusion struct{} func (Fusion) flagsafe(bb *BasicBlock, ins IrNode) bool { - i := -1 - p := IrNode(nil) - - /* find the instruction */ - for i, p = range bb.Ins { - if p == ins { - break - } - } - - /* not found, the instruction is in another basic - * block, which cannot guarantee it's flags preserving */ - if p != ins { - return false - } - - /* check for instructions after it, only some instructions that are - * known to preserve flags, all other instructions are assumed to clobber */ - for _, p = range bb.Ins[i + 1:] { - switch p.(type) { - case *IrAMD64_INT : break - case *IrAMD64_LEA : break - case *IrAMD64_BSWAP : break - case *IrAMD64_MOVSLQ : break - case *IrAMD64_MOV_abs : break - case *IrAMD64_MOV_ptr : break - case *IrAMD64_MOV_reg : break - case *IrAMD64_MOV_load : break - case *IrAMD64_MOV_store_r : break - case *IrAMD64_MOV_store_i : break - case *IrAMD64_MOV_load_be : break - case *IrAMD64_MOV_store_be : break - case *IrAMD64_CALL_gcwb : break - default : return false - } - } - - /* everything checked fine */ - return true + i := -1 + p := IrNode(nil) + + /* find the instruction */ + for i, p = range bb.Ins { + if p == ins { + break + } + } + + /* not found, the instruction is in another basic + * block, which cannot guarantee it's flags preserving */ + if p != ins { + return false + } + + /* check for instructions after it, only some instructions that are + * known to preserve flags, all other instructions are assumed to clobber */ + for _, p = range bb.Ins[i+1:] { + switch p.(type) { + case *IrAMD64_INT: + break + case *IrAMD64_LEA: + break + case *IrAMD64_BSWAP: + break + case *IrAMD64_MOVSLQ: + break + case *IrAMD64_MOV_abs: + break + case *IrAMD64_MOV_ptr: + break + case *IrAMD64_MOV_reg: + break + case *IrAMD64_MOV_load: + break + case *IrAMD64_MOV_store_r: + break + case *IrAMD64_MOV_store_i: + break + case *IrAMD64_MOV_load_be: + break + case *IrAMD64_MOV_store_be: + break + case *IrAMD64_CALL_gcwb: + break + default: + return false + } + } + + /* everything checked fine */ + return true } func (self Fusion) Apply(cfg *CFG) { - done := false - defs := make(map[Reg]IrNode) - - /* leaq {mem}, %r1 ; op {disp}(%r1), %r2 --> op {disp}{mem}, %r2 - * leaq {off1}(%r0), %r2 ; op {off2}(%r1,%r2), %r3 --> op {off1+off2}(%r1,%r0), %r3 - * leaq {off1}(%r0), %r1 ; op {off2}(%r1,%r2,{scale}), %r3 --> op {off1+off2}(%r0,%r2,{scale}), %r3 - * addsub $imm, %r1, %r2 ; op {disp}(%r3,%r2), %r4 --> op {disp+imm}(%r3,%r1), %r4 - * movabsq $imm, %r1 ; op {disp}(%r0,%r1,{scale}), %r2 --> op {disp+imm*scale}(%r0), %r2 */ - fusemem := func(m *Mem) { - if m.I == Rz { - if ins, ok := defs[m.M].(*IrAMD64_LEA); ok { - if x := int64(m.D) + int64(ins.M.D); isi32(x) { - m.M = ins.M.M - m.I = ins.M.I - m.S = ins.M.S - m.D = int32(x) - done = false - } - } - } else { - if ins, ok := defs[m.M].(*IrAMD64_LEA); ok && ins.M.I == Rz { - if x := int64(m.D) + int64(ins.M.D); isi32(x) { - m.M = ins.M.M - m.D = int32(x) - done = false - } - } else if ins, ok := defs[m.I].(*IrAMD64_LEA); ok && m.S == 1 && ins.M.I == Rz { - if x := int64(m.D) + int64(ins.M.D); isi32(x) { - m.I = ins.M.M - m.D = int32(x) - done = false - } - } else if ins, ok := defs[m.I].(*IrAMD64_BinOp_ri); ok && m.S == 1 && ins.Op.IsAdditive() { - if x := int64(m.D) + int64(ins.Y * ins.Op.ScaleFactor()); isi32(x) { - m.I = ins.X - m.D = int32(x) - done = false - } - } else if ins, ok := defs[m.I].(*IrAMD64_MOV_abs); ok { - if x := int64(m.D) + ins.V; isi32(x) { - m.I = Rz - m.D = int32(x) - done = false - } - } - } - } - - /* retry until no more modifications */ - for !done { - done = true - rt.MapClear(defs) - - /* pseudo-definition for zero registers */ - defs[Rz] = &IrAMD64_MOV_abs { R: Rz, V: 0 } - defs[Pn] = &IrAMD64_MOV_ptr { R: Rz, P: nil } - - /* check every block */ - for _, bb := range cfg.PostOrder().Reversed() { - var r *Reg - var ok bool - - /* mark all the definitions in Phi nodes */ - for _, v := range bb.Phi { - for _, r = range v.Definitions() { - if _, ok = defs[*r]; !ok { - defs[*r] = v - } else if r.Kind() != K_zero { - panic("register redefined: " + r.String()) - } - } - } - - /* scan all the instructions */ - for i, v := range bb.Ins { - var m IrAMD64_MemOp - var d IrDefinitions - - /* fuse memory addresses in instructions */ - if m, ok = v.(IrAMD64_MemOp); ok { - fusemem(m.MemOp()) - } - - /* fuse instructions if possible */ - switch p := v.(type) { - default: { - break - } - - /* movx {mem}, %r0; bswapx %r0, %r1 --> movbex {mem}, %r1 */ - case *IrAMD64_BSWAP: { - if ins, ok := defs[p.V].(*IrAMD64_MOV_load); ok && ins.N != 1 && cpu.HasMOVBE { - done = false - bb.Ins[i] = &IrAMD64_MOV_load_be { R: p.R, M: ins.M, N: ins.N } - } - } - - /* movq {i32}, %r0; movx %r0, {mem} --> movx {i32}, {mem} - * movq {p32}, %r0; movx %r0, {mem} --> movx {p32}, {mem} - * bswapx %r0, %r1; movx %r1, {mm1} --> movbex %r0, {mm1} */ - case *IrAMD64_MOV_store_r: { - if ins, ok := defs[p.R].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { - done = false - bb.Ins[i] = &IrAMD64_MOV_store_i { V: int32(ins.V), M: p.M, N: p.N } - } else if ins, ok := defs[p.R].(*IrAMD64_MOV_ptr); ok && isp32(ins.P) && p.N == abi.PtrSize { - done = false - bb.Ins[i] = &IrAMD64_MOV_store_p { P: ins.P, M: p.M } - } else if ins, ok := defs[p.R].(*IrAMD64_BSWAP); ok && p.N != 1 && cpu.HasMOVBE { - done = false - bb.Ins[i] = &IrAMD64_MOV_store_be { R: ins.V, M: p.M, N: p.N } - } - } - - /* movq {i32}, %r0; binop %r0, %r1 --> binop {i32}, %r1 - * movq {mem}, %r0; binop %r0, %r1 --> binop {mem}, %r1 */ - case *IrAMD64_BinOp_rr: { - if ins, ok := defs[p.Y].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { - done = false - bb.Ins[i] = &IrAMD64_BinOp_ri { R: p.R, X: p.X, Y: int32(ins.V), Op: p.Op } - } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_load); ok && ins.N == abi.PtrSize { - done = false - bb.Ins[i] = &IrAMD64_BinOp_rm { R: p.R, X: p.X, Y: ins.M, Op: p.Op } - } - } - - /* movq {u8}, %r0; btsq %r0, %r1; setc %r2 --> btsq {u8}, %r1; setc %r2 */ - case *IrAMD64_BTSQ_rr: { - if ins, ok := defs[p.Y].(*IrAMD64_MOV_abs); ok && isu8(ins.V) { - done = false - bb.Ins[i] = &IrAMD64_BTSQ_ri { T: p.T, S: p.S, X: p.X, Y: uint8(ins.V) } - } - } - - /* movq {i32}, %r0; cmpq %r0, %r1 --> cmpq {i32}, %r1 - * movx {ptr}, %p0; cmpq %p0, %p1 --> cmpq {ptr}, %p1 - * movq {mem}, %r0; cmpq %r0, %r1 --> cmpx {mem}, %r1 - * movq {i32}, %r1; cmpq %r0, %r1 --> cmpq %r0, {i32} - * movq {ptr}, %p1; cmpq %p0, %p1 --> cmpq %p0, {ptr} - * movx {mem}, %r1; cmpq %r0, %r1 --> cmpx %r0, {mem} */ - case *IrAMD64_CMPQ_rr: { - if ins, ok := defs[p.X].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { - done = false - bb.Ins[i] = &IrAMD64_CMPQ_ir { R: p.R, X: int32(ins.V), Y: p.Y, Op: p.Op } - } else if ins, ok := defs[p.X].(*IrAMD64_MOV_ptr); ok { - done = false - bb.Ins[i] = &IrAMD64_CMPQ_pr { R: p.R, X: ins.P, Y: p.Y, Op: p.Op } - } else if ins, ok := defs[p.X].(*IrAMD64_MOV_load); ok && ins.N != 16 { - done = false - bb.Ins[i] = &IrAMD64_CMPQ_mr { R: p.R, X: ins.M, Y: p.Y, Op: p.Op, N: ins.N } - } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { - done = false - bb.Ins[i] = &IrAMD64_CMPQ_ri { R: p.R, X: p.X, Y: int32(ins.V), Op: p.Op } - } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_ptr); ok { - done = false - bb.Ins[i] = &IrAMD64_CMPQ_rp { R: p.R, X: p.X, Y: ins.P, Op: p.Op } - } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_load); ok && ins.N != 16 { - done = false - bb.Ins[i] = &IrAMD64_CMPQ_rm { R: p.R, X: p.X, Y: ins.M, Op: p.Op, N: ins.N } - } - } - - /* movq {i32}, %r0; cmpx %r0, {mem} --> cmpx {i32}, {mem} - * movq {p32}, %p0; cmpq %p0, {mem} --> cmpq {p32}, {mem} */ - case *IrAMD64_CMPQ_rm: { - if ins, ok := defs[p.X].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { - done = false - bb.Ins[i] = &IrAMD64_CMPQ_im { R: p.R, X: int32(ins.V), Y: p.Y, Op: p.Op, N: p.N } - } else if ins, ok := defs[p.X].(*IrAMD64_MOV_ptr); ok && isp32(ins.P) && p.N == abi.PtrSize { - done = false - bb.Ins[i] = &IrAMD64_CMPQ_pm { R: p.R, X: ins.P, Y: p.Y, Op: p.Op } - } - } - - /* movq {i32}, %r0; cmpx {mem}, %r0 --> cmpx {mem}, {i32} - * movq {p32}, %p0; cmpq {mem}, %p0 --> cmpq {mem}, {p32} */ - case *IrAMD64_CMPQ_mr: { - if ins, ok := defs[p.Y].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { - done = false - bb.Ins[i] = &IrAMD64_CMPQ_mi { R: p.R, X: p.X, Y: int32(ins.V), Op: p.Op, N: p.N } - } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_ptr); ok && isp32(ins.P) && p.N == abi.PtrSize { - done = false - bb.Ins[i] = &IrAMD64_CMPQ_mp { R: p.R, X: p.X, Y: ins.P, Op: p.Op } - } - } - } - - /* mark all the definitions in instructions */ - if d, ok = v.(IrDefinitions); ok { - for _, r = range d.Definitions() { - if _, ok = defs[*r]; !ok { - defs[*r] = v - } else if r.Kind() != K_zero { - panic("register redefined: " + r.String()) - } - } - } - } - - /* fuse memory operation in terminators */ - if m, ok := bb.Term.(IrAMD64_MemOp); ok { - fusemem(m.MemOp()) - } - - /* fuse terminators if possible */ - switch p := bb.Term.(type) { - default: { - break - } - - /* movq {i32}, %r0; cmpq %r0, %r1; jcc {label} --> cmpq {i32}, %r1; jcc {label} - * movq {ptr}, %p0; cmpq %p0, %p1; jcc {label} --> cmpq {ptr}, %p1; jcc {label} - * movx {mem}, %r0; cmpq %r0, %r1; jcc {label} --> cmpx {mem}, %r1; jcc {label} - * movq {i32}, %r1; cmpq %r0, %r1; jcc {label} --> cmpq %r0, {i32}; jcc {label} - * movq {ptr}, %p1; cmpq %p0, %p1; jcc {label} --> cmpq %p0, {ptr}; jcc {label} - * movx {mem}, %r1; cmpq %r0, %r1; jcc {label} --> cmpx %r0, {mem}; jcc {label} */ - case *IrAMD64_Jcc_rr: { - if ins, ok := defs[p.X].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { - done = false - bb.Term = &IrAMD64_Jcc_ir { X: int32(ins.V), Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op } - } else if ins, ok := defs[p.X].(*IrAMD64_MOV_ptr); ok { - done = false - bb.Term = &IrAMD64_Jcc_pr { X: ins.P, Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op } - } else if ins, ok := defs[p.X].(*IrAMD64_MOV_load); ok { - done = false - bb.Term = &IrAMD64_Jcc_mr { X: ins.M, Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op, N: ins.N } - } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { - done = false - bb.Term = &IrAMD64_Jcc_ri { X: p.X, Y: int32(ins.V), To: p.To, Ln: p.Ln, Op: p.Op } - } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_ptr); ok { - done = false - bb.Term = &IrAMD64_Jcc_rp { X: p.X, Y: ins.P, To: p.To, Ln: p.Ln, Op: p.Op } - } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_load); ok { - done = false - bb.Term = &IrAMD64_Jcc_rm { X: p.X, Y: ins.M, To: p.To, Ln: p.Ln, Op: p.Op, N: ins.N } - } - } - - /* setcc %r0; cmpq %r0, $0; je {label} --> jncc {label} */ - case *IrAMD64_Jcc_ri: { - if p.Y == 0 && p.Op == IrAMD64_CmpEq { - if ins, ok := defs[p.X].(*IrAMD64_CMPQ_rr); ok { - done = false - bb.Term = &IrAMD64_Jcc_rr { X: ins.X, Y: ins.Y, To: p.To, Ln: p.Ln, Op: ins.Op.Negated() } - } else if ins, ok := defs[p.X].(*IrAMD64_CMPQ_rm); ok { - done = false - bb.Term = &IrAMD64_Jcc_rm { X: ins.X, Y: ins.Y, To: p.To, Ln: p.Ln, Op: ins.Op.Negated(), N: ins.N } - } else if ins, ok := defs[p.X].(*IrAMD64_CMPQ_mr); ok { - done = false - bb.Term = &IrAMD64_Jcc_mr { X: ins.X, Y: ins.Y, To: p.To, Ln: p.Ln, Op: ins.Op.Negated(), N: ins.N } - } else if ins, ok := defs[p.X].(*IrAMD64_BTSQ_rr); ok && p.X == ins.T && self.flagsafe(bb, ins) { - done = false - bb.Term, ins.T = &IrAMD64_JNC { To: p.To, Ln: p.Ln }, Rz - } else if ins, ok := defs[p.X].(*IrAMD64_BTSQ_ri); ok && p.X == ins.T && self.flagsafe(bb, ins) { - done = false - bb.Term, ins.T = &IrAMD64_JNC { To: p.To, Ln: p.Ln }, Rz - } - } - } - - /* setcc %r0; cmpq $0, %r0; je {label} --> jncc {label} */ - case *IrAMD64_Jcc_ir: { - if p.Y == 0 && p.Op == IrAMD64_CmpEq { - if ins, ok := defs[p.Y].(*IrAMD64_CMPQ_rr); ok { - done = false - bb.Term = &IrAMD64_Jcc_rr { X: ins.X, Y: ins.Y, To: p.To, Ln: p.Ln, Op: ins.Op.Negated() } - } else if ins, ok := defs[p.Y].(*IrAMD64_CMPQ_rm); ok { - done = false - bb.Term = &IrAMD64_Jcc_rm { X: ins.X, Y: ins.Y, To: p.To, Ln: p.Ln, Op: ins.Op.Negated(), N: ins.N } - } else if ins, ok := defs[p.Y].(*IrAMD64_CMPQ_mr); ok { - done = false - bb.Term = &IrAMD64_Jcc_mr { X: ins.X, Y: ins.Y, To: p.To, Ln: p.Ln, Op: ins.Op.Negated(), N: ins.N } - } else if ins, ok := defs[p.Y].(*IrAMD64_BTSQ_rr); ok && p.Y == ins.T && self.flagsafe(bb, ins) { - done = false - bb.Term, ins.T = &IrAMD64_JNC { To: p.To, Ln: p.Ln }, Rz - } else if ins, ok := defs[p.Y].(*IrAMD64_BTSQ_ri); ok && p.Y == ins.T && self.flagsafe(bb, ins) { - done = false - bb.Term, ins.T = &IrAMD64_JNC { To: p.To, Ln: p.Ln }, Rz - } - } - } - - /* movq {i32}, %r0; cmpq %r0, {mem}; jcc {label} --> cmpq {i32}, {mem}; jcc {label} - * movq {p32}, %p0; cmpq %p0, {mem}; jcc {label} --> cmpq {p32}, {mem}; jcc {label} */ - case *IrAMD64_Jcc_rm: { - if ins, ok := defs[p.X].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { - done = false - bb.Term = &IrAMD64_Jcc_im { X: int32(ins.V), Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op, N: p.N } - } else if ins, ok := defs[p.X].(*IrAMD64_MOV_ptr); ok && isp32(ins.P) && p.N == abi.PtrSize { - done = false - bb.Term = &IrAMD64_Jcc_pm { X: ins.P, Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op } - } - } - - /* movq {i32}, %r0; cmpq {mem}, %r0; jcc {label} --> cmpq {mem}, {i32}; jcc {label} - * movq {p32}, %p0; cmpq {mem}, %p0; jcc {label} --> cmpq {mem}, {p32}; jcc {label} */ - case *IrAMD64_Jcc_mr: { - if ins, ok := defs[p.Y].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { - done = false - bb.Term = &IrAMD64_Jcc_mi { X: p.X, Y: int32(ins.V), To: p.To, Ln: p.Ln, Op: p.Op, N: p.N } - } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_ptr); ok && isp32(ins.P) && p.N == abi.PtrSize { - done = false - bb.Term = &IrAMD64_Jcc_mp { X: p.X, Y: ins.P, To: p.To, Ln: p.Ln, Op: p.Op } - } - } - } - } - - /* perform TDCE & reorder after each round */ - new(TDCE).Apply(cfg) - new(Reorder).Apply(cfg) - } + done := false + defs := make(map[Reg]IrNode) + + /* leaq {mem}, %r1 ; op {disp}(%r1), %r2 --> op {disp}{mem}, %r2 + * leaq {off1}(%r0), %r2 ; op {off2}(%r1,%r2), %r3 --> op {off1+off2}(%r1,%r0), %r3 + * leaq {off1}(%r0), %r1 ; op {off2}(%r1,%r2,{scale}), %r3 --> op {off1+off2}(%r0,%r2,{scale}), %r3 + * addsub $imm, %r1, %r2 ; op {disp}(%r3,%r2), %r4 --> op {disp+imm}(%r3,%r1), %r4 + * movabsq $imm, %r1 ; op {disp}(%r0,%r1,{scale}), %r2 --> op {disp+imm*scale}(%r0), %r2 */ + fusemem := func(m *Mem) { + if m.I == Rz { + if ins, ok := defs[m.M].(*IrAMD64_LEA); ok { + if x := int64(m.D) + int64(ins.M.D); isi32(x) { + m.M = ins.M.M + m.I = ins.M.I + m.S = ins.M.S + m.D = int32(x) + done = false + } + } + } else { + if ins, ok := defs[m.M].(*IrAMD64_LEA); ok && ins.M.I == Rz { + if x := int64(m.D) + int64(ins.M.D); isi32(x) { + m.M = ins.M.M + m.D = int32(x) + done = false + } + } else if ins, ok := defs[m.I].(*IrAMD64_LEA); ok && m.S == 1 && ins.M.I == Rz { + if x := int64(m.D) + int64(ins.M.D); isi32(x) { + m.I = ins.M.M + m.D = int32(x) + done = false + } + } else if ins, ok := defs[m.I].(*IrAMD64_BinOp_ri); ok && m.S == 1 && ins.Op.IsAdditive() { + if x := int64(m.D) + int64(ins.Y*ins.Op.ScaleFactor()); isi32(x) { + m.I = ins.X + m.D = int32(x) + done = false + } + } else if ins, ok := defs[m.I].(*IrAMD64_MOV_abs); ok { + if x := int64(m.D) + ins.V; isi32(x) { + m.I = Rz + m.D = int32(x) + done = false + } + } + } + } + + /* retry until no more modifications */ + for !done { + done = true + rt.MapClear(defs) + + /* pseudo-definition for zero registers */ + defs[Rz] = &IrAMD64_MOV_abs{R: Rz, V: 0} + defs[Pn] = &IrAMD64_MOV_ptr{R: Rz, P: nil} + + /* check every block */ + for _, bb := range cfg.PostOrder().Reversed() { + var r *Reg + var ok bool + + /* mark all the definitions in Phi nodes */ + for _, v := range bb.Phi { + for _, r = range v.Definitions() { + if _, ok = defs[*r]; !ok { + defs[*r] = v + } else if r.Kind() != K_zero { + panic("register redefined: " + r.String()) + } + } + } + + /* scan all the instructions */ + for i, v := range bb.Ins { + var m IrAMD64_MemOp + var d IrDefinitions + + /* fuse memory addresses in instructions */ + if m, ok = v.(IrAMD64_MemOp); ok { + fusemem(m.MemOp()) + } + + /* fuse instructions if possible */ + switch p := v.(type) { + default: + { + break + } + + /* movx {mem}, %r0; bswapx %r0, %r1 --> movbex {mem}, %r1 */ + case *IrAMD64_BSWAP: + { + if ins, ok := defs[p.V].(*IrAMD64_MOV_load); ok && ins.N != 1 && cpu.HasMOVBE { + done = false + bb.Ins[i] = &IrAMD64_MOV_load_be{R: p.R, M: ins.M, N: ins.N} + } + } + + /* movq {i32}, %r0; movx %r0, {mem} --> movx {i32}, {mem} + * movq {p32}, %r0; movx %r0, {mem} --> movx {p32}, {mem} + * bswapx %r0, %r1; movx %r1, {mm1} --> movbex %r0, {mm1} */ + case *IrAMD64_MOV_store_r: + { + if ins, ok := defs[p.R].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { + done = false + bb.Ins[i] = &IrAMD64_MOV_store_i{V: int32(ins.V), M: p.M, N: p.N} + } else if ins, ok := defs[p.R].(*IrAMD64_MOV_ptr); ok && isp32(ins.P) && p.N == abi.PtrSize { + done = false + bb.Ins[i] = &IrAMD64_MOV_store_p{P: ins.P, M: p.M} + } else if ins, ok := defs[p.R].(*IrAMD64_BSWAP); ok && p.N != 1 && cpu.HasMOVBE { + done = false + bb.Ins[i] = &IrAMD64_MOV_store_be{R: ins.V, M: p.M, N: p.N} + } + } + + /* movq {i32}, %r0; binop %r0, %r1 --> binop {i32}, %r1 + * movq {mem}, %r0; binop %r0, %r1 --> binop {mem}, %r1 */ + case *IrAMD64_BinOp_rr: + { + if ins, ok := defs[p.Y].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { + done = false + bb.Ins[i] = &IrAMD64_BinOp_ri{R: p.R, X: p.X, Y: int32(ins.V), Op: p.Op} + } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_load); ok && ins.N == abi.PtrSize { + done = false + bb.Ins[i] = &IrAMD64_BinOp_rm{R: p.R, X: p.X, Y: ins.M, Op: p.Op} + } + } + + /* movq {u8}, %r0; btsq %r0, %r1; setc %r2 --> btsq {u8}, %r1; setc %r2 */ + case *IrAMD64_BTSQ_rr: + { + if ins, ok := defs[p.Y].(*IrAMD64_MOV_abs); ok && isu8(ins.V) { + done = false + bb.Ins[i] = &IrAMD64_BTSQ_ri{T: p.T, S: p.S, X: p.X, Y: uint8(ins.V)} + } + } + + /* movq {i32}, %r0; cmpq %r0, %r1 --> cmpq {i32}, %r1 + * movx {ptr}, %p0; cmpq %p0, %p1 --> cmpq {ptr}, %p1 + * movq {mem}, %r0; cmpq %r0, %r1 --> cmpx {mem}, %r1 + * movq {i32}, %r1; cmpq %r0, %r1 --> cmpq %r0, {i32} + * movq {ptr}, %p1; cmpq %p0, %p1 --> cmpq %p0, {ptr} + * movx {mem}, %r1; cmpq %r0, %r1 --> cmpx %r0, {mem} */ + case *IrAMD64_CMPQ_rr: + { + if ins, ok := defs[p.X].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { + done = false + bb.Ins[i] = &IrAMD64_CMPQ_ir{R: p.R, X: int32(ins.V), Y: p.Y, Op: p.Op} + } else if ins, ok := defs[p.X].(*IrAMD64_MOV_ptr); ok { + done = false + bb.Ins[i] = &IrAMD64_CMPQ_pr{R: p.R, X: ins.P, Y: p.Y, Op: p.Op} + } else if ins, ok := defs[p.X].(*IrAMD64_MOV_load); ok && ins.N != 16 { + done = false + bb.Ins[i] = &IrAMD64_CMPQ_mr{R: p.R, X: ins.M, Y: p.Y, Op: p.Op, N: ins.N} + } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { + done = false + bb.Ins[i] = &IrAMD64_CMPQ_ri{R: p.R, X: p.X, Y: int32(ins.V), Op: p.Op} + } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_ptr); ok { + done = false + bb.Ins[i] = &IrAMD64_CMPQ_rp{R: p.R, X: p.X, Y: ins.P, Op: p.Op} + } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_load); ok && ins.N != 16 { + done = false + bb.Ins[i] = &IrAMD64_CMPQ_rm{R: p.R, X: p.X, Y: ins.M, Op: p.Op, N: ins.N} + } + } + + /* movq {i32}, %r0; cmpx %r0, {mem} --> cmpx {i32}, {mem} + * movq {p32}, %p0; cmpq %p0, {mem} --> cmpq {p32}, {mem} */ + case *IrAMD64_CMPQ_rm: + { + if ins, ok := defs[p.X].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { + done = false + bb.Ins[i] = &IrAMD64_CMPQ_im{R: p.R, X: int32(ins.V), Y: p.Y, Op: p.Op, N: p.N} + } else if ins, ok := defs[p.X].(*IrAMD64_MOV_ptr); ok && isp32(ins.P) && p.N == abi.PtrSize { + done = false + bb.Ins[i] = &IrAMD64_CMPQ_pm{R: p.R, X: ins.P, Y: p.Y, Op: p.Op} + } + } + + /* movq {i32}, %r0; cmpx {mem}, %r0 --> cmpx {mem}, {i32} + * movq {p32}, %p0; cmpq {mem}, %p0 --> cmpq {mem}, {p32} */ + case *IrAMD64_CMPQ_mr: + { + if ins, ok := defs[p.Y].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { + done = false + bb.Ins[i] = &IrAMD64_CMPQ_mi{R: p.R, X: p.X, Y: int32(ins.V), Op: p.Op, N: p.N} + } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_ptr); ok && isp32(ins.P) && p.N == abi.PtrSize { + done = false + bb.Ins[i] = &IrAMD64_CMPQ_mp{R: p.R, X: p.X, Y: ins.P, Op: p.Op} + } + } + } + + /* mark all the definitions in instructions */ + if d, ok = v.(IrDefinitions); ok { + for _, r = range d.Definitions() { + if _, ok = defs[*r]; !ok { + defs[*r] = v + } else if r.Kind() != K_zero { + panic("register redefined: " + r.String()) + } + } + } + } + + /* fuse memory operation in terminators */ + if m, ok := bb.Term.(IrAMD64_MemOp); ok { + fusemem(m.MemOp()) + } + + /* fuse terminators if possible */ + switch p := bb.Term.(type) { + default: + { + break + } + + /* movq {i32}, %r0; cmpq %r0, %r1; jcc {label} --> cmpq {i32}, %r1; jcc {label} + * movq {ptr}, %p0; cmpq %p0, %p1; jcc {label} --> cmpq {ptr}, %p1; jcc {label} + * movx {mem}, %r0; cmpq %r0, %r1; jcc {label} --> cmpx {mem}, %r1; jcc {label} + * movq {i32}, %r1; cmpq %r0, %r1; jcc {label} --> cmpq %r0, {i32}; jcc {label} + * movq {ptr}, %p1; cmpq %p0, %p1; jcc {label} --> cmpq %p0, {ptr}; jcc {label} + * movx {mem}, %r1; cmpq %r0, %r1; jcc {label} --> cmpx %r0, {mem}; jcc {label} */ + case *IrAMD64_Jcc_rr: + { + if ins, ok := defs[p.X].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { + done = false + bb.Term = &IrAMD64_Jcc_ir{X: int32(ins.V), Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op} + } else if ins, ok := defs[p.X].(*IrAMD64_MOV_ptr); ok { + done = false + bb.Term = &IrAMD64_Jcc_pr{X: ins.P, Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op} + } else if ins, ok := defs[p.X].(*IrAMD64_MOV_load); ok { + done = false + bb.Term = &IrAMD64_Jcc_mr{X: ins.M, Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op, N: ins.N} + } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { + done = false + bb.Term = &IrAMD64_Jcc_ri{X: p.X, Y: int32(ins.V), To: p.To, Ln: p.Ln, Op: p.Op} + } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_ptr); ok { + done = false + bb.Term = &IrAMD64_Jcc_rp{X: p.X, Y: ins.P, To: p.To, Ln: p.Ln, Op: p.Op} + } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_load); ok { + done = false + bb.Term = &IrAMD64_Jcc_rm{X: p.X, Y: ins.M, To: p.To, Ln: p.Ln, Op: p.Op, N: ins.N} + } + } + + /* setcc %r0; cmpq %r0, $0; je {label} --> jncc {label} */ + case *IrAMD64_Jcc_ri: + { + if p.Y == 0 && p.Op == IrAMD64_CmpEq { + if ins, ok := defs[p.X].(*IrAMD64_CMPQ_rr); ok { + done = false + bb.Term = &IrAMD64_Jcc_rr{X: ins.X, Y: ins.Y, To: p.To, Ln: p.Ln, Op: ins.Op.Negated()} + } else if ins, ok := defs[p.X].(*IrAMD64_CMPQ_rm); ok { + done = false + bb.Term = &IrAMD64_Jcc_rm{X: ins.X, Y: ins.Y, To: p.To, Ln: p.Ln, Op: ins.Op.Negated(), N: ins.N} + } else if ins, ok := defs[p.X].(*IrAMD64_CMPQ_mr); ok { + done = false + bb.Term = &IrAMD64_Jcc_mr{X: ins.X, Y: ins.Y, To: p.To, Ln: p.Ln, Op: ins.Op.Negated(), N: ins.N} + } else if ins, ok := defs[p.X].(*IrAMD64_BTSQ_rr); ok && p.X == ins.T && self.flagsafe(bb, ins) { + done = false + bb.Term, ins.T = &IrAMD64_JNC{To: p.To, Ln: p.Ln}, Rz + } else if ins, ok := defs[p.X].(*IrAMD64_BTSQ_ri); ok && p.X == ins.T && self.flagsafe(bb, ins) { + done = false + bb.Term, ins.T = &IrAMD64_JNC{To: p.To, Ln: p.Ln}, Rz + } + } + } + + /* setcc %r0; cmpq $0, %r0; je {label} --> jncc {label} */ + case *IrAMD64_Jcc_ir: + { + if p.Y == 0 && p.Op == IrAMD64_CmpEq { + if ins, ok := defs[p.Y].(*IrAMD64_CMPQ_rr); ok { + done = false + bb.Term = &IrAMD64_Jcc_rr{X: ins.X, Y: ins.Y, To: p.To, Ln: p.Ln, Op: ins.Op.Negated()} + } else if ins, ok := defs[p.Y].(*IrAMD64_CMPQ_rm); ok { + done = false + bb.Term = &IrAMD64_Jcc_rm{X: ins.X, Y: ins.Y, To: p.To, Ln: p.Ln, Op: ins.Op.Negated(), N: ins.N} + } else if ins, ok := defs[p.Y].(*IrAMD64_CMPQ_mr); ok { + done = false + bb.Term = &IrAMD64_Jcc_mr{X: ins.X, Y: ins.Y, To: p.To, Ln: p.Ln, Op: ins.Op.Negated(), N: ins.N} + } else if ins, ok := defs[p.Y].(*IrAMD64_BTSQ_rr); ok && p.Y == ins.T && self.flagsafe(bb, ins) { + done = false + bb.Term, ins.T = &IrAMD64_JNC{To: p.To, Ln: p.Ln}, Rz + } else if ins, ok := defs[p.Y].(*IrAMD64_BTSQ_ri); ok && p.Y == ins.T && self.flagsafe(bb, ins) { + done = false + bb.Term, ins.T = &IrAMD64_JNC{To: p.To, Ln: p.Ln}, Rz + } + } + } + + /* movq {i32}, %r0; cmpq %r0, {mem}; jcc {label} --> cmpq {i32}, {mem}; jcc {label} + * movq {p32}, %p0; cmpq %p0, {mem}; jcc {label} --> cmpq {p32}, {mem}; jcc {label} */ + case *IrAMD64_Jcc_rm: + { + if ins, ok := defs[p.X].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { + done = false + bb.Term = &IrAMD64_Jcc_im{X: int32(ins.V), Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op, N: p.N} + } else if ins, ok := defs[p.X].(*IrAMD64_MOV_ptr); ok && isp32(ins.P) && p.N == abi.PtrSize { + done = false + bb.Term = &IrAMD64_Jcc_pm{X: ins.P, Y: p.Y, To: p.To, Ln: p.Ln, Op: p.Op} + } + } + + /* movq {i32}, %r0; cmpq {mem}, %r0; jcc {label} --> cmpq {mem}, {i32}; jcc {label} + * movq {p32}, %p0; cmpq {mem}, %p0; jcc {label} --> cmpq {mem}, {p32}; jcc {label} */ + case *IrAMD64_Jcc_mr: + { + if ins, ok := defs[p.Y].(*IrAMD64_MOV_abs); ok && isi32(ins.V) { + done = false + bb.Term = &IrAMD64_Jcc_mi{X: p.X, Y: int32(ins.V), To: p.To, Ln: p.Ln, Op: p.Op, N: p.N} + } else if ins, ok := defs[p.Y].(*IrAMD64_MOV_ptr); ok && isp32(ins.P) && p.N == abi.PtrSize { + done = false + bb.Term = &IrAMD64_Jcc_mp{X: p.X, Y: ins.P, To: p.To, Ln: p.Ln, Op: p.Op} + } + } + } + } + + /* perform TDCE & reorder after each round */ + new(TDCE).Apply(cfg) + new(Reorder).Apply(cfg) + } } diff --git a/internal/atm/ssa/pass_layout.go b/internal/atm/ssa/pass_layout.go index 95b447b..0a45305 100644 --- a/internal/atm/ssa/pass_layout.go +++ b/internal/atm/ssa/pass_layout.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,106 +17,110 @@ package ssa import ( - `sort` + "sort" ) type _Successor struct { - bb *BasicBlock - prob Likeliness + bb *BasicBlock + prob Likeliness } // Layout flattens the CFG into a linear FuncLayout type Layout struct{} func (self Layout) flatten(fn *FuncLayout, bb *BasicBlock) { - var ok bool - var nx []_Successor - - /* check for visited blocks */ - if _, ok = fn.Start[bb.Id]; ok { - return - } - - /* mark the starting position, and the corresponding block */ - fn.Start[bb.Id] = len(fn.Ins) - fn.Block[len(fn.Ins)] = bb - - /* add instructions and the terminator */ - fn.Ins = append(fn.Ins, bb.Ins...) - fn.Ins = append(fn.Ins, bb.Term) - - /* get all it's successors */ - for it := bb.Term.Successors(); it.Next(); { - nx = append(nx, _Successor { - bb : it.Block(), - prob : it.Likeliness(), - }) - } - - /* sort the likely blocks at front */ - sort.Slice(nx, func(i int, j int) bool { - return nx[i].prob == Likely && nx[j].prob == Unlikely - }) - - /* visit all the successors */ - for _, v := range nx { - self.flatten(fn, v.bb) - } + var ok bool + var nx []_Successor + + /* check for visited blocks */ + if _, ok = fn.Start[bb.Id]; ok { + return + } + + /* mark the starting position, and the corresponding block */ + fn.Start[bb.Id] = len(fn.Ins) + fn.Block[len(fn.Ins)] = bb + + /* add instructions and the terminator */ + fn.Ins = append(fn.Ins, bb.Ins...) + fn.Ins = append(fn.Ins, bb.Term) + + /* get all it's successors */ + for it := bb.Term.Successors(); it.Next(); { + nx = append(nx, _Successor{ + bb: it.Block(), + prob: it.Likeliness(), + }) + } + + /* sort the likely blocks at front */ + sort.Slice(nx, func(i int, j int) bool { + return nx[i].prob == Likely && nx[j].prob == Unlikely + }) + + /* visit all the successors */ + for _, v := range nx { + self.flatten(fn, v.bb) + } } func (self Layout) Apply(cfg *CFG) { - cfg.Func.Layout = new(FuncLayout) - cfg.Func.Layout.Start = make(map[int]int, cfg.MaxBlock()) - cfg.Func.Layout.Block = make(map[int]*BasicBlock, cfg.MaxBlock()) - - /* remove all virtual instructions */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - ins := bb.Ins - bb.Ins = bb.Ins[:0] - - /* filter the instructions */ - for _, v := range ins { - if _, ok := v.(*IrEntry) ; ok { continue } - if _, ok := v.(*IrClobberList) ; ok { continue } - bb.Ins = append(bb.Ins, v) - } - }) - - /* retry until no more intermediate blocks found */ - for { - var rt bool - var nb *BasicBlock - - /* collapse all the intermediate blocks */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - tr := bb.Term - it := tr.Successors() - - /* scan every successors */ - for it.Next() { - nx := it.Block() - st := nx.Term.Successors() - - /* must have exactly 1 predecessor, exactly 1 successor, and no instructions - * if so, update the successor to skip the intermediate block */ - if st.Next() && len(nx.Ins) == 0 && len(nx.Pred) == 1 { - if nb = st.Block(); !st.Next() { - rt = true - it.UpdateBlock(nb) - } - } - } - }) - - /* rebuild the CFG if needed */ - if !rt { - break - } else { - cfg.Rebuild() - } - } - - /* flatten the CFG */ - root := cfg.Root - self.flatten(cfg.Func.Layout, root) + cfg.Func.Layout = new(FuncLayout) + cfg.Func.Layout.Start = make(map[int]int, cfg.MaxBlock()) + cfg.Func.Layout.Block = make(map[int]*BasicBlock, cfg.MaxBlock()) + + /* remove all virtual instructions */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + ins := bb.Ins + bb.Ins = bb.Ins[:0] + + /* filter the instructions */ + for _, v := range ins { + if _, ok := v.(*IrEntry); ok { + continue + } + if _, ok := v.(*IrClobberList); ok { + continue + } + bb.Ins = append(bb.Ins, v) + } + }) + + /* retry until no more intermediate blocks found */ + for { + var rt bool + var nb *BasicBlock + + /* collapse all the intermediate blocks */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + tr := bb.Term + it := tr.Successors() + + /* scan every successors */ + for it.Next() { + nx := it.Block() + st := nx.Term.Successors() + + /* must have exactly 1 predecessor, exactly 1 successor, and no instructions + * if so, update the successor to skip the intermediate block */ + if st.Next() && len(nx.Ins) == 0 && len(nx.Pred) == 1 { + if nb = st.Block(); !st.Next() { + rt = true + it.UpdateBlock(nb) + } + } + } + }) + + /* rebuild the CFG if needed */ + if !rt { + break + } else { + cfg.Rebuild() + } + } + + /* flatten the CFG */ + root := cfg.Root + self.flatten(cfg.Func.Layout, root) } diff --git a/internal/atm/ssa/pass_lowering_amd64.go b/internal/atm/ssa/pass_lowering_amd64.go index ced827f..91771ae 100644 --- a/internal/atm/ssa/pass_lowering_amd64.go +++ b/internal/atm/ssa/pass_lowering_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,141 +20,176 @@ package ssa type Lowering struct{} func (Lowering) Apply(cfg *CFG) { - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - ins := bb.Ins - bb.Ins = make([]IrNode, 0, len(ins)) - - /* lower every instruction */ - for _, v := range ins { - switch p := v.(type) { - default: { - bb.Ins = append(bb.Ins, p) - } - - /* load from memory */ - case *IrLoad: { - bb.Ins = append(bb.Ins, &IrAMD64_MOV_load { - R: p.R, - N: p.Size, - M: Mem { - M: p.Mem, - I: Rz, - S: 1, - D: 0, - }, - }) - } - - /* store into memory */ - case *IrStore: { - bb.Ins = append(bb.Ins, &IrAMD64_MOV_store_r { - R: p.R, - N: p.Size, - M: Mem { - M: p.Mem, - I: Rz, - S: 1, - D: 0, - }, - }) - } - - /* load constant into register */ - case *IrConstInt: { - bb.Ins = append(bb.Ins, &IrAMD64_MOV_abs { - R: p.R, - V: p.V, - }) - } - - /* load pointer constant into register */ - case *IrConstPtr: { - bb.Ins = append(bb.Ins, &IrAMD64_MOV_ptr { - R: p.R, - P: p.P, - }) - } - - /* load effective address */ - case *IrLEA: { - bb.Ins = append(bb.Ins, &IrAMD64_LEA { - R: p.R, - M: Mem { - M: p.Mem, - I: p.Off, - S: 1, - D: 0, - }, - }) - } - - /* unary operators */ - case *IrUnaryExpr: { - switch p.Op { - case IrOpNegate : bb.Ins = append(bb.Ins, &IrAMD64_NEG { R: p.R, V: p.V }) - case IrOpSwap16 : bb.Ins = append(bb.Ins, &IrAMD64_BSWAP { R: p.R, V: p.V, N: 2 }) - case IrOpSwap32 : bb.Ins = append(bb.Ins, &IrAMD64_BSWAP { R: p.R, V: p.V, N: 4 }) - case IrOpSwap64 : bb.Ins = append(bb.Ins, &IrAMD64_BSWAP { R: p.R, V: p.V, N: 8 }) - case IrOpSx32to64 : bb.Ins = append(bb.Ins, &IrAMD64_MOVSLQ { R: p.R, V: p.V }) - default : panic("unreachable") - } - } - - /* binary operators */ - case *IrBinaryExpr: { - switch p.Op { - case IrOpAdd : bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr { R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinAdd }) - case IrOpSub : bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr { R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinSub }) - case IrOpMul : bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr { R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinMul }) - case IrOpAnd : bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr { R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinAnd }) - case IrOpOr : bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr { R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinOr }) - case IrOpXor : bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr { R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinXor }) - case IrOpShr : bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr { R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinShr }) - case IrCmpEq : bb.Ins = append(bb.Ins, &IrAMD64_CMPQ_rr { R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_CmpEq }) - case IrCmpNe : bb.Ins = append(bb.Ins, &IrAMD64_CMPQ_rr { R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_CmpNe }) - case IrCmpLt : bb.Ins = append(bb.Ins, &IrAMD64_CMPQ_rr { R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_CmpLt }) - case IrCmpLtu : bb.Ins = append(bb.Ins, &IrAMD64_CMPQ_rr { R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_CmpLtu }) - case IrCmpGeu : bb.Ins = append(bb.Ins, &IrAMD64_CMPQ_rr { R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_CmpGeu }) - default : panic("unreachable") - } - } - - /* bit test and set */ - case *IrBitTestSet: { - bb.Ins = append(bb.Ins, &IrAMD64_BTSQ_rr { - T: p.T, - S: p.S, - X: p.X, - Y: p.Y, - }) - } - - /* breakpoint */ - case *IrBreakpoint: { - bb.Ins = append(bb.Ins, &IrAMD64_INT { 3 }) - } - } - } - - /* lower the terminator */ - switch p := bb.Term.(type) { - default: { - panic("invalid terminator: " + bb.Term.String()) - } - - /* branch terminator */ - case *IrSwitch: { - switch t := p.iter().t; len(p.Br) { - case 0 : bb.Term = &IrAMD64_JMP { To: p.Ln } - case 1 : bb.Term = &IrAMD64_Jcc_ri { X: p.V, Y: t[0].i, To: p.Br[t[0].i], Ln: p.Ln, Op: IrAMD64_CmpEq } - default : break - } - } - - /* return terminator, ABI specific, will be lowered in later pass */ - case *IrReturn: { - break - } - } - }) + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + ins := bb.Ins + bb.Ins = make([]IrNode, 0, len(ins)) + + /* lower every instruction */ + for _, v := range ins { + switch p := v.(type) { + default: + { + bb.Ins = append(bb.Ins, p) + } + + /* load from memory */ + case *IrLoad: + { + bb.Ins = append(bb.Ins, &IrAMD64_MOV_load{ + R: p.R, + N: p.Size, + M: Mem{ + M: p.Mem, + I: Rz, + S: 1, + D: 0, + }, + }) + } + + /* store into memory */ + case *IrStore: + { + bb.Ins = append(bb.Ins, &IrAMD64_MOV_store_r{ + R: p.R, + N: p.Size, + M: Mem{ + M: p.Mem, + I: Rz, + S: 1, + D: 0, + }, + }) + } + + /* load constant into register */ + case *IrConstInt: + { + bb.Ins = append(bb.Ins, &IrAMD64_MOV_abs{ + R: p.R, + V: p.V, + }) + } + + /* load pointer constant into register */ + case *IrConstPtr: + { + bb.Ins = append(bb.Ins, &IrAMD64_MOV_ptr{ + R: p.R, + P: p.P, + }) + } + + /* load effective address */ + case *IrLEA: + { + bb.Ins = append(bb.Ins, &IrAMD64_LEA{ + R: p.R, + M: Mem{ + M: p.Mem, + I: p.Off, + S: 1, + D: 0, + }, + }) + } + + /* unary operators */ + case *IrUnaryExpr: + { + switch p.Op { + case IrOpNegate: + bb.Ins = append(bb.Ins, &IrAMD64_NEG{R: p.R, V: p.V}) + case IrOpSwap16: + bb.Ins = append(bb.Ins, &IrAMD64_BSWAP{R: p.R, V: p.V, N: 2}) + case IrOpSwap32: + bb.Ins = append(bb.Ins, &IrAMD64_BSWAP{R: p.R, V: p.V, N: 4}) + case IrOpSwap64: + bb.Ins = append(bb.Ins, &IrAMD64_BSWAP{R: p.R, V: p.V, N: 8}) + case IrOpSx32to64: + bb.Ins = append(bb.Ins, &IrAMD64_MOVSLQ{R: p.R, V: p.V}) + default: + panic("unreachable") + } + } + + /* binary operators */ + case *IrBinaryExpr: + { + switch p.Op { + case IrOpAdd: + bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr{R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinAdd}) + case IrOpSub: + bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr{R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinSub}) + case IrOpMul: + bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr{R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinMul}) + case IrOpAnd: + bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr{R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinAnd}) + case IrOpOr: + bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr{R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinOr}) + case IrOpXor: + bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr{R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinXor}) + case IrOpShr: + bb.Ins = append(bb.Ins, &IrAMD64_BinOp_rr{R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_BinShr}) + case IrCmpEq: + bb.Ins = append(bb.Ins, &IrAMD64_CMPQ_rr{R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_CmpEq}) + case IrCmpNe: + bb.Ins = append(bb.Ins, &IrAMD64_CMPQ_rr{R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_CmpNe}) + case IrCmpLt: + bb.Ins = append(bb.Ins, &IrAMD64_CMPQ_rr{R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_CmpLt}) + case IrCmpLtu: + bb.Ins = append(bb.Ins, &IrAMD64_CMPQ_rr{R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_CmpLtu}) + case IrCmpGeu: + bb.Ins = append(bb.Ins, &IrAMD64_CMPQ_rr{R: p.R, X: p.X, Y: p.Y, Op: IrAMD64_CmpGeu}) + default: + panic("unreachable") + } + } + + /* bit test and set */ + case *IrBitTestSet: + { + bb.Ins = append(bb.Ins, &IrAMD64_BTSQ_rr{ + T: p.T, + S: p.S, + X: p.X, + Y: p.Y, + }) + } + + /* breakpoint */ + case *IrBreakpoint: + { + bb.Ins = append(bb.Ins, &IrAMD64_INT{3}) + } + } + } + + /* lower the terminator */ + switch p := bb.Term.(type) { + default: + { + panic("invalid terminator: " + bb.Term.String()) + } + + /* branch terminator */ + case *IrSwitch: + { + switch t := p.iter().t; len(p.Br) { + case 0: + bb.Term = &IrAMD64_JMP{To: p.Ln} + case 1: + bb.Term = &IrAMD64_Jcc_ri{X: p.V, Y: t[0].i, To: p.Br[t[0].i], Ln: p.Ln, Op: IrAMD64_CmpEq} + default: + break + } + } + + /* return terminator, ABI specific, will be lowered in later pass */ + case *IrReturn: + { + break + } + } + }) } diff --git a/internal/atm/ssa/pass_mbarrier_amd64.go b/internal/atm/ssa/pass_mbarrier_amd64.go index 1437096..0daf987 100644 --- a/internal/atm/ssa/pass_mbarrier_amd64.go +++ b/internal/atm/ssa/pass_mbarrier_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,138 +17,138 @@ package ssa import ( - `sort` - `unsafe` + "sort" + "unsafe" - `github.com/cloudwego/frugal/internal/atm/abi` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/abi" + "github.com/cloudwego/frugal/internal/rt" ) // WriteBarrier inserts write barriers for pointer stores. type WriteBarrier struct{} func (WriteBarrier) Apply(cfg *CFG) { - more := true - mbir := make(map[*BasicBlock]int) - ptrs := make(map[Reg]unsafe.Pointer) - - /* find all constant pointers */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - for _, v := range bb.Ins { - if p, ok := v.(*IrAMD64_MOV_ptr); ok { - ptrs[p.R] = p.P - } - } - }) - - /* loop until no more write barriers */ - for more { - more = false - rt.MapClear(mbir) - - /* Phase 1: Find all the memory barriers and pointer constants */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - for i, v := range bb.Ins { - if _, ok := v.(*IrWriteBarrier); ok { - if _, ok = mbir[bb]; ok { - more = true - } else { - mbir[bb] = i - } - } - } - }) - - /* split pair buffer */ - nb := len(mbir) - mb := make([]Pos, 0, nb) - - /* extract from the map */ - for p, i := range mbir { - mb = append(mb, pos(p, i)) - } - - /* sort by block ID */ - sort.Slice(mb, func(i int, j int) bool { - return mb[i].isPriorTo(mb[j]) - }) - - /* Phase 2: Split basic block at write barrier */ - for _, p := range mb { - bb := cfg.CreateBlock() - ds := cfg.CreateBlock() - wb := cfg.CreateBlock() - ir := p.B.Ins[p.I].(*IrWriteBarrier) - - /* move instructions after the write barrier into a new block */ - bb.Ins = p.B.Ins[p.I + 1:] - bb.Term = p.B.Term - bb.Pred = []*BasicBlock { ds, wb } - - /* update all the predecessors & Phi nodes */ - for it := p.B.Term.Successors(); it.Next(); { - succ := it.Block() - pred := succ.Pred - - /* update predecessors */ - for x, v := range pred { - if v == p.B { - pred[x] = bb - break - } - } - - /* update Phi nodes */ - for _, phi := range succ.Phi { - phi.V[bb] = phi.V[p.B] - delete(phi.V, p.B) - } - } - - /* rewrite the direct store instruction */ - st := &IrAMD64_MOV_store_r { - R: ir.R, - M: Ptr(ir.M, 0), - N: abi.PtrSize, - } - - /* construct the direct store block */ - ds.Ins = []IrNode { st } - ds.Term = &IrAMD64_JMP { To: IrLikely(bb) } - ds.Pred = []*BasicBlock { p.B} - - /* rewrite the write barrier instruction */ - fn := &IrAMD64_CALL_gcwb { - R : ir.R, - M : ir.M, - Fn : ptrs[ir.Fn], - } - - /* function address must exist */ - if fn.Fn == nil { - panic("missing write barrier function address") - } - - /* construct the write barrier block */ - wb.Ins = []IrNode { fn } - wb.Term = &IrAMD64_JMP { To: IrLikely(bb) } - wb.Pred = []*BasicBlock { p.B} - - /* rewrite the terminator to check for write barrier */ - p.B.Ins = p.B.Ins[:p.I] - p.B.Term = &IrAMD64_Jcc_mi { - X : Ptr(ir.Var, 0), - Y : 0, - N : 1, - To : IrUnlikely(wb), - Ln : IrLikely(ds), - Op : IrAMD64_CmpNe, - } - } - - /* Phase 3: Rebuild the CFG */ - if len(mbir) != 0 { - cfg.Rebuild() - } - } + more := true + mbir := make(map[*BasicBlock]int) + ptrs := make(map[Reg]unsafe.Pointer) + + /* find all constant pointers */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + for _, v := range bb.Ins { + if p, ok := v.(*IrAMD64_MOV_ptr); ok { + ptrs[p.R] = p.P + } + } + }) + + /* loop until no more write barriers */ + for more { + more = false + rt.MapClear(mbir) + + /* Phase 1: Find all the memory barriers and pointer constants */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + for i, v := range bb.Ins { + if _, ok := v.(*IrWriteBarrier); ok { + if _, ok = mbir[bb]; ok { + more = true + } else { + mbir[bb] = i + } + } + } + }) + + /* split pair buffer */ + nb := len(mbir) + mb := make([]Pos, 0, nb) + + /* extract from the map */ + for p, i := range mbir { + mb = append(mb, pos(p, i)) + } + + /* sort by block ID */ + sort.Slice(mb, func(i int, j int) bool { + return mb[i].isPriorTo(mb[j]) + }) + + /* Phase 2: Split basic block at write barrier */ + for _, p := range mb { + bb := cfg.CreateBlock() + ds := cfg.CreateBlock() + wb := cfg.CreateBlock() + ir := p.B.Ins[p.I].(*IrWriteBarrier) + + /* move instructions after the write barrier into a new block */ + bb.Ins = p.B.Ins[p.I+1:] + bb.Term = p.B.Term + bb.Pred = []*BasicBlock{ds, wb} + + /* update all the predecessors & Phi nodes */ + for it := p.B.Term.Successors(); it.Next(); { + succ := it.Block() + pred := succ.Pred + + /* update predecessors */ + for x, v := range pred { + if v == p.B { + pred[x] = bb + break + } + } + + /* update Phi nodes */ + for _, phi := range succ.Phi { + phi.V[bb] = phi.V[p.B] + delete(phi.V, p.B) + } + } + + /* rewrite the direct store instruction */ + st := &IrAMD64_MOV_store_r{ + R: ir.R, + M: Ptr(ir.M, 0), + N: abi.PtrSize, + } + + /* construct the direct store block */ + ds.Ins = []IrNode{st} + ds.Term = &IrAMD64_JMP{To: IrLikely(bb)} + ds.Pred = []*BasicBlock{p.B} + + /* rewrite the write barrier instruction */ + fn := &IrAMD64_CALL_gcwb{ + R: ir.R, + M: ir.M, + Fn: ptrs[ir.Fn], + } + + /* function address must exist */ + if fn.Fn == nil { + panic("missing write barrier function address") + } + + /* construct the write barrier block */ + wb.Ins = []IrNode{fn} + wb.Term = &IrAMD64_JMP{To: IrLikely(bb)} + wb.Pred = []*BasicBlock{p.B} + + /* rewrite the terminator to check for write barrier */ + p.B.Ins = p.B.Ins[:p.I] + p.B.Term = &IrAMD64_Jcc_mi{ + X: Ptr(ir.Var, 0), + Y: 0, + N: 1, + To: IrUnlikely(wb), + Ln: IrLikely(ds), + Op: IrAMD64_CmpNe, + } + } + + /* Phase 3: Rebuild the CFG */ + if len(mbir) != 0 { + cfg.Rebuild() + } + } } diff --git a/internal/atm/ssa/pass_operandalloc_amd64.go b/internal/atm/ssa/pass_operandalloc_amd64.go index 07b7ff0..bde2305 100644 --- a/internal/atm/ssa/pass_operandalloc_amd64.go +++ b/internal/atm/ssa/pass_operandalloc_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,81 +21,88 @@ package ssa type OperandAlloc struct{} func (OperandAlloc) Apply(cfg *CFG) { - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - ins := bb.Ins - bb.Ins = make([]IrNode, 0, len(ins)) + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + ins := bb.Ins + bb.Ins = make([]IrNode, 0, len(ins)) - /* check for every instruction */ - for _, v := range ins { - switch p := v.(type) { - default: { - bb.Ins = append(bb.Ins, v) - } + /* check for every instruction */ + for _, v := range ins { + switch p := v.(type) { + default: + { + bb.Ins = append(bb.Ins, v) + } - /* negation */ - case *IrAMD64_NEG: { - if p.R == p.V { - bb.Ins = append(bb.Ins, v) - } else { - bb.Ins, p.V = append(bb.Ins, IrArchCopy(p.R, p.V), v), p.R - } - } + /* negation */ + case *IrAMD64_NEG: + { + if p.R == p.V { + bb.Ins = append(bb.Ins, v) + } else { + bb.Ins, p.V = append(bb.Ins, IrArchCopy(p.R, p.V), v), p.R + } + } - /* byte swap */ - case *IrAMD64_BSWAP: { - if p.R == p.V { - bb.Ins = append(bb.Ins, v) - } else { - bb.Ins, p.V = append(bb.Ins, IrArchCopy(p.R, p.V), v), p.R - } - } + /* byte swap */ + case *IrAMD64_BSWAP: + { + if p.R == p.V { + bb.Ins = append(bb.Ins, v) + } else { + bb.Ins, p.V = append(bb.Ins, IrArchCopy(p.R, p.V), v), p.R + } + } - /* binary operations, register to register */ - case *IrAMD64_BinOp_rr: { - if p.R == p.X { - bb.Ins = append(bb.Ins, v) - } else { - bb.Ins, p.X = append(bb.Ins, IrArchCopy(p.R, p.X), v), p.R - } - } + /* binary operations, register to register */ + case *IrAMD64_BinOp_rr: + { + if p.R == p.X { + bb.Ins = append(bb.Ins, v) + } else { + bb.Ins, p.X = append(bb.Ins, IrArchCopy(p.R, p.X), v), p.R + } + } - /* binary operations, register to immediate */ - case *IrAMD64_BinOp_ri: { - if p.R == p.X || p.Op == IrAMD64_BinMul { - bb.Ins = append(bb.Ins, v) - } else { - bb.Ins, p.X = append(bb.Ins, IrArchCopy(p.R, p.X), v), p.R - } - } + /* binary operations, register to immediate */ + case *IrAMD64_BinOp_ri: + { + if p.R == p.X || p.Op == IrAMD64_BinMul { + bb.Ins = append(bb.Ins, v) + } else { + bb.Ins, p.X = append(bb.Ins, IrArchCopy(p.R, p.X), v), p.R + } + } - /* binary operations, register to memory */ - case *IrAMD64_BinOp_rm: { - if p.R == p.X { - bb.Ins = append(bb.Ins, v) - } else { - bb.Ins, p.X = append(bb.Ins, IrArchCopy(p.R, p.X), v), p.R - } - } + /* binary operations, register to memory */ + case *IrAMD64_BinOp_rm: + { + if p.R == p.X { + bb.Ins = append(bb.Ins, v) + } else { + bb.Ins, p.X = append(bb.Ins, IrArchCopy(p.R, p.X), v), p.R + } + } - /* bit test and set, register to register */ - case *IrAMD64_BTSQ_rr: { - if p.S == p.X { - bb.Ins = append(bb.Ins, v) - } else { - bb.Ins, p.X = append(bb.Ins, IrArchCopy(p.S, p.X), v), p.S - } - } + /* bit test and set, register to register */ + case *IrAMD64_BTSQ_rr: + { + if p.S == p.X { + bb.Ins = append(bb.Ins, v) + } else { + bb.Ins, p.X = append(bb.Ins, IrArchCopy(p.S, p.X), v), p.S + } + } - /* bit test and set, register to immediate */ - case *IrAMD64_BTSQ_ri: { - if p.S == p.X { - bb.Ins = append(bb.Ins, v) - } else { - bb.Ins, p.X = append(bb.Ins, IrArchCopy(p.S, p.X), v), p.S - } - } - } - } - }) + /* bit test and set, register to immediate */ + case *IrAMD64_BTSQ_ri: + { + if p.S == p.X { + bb.Ins = append(bb.Ins, v) + } else { + bb.Ins, p.X = append(bb.Ins, IrArchCopy(p.S, p.X), v), p.S + } + } + } + } + }) } - diff --git a/internal/atm/ssa/pass_phielim.go b/internal/atm/ssa/pass_phielim.go index ebeeae5..73fcdb4 100644 --- a/internal/atm/ssa/pass_phielim.go +++ b/internal/atm/ssa/pass_phielim.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,98 +17,98 @@ package ssa import ( - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) type _RegSrc struct { - r Reg - p IrNode + r Reg + p IrNode } // PhiElim transforms Phi nodes into copies if possible. type PhiElim struct{} func (self PhiElim) addSource(src map[_RegSrc]struct{}, phi *IrPhi, defs map[Reg]IrNode, visited map[*IrPhi]struct{}) { - var ok bool - var pp *IrPhi - var def IrNode - - /* circles back to itself */ - if _, ok = visited[phi]; ok { - return - } else { - visited[phi] = struct{}{} - } - - /* add definitions for this node */ - for _, r := range phi.V { - if r.Kind() == K_zero { - src[_RegSrc { r: *r, p: nil }] = struct{}{} - } else if def, ok = defs[*r]; !ok { - panic("phixform: undefined register: " + r.String()) - } else if pp, ok = def.(*IrPhi); ok { - self.addSource(src, pp, defs, visited) - } else { - src[_RegSrc { r: *r, p: def }] = struct{}{} - } - } + var ok bool + var pp *IrPhi + var def IrNode + + /* circles back to itself */ + if _, ok = visited[phi]; ok { + return + } else { + visited[phi] = struct{}{} + } + + /* add definitions for this node */ + for _, r := range phi.V { + if r.Kind() == K_zero { + src[_RegSrc{r: *r, p: nil}] = struct{}{} + } else if def, ok = defs[*r]; !ok { + panic("phixform: undefined register: " + r.String()) + } else if pp, ok = def.(*IrPhi); ok { + self.addSource(src, pp, defs, visited) + } else { + src[_RegSrc{r: *r, p: def}] = struct{}{} + } + } } func (self PhiElim) Apply(cfg *CFG) { - buf := make(map[Reg]IrNode) - vis := make(map[*IrPhi]struct{}) - src := make(map[_RegSrc]struct{}) - - /* scan for all instruction usages and register definitions */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - var ok bool - var def IrDefinitions - - /* mark all Phi definitions */ - for _, v := range bb.Phi { - buf[v.R] = v - } - - /* scan instructions */ - for _, v := range bb.Ins { - if def, ok = v.(IrDefinitions); ok { - for _, r := range def.Definitions() { - buf[*r] = v - } - } - } - }) - - /* scan for unused Phi nodes */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - var p *IrPhi - var ins []IrNode - - /* filter the Phi nodes */ - phi := bb.Phi - bb.Phi = bb.Phi[:0] - - /* check all Phi nodes */ - for _, p = range phi { - rt.MapClear(src) - rt.MapClear(vis) - - /* resolve all the value sources */ - if self.addSource(src, p, buf, vis); len(src) != 1 { - bb.Phi = append(bb.Phi, p) - continue - } - - /* all values come from a single source */ - for s := range src { - ins = append(ins, IrCopy(p.R, s.r)) - break - } - } - - /* patch instructions if needed */ - if len(ins) != 0 { - bb.Ins = append(ins, bb.Ins...) - } - }) + buf := make(map[Reg]IrNode) + vis := make(map[*IrPhi]struct{}) + src := make(map[_RegSrc]struct{}) + + /* scan for all instruction usages and register definitions */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + var ok bool + var def IrDefinitions + + /* mark all Phi definitions */ + for _, v := range bb.Phi { + buf[v.R] = v + } + + /* scan instructions */ + for _, v := range bb.Ins { + if def, ok = v.(IrDefinitions); ok { + for _, r := range def.Definitions() { + buf[*r] = v + } + } + } + }) + + /* scan for unused Phi nodes */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + var p *IrPhi + var ins []IrNode + + /* filter the Phi nodes */ + phi := bb.Phi + bb.Phi = bb.Phi[:0] + + /* check all Phi nodes */ + for _, p = range phi { + rt.MapClear(src) + rt.MapClear(vis) + + /* resolve all the value sources */ + if self.addSource(src, p, buf, vis); len(src) != 1 { + bb.Phi = append(bb.Phi, p) + continue + } + + /* all values come from a single source */ + for s := range src { + ins = append(ins, IrCopy(p.R, s.r)) + break + } + } + + /* patch instructions if needed */ + if len(ins) != 0 { + bb.Ins = append(ins, bb.Ins...) + } + }) } diff --git a/internal/atm/ssa/pass_phiprop.go b/internal/atm/ssa/pass_phiprop.go index 57ffde1..6940187 100644 --- a/internal/atm/ssa/pass_phiprop.go +++ b/internal/atm/ssa/pass_phiprop.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,22 +17,22 @@ package ssa import ( - `fmt` + "fmt" - `github.com/cloudwego/frugal/internal/rt` - `gonum.org/v1/gonum/graph` - `gonum.org/v1/gonum/graph/simple` - `gonum.org/v1/gonum/graph/topo` + "github.com/cloudwego/frugal/internal/rt" + "gonum.org/v1/gonum/graph" + "gonum.org/v1/gonum/graph/simple" + "gonum.org/v1/gonum/graph/topo" ) const ( - _W_likely = 7.0 / 8.0 // must be exactly representable to avoid float precision loss - _W_unlikely = 1.0 - _W_likely + _W_likely = 7.0 / 8.0 // must be exactly representable to avoid float precision loss + _W_unlikely = 1.0 - _W_likely ) -var _WeightTab = [...]float64 { - Likely : _W_likely, - Unlikely : _W_unlikely, +var _WeightTab = [...]float64{ + Likely: _W_likely, + Unlikely: _W_unlikely, } // PhiProp propagates Phi nodes into it's source blocks, @@ -41,170 +41,176 @@ var _WeightTab = [...]float64 { type PhiProp struct{} func (self PhiProp) dfs(dag *simple.DirectedGraph, bb *BasicBlock, vis map[int]*BasicBlock, path map[int]struct{}) { - vis[bb.Id] = bb - path[bb.Id] = struct{}{} - - /* traverse all the sucessors */ - for it := bb.Term.Successors(); it.Next(); { - v := it.Block() - s, d := bb.Id, v.Id - - /* back edge */ - if _, ok := path[d]; ok { - continue - } - - /* forward or cross edge */ - p, _ := dag.NodeWithID(int64(s)) - q, _ := dag.NodeWithID(int64(d)) - dag.SetEdge(dag.NewEdge(p, q)) - - /* visit the successor if not already */ - if _, ok := vis[d]; !ok { - self.dfs(dag, v, vis, path) - } - } - - /* remove the node from path */ - if _, ok := path[bb.Id]; !ok { - panic("phiprop: corrupted DFS stack") - } else { - delete(path, bb.Id) - } + vis[bb.Id] = bb + path[bb.Id] = struct{}{} + + /* traverse all the successors */ + for it := bb.Term.Successors(); it.Next(); { + v := it.Block() + s, d := bb.Id, v.Id + + /* back edge */ + if _, ok := path[d]; ok { + continue + } + + /* forward or cross edge */ + p, _ := dag.NodeWithID(int64(s)) + q, _ := dag.NodeWithID(int64(d)) + dag.SetEdge(dag.NewEdge(p, q)) + + /* visit the successor if not already */ + if _, ok := vis[d]; !ok { + self.dfs(dag, v, vis, path) + } + } + + /* remove the node from path */ + if _, ok := path[bb.Id]; !ok { + panic("phiprop: corrupted DFS stack") + } else { + delete(path, bb.Id) + } } func (self PhiProp) Apply(cfg *CFG) { - var err error - var ord []graph.Node - - /* convert to DAG by removing back edges (assuming they never takes) */ - // FIXME: this might cause inaccuracy, loops don't affect path probabilities - // if they are looked as a whole. - dag := simple.NewDirectedGraph() - bbs := make(map[int]*BasicBlock, cfg.MaxBlock()) - self.dfs(dag, cfg.Root, bbs, make(map[int]struct{}, cfg.MaxBlock())) - - /* topologically sort the DAG */ - if ord, err = topo.Sort(dag); err != nil { - panic("phiprop: topology sort: " + err.Error()) - } - - /* weight from block to another block */ - subs := make(map[Reg]Reg) - bias := make(map[int]float64) - weight := make(map[int]map[int]float64, cfg.MaxBlock()) - - /* calculate block weights in topological order */ - for _, p := range ord { - var in float64 - var sum float64 - - /* find the basic block */ - id := p.ID() - bb := bbs[int(id)] - tr := bb.Term.Successors() - - /* special case for root node */ - if len(bb.Pred) == 0 { - in = 1.0 - } - - /* add all the incoming weights */ - for _, v := range bb.Pred { - if dag.HasEdgeFromTo(int64(v.Id), id) { - in += weight[v.Id][bb.Id] - } - } - - /* allocate the output probability map */ - weight[bb.Id] = make(map[int]float64) - rt.MapClear(bias) - - /* calculate the output bias factor */ - for tr.Next() { - if vv := tr.Block(); dag.HasEdgeFromTo(id, int64(vv.Id)) { - w := _WeightTab[tr.Likeliness()] - sum += w - bias[vv.Id] = w - } - } - - /* bias the weight with the bias factor */ - for i, v := range bias { - weight[bb.Id][i] = in * (v / sum) - } - } - - /* choose the register with highest probability as the "primary" register */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - for _, p := range bb.Phi { - var rs Reg - var ps float64 - var pp float64 - - /* find the branch with the hightest probability */ - for b, r := range p.V { - if pp = weight[b.Id][bb.Id]; ps < pp { - rs = *r - ps = pp - } - } - - /* mark the substitution */ - if _, ok := subs[p.R]; ok { - panic(fmt.Sprintf("phiprop: duplicated substitution: %s -> %s", p.R, rs)) - } else { - subs[p.R] = rs - } - } - }) - - /* register substitution routine */ - substitute := func(rr []*Reg) { - for _, r := range rr { - if d, ok := subs[*r]; ok { - for subs[d] != 0 { d = subs[d] } - *r = d - } - } - } - - /* substitute every register */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - var ok bool - var use IrUsages - var def IrDefinitions - - /* process Phi nodes */ - for _, v := range bb.Phi { - substitute(v.Usages()) - substitute(v.Definitions()) - } - - /* process instructions */ - for _, v := range bb.Ins { - if use, ok = v.(IrUsages) ; ok { substitute(use.Usages()) } - if def, ok = v.(IrDefinitions) ; ok { substitute(def.Definitions()) } - } - - /* process the terminator */ - if use, ok = bb.Term.(IrUsages); ok { - substitute(use.Usages()) - } - }) - - /* propagate Phi nodes upward */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - pp := bb.Phi - bb.Phi = nil - - /* process every Phi node */ - for _, p := range pp { - for b, r := range p.V { - if *r != p.R { - b.Ins = append(b.Ins, IrArchCopy(p.R, *r)) - } - } - } - }) + var err error + var ord []graph.Node + + /* convert to DAG by removing back edges (assuming they never takes) */ + // FIXME: this might cause inaccuracy, loops don't affect path probabilities + // if they are looked as a whole. + dag := simple.NewDirectedGraph() + bbs := make(map[int]*BasicBlock, cfg.MaxBlock()) + self.dfs(dag, cfg.Root, bbs, make(map[int]struct{}, cfg.MaxBlock())) + + /* topologically sort the DAG */ + if ord, err = topo.Sort(dag); err != nil { + panic("phiprop: topology sort: " + err.Error()) + } + + /* weight from block to another block */ + subs := make(map[Reg]Reg) + bias := make(map[int]float64) + weight := make(map[int]map[int]float64, cfg.MaxBlock()) + + /* calculate block weights in topological order */ + for _, p := range ord { + var in float64 + var sum float64 + + /* find the basic block */ + id := p.ID() + bb := bbs[int(id)] + tr := bb.Term.Successors() + + /* special case for root node */ + if len(bb.Pred) == 0 { + in = 1.0 + } + + /* add all the incoming weights */ + for _, v := range bb.Pred { + if dag.HasEdgeFromTo(int64(v.Id), id) { + in += weight[v.Id][bb.Id] + } + } + + /* allocate the output probability map */ + weight[bb.Id] = make(map[int]float64) + rt.MapClear(bias) + + /* calculate the output bias factor */ + for tr.Next() { + if vv := tr.Block(); dag.HasEdgeFromTo(id, int64(vv.Id)) { + w := _WeightTab[tr.Likeliness()] + sum += w + bias[vv.Id] = w + } + } + + /* bias the weight with the bias factor */ + for i, v := range bias { + weight[bb.Id][i] = in * (v / sum) + } + } + + /* choose the register with highest probability as the "primary" register */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + for _, p := range bb.Phi { + var rs Reg + var ps float64 + var pp float64 + + /* find the branch with the highest probability */ + for b, r := range p.V { + if pp = weight[b.Id][bb.Id]; ps < pp { + rs = *r + ps = pp + } + } + + /* mark the substitution */ + if _, ok := subs[p.R]; ok { + panic(fmt.Sprintf("phiprop: duplicated substitution: %s -> %s", p.R, rs)) + } else { + subs[p.R] = rs + } + } + }) + + /* register substitution routine */ + substitute := func(rr []*Reg) { + for _, r := range rr { + if d, ok := subs[*r]; ok { + for subs[d] != 0 { + d = subs[d] + } + *r = d + } + } + } + + /* substitute every register */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + var ok bool + var use IrUsages + var def IrDefinitions + + /* process Phi nodes */ + for _, v := range bb.Phi { + substitute(v.Usages()) + substitute(v.Definitions()) + } + + /* process instructions */ + for _, v := range bb.Ins { + if use, ok = v.(IrUsages); ok { + substitute(use.Usages()) + } + if def, ok = v.(IrDefinitions); ok { + substitute(def.Definitions()) + } + } + + /* process the terminator */ + if use, ok = bb.Term.(IrUsages); ok { + substitute(use.Usages()) + } + }) + + /* propagate Phi nodes upward */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + pp := bb.Phi + bb.Phi = nil + + /* process every Phi node */ + for _, p := range pp { + for b, r := range p.V { + if *r != p.R { + b.Ins = append(b.Ins, IrArchCopy(p.R, *r)) + } + } + } + }) } diff --git a/internal/atm/ssa/pass_reduce.go b/internal/atm/ssa/pass_reduce.go index 7b150e2..9d3a25d 100644 --- a/internal/atm/ssa/pass_reduce.go +++ b/internal/atm/ssa/pass_reduce.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,16 +19,15 @@ package ssa // Reduce combines CSE, PhiElim, CopyElim and TDCE. type Reduce struct{} -var Reductions = []PassDescriptor { - { Name: "Common Sub-expression Elimination" , Pass: new(CSE) }, - { Name: "Phi Elimination" , Pass: new(PhiElim) }, - { Name: "Copy Elimination" , Pass: new(CopyElim) }, - { Name: "Trivial Dead Code Elimination" , Pass: new(TDCE) }, +var Reductions = []PassDescriptor{ + {Name: "Common Sub-expression Elimination", Pass: new(CSE)}, + {Name: "Phi Elimination", Pass: new(PhiElim)}, + {Name: "Copy Elimination", Pass: new(CopyElim)}, + {Name: "Trivial Dead Code Elimination", Pass: new(TDCE)}, } func (Reduce) Apply(cfg *CFG) { - for _, r := range Reductions { - r.Pass.Apply(cfg) - } + for _, r := range Reductions { + r.Pass.Apply(cfg) + } } - diff --git a/internal/atm/ssa/pass_regalloc.go b/internal/atm/ssa/pass_regalloc.go index 8fe1f8b..97d8a7c 100644 --- a/internal/atm/ssa/pass_regalloc.go +++ b/internal/atm/ssa/pass_regalloc.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,854 +17,888 @@ package ssa import ( - `fmt` - `sort` - `strings` - `unsafe` - - `github.com/cloudwego/frugal/internal/rt` - `gonum.org/v1/gonum/graph/coloring` - `gonum.org/v1/gonum/graph/simple` + "fmt" + "sort" + "strings" + "unsafe" + + "github.com/cloudwego/frugal/internal/rt" + "gonum.org/v1/gonum/graph/coloring" + "gonum.org/v1/gonum/graph/simple" ) type ( - _RegSet map[Reg]struct{} + _RegSet map[Reg]struct{} ) func (self _RegSet) add(r Reg) _RegSet { - self[r] = struct{}{} - return self + self[r] = struct{}{} + return self } func (self _RegSet) union(rs _RegSet) { - for r := range rs { - self.add(r) - } + for r := range rs { + self.add(r) + } } func (self _RegSet) remove(r Reg) { - delete(self, r) + delete(self, r) } func (self _RegSet) subtract(rs _RegSet) { - for r := range rs { - self.remove(r) - } + for r := range rs { + self.remove(r) + } } func (self _RegSet) toslice() []Reg { - nb := len(self) - rr := make([]Reg, 0, nb) + nb := len(self) + rr := make([]Reg, 0, nb) - /* extract all registers */ - for r := range self { - rr = append(rr, r) - } + /* extract all registers */ + for r := range self { + rr = append(rr, r) + } - /* sort by register ID */ - sort.Slice(rr, func(i int, j int) bool { return rr[i] < rr[j] }) - return rr + /* sort by register ID */ + sort.Slice(rr, func(i int, j int) bool { return rr[i] < rr[j] }) + return rr } func (self _RegSet) contains(r Reg) (ret bool) { - _, ret = self[r] - return + _, ret = self[r] + return } func (self _RegSet) String() string { - nb := len(self) - rs := make([]string, 0, nb) - - /* convert every register */ - for _, r := range self.toslice() { - rs = append(rs, r.String()) - } - - /* join them together */ - return fmt.Sprintf( - "{%s}", - strings.Join(rs, ", "), - ) + nb := len(self) + rs := make([]string, 0, nb) + + /* convert every register */ + for _, r := range self.toslice() { + rs = append(rs, r.String()) + } + + /* join them together */ + return fmt.Sprintf( + "{%s}", + strings.Join(rs, ", "), + ) } type _RegTabPair struct { - rr Reg - rs _RegSet + rr Reg + rs _RegSet } type _RegTab struct { - p []_RegSet - m map[Reg]_RegSet + p []_RegSet + m map[Reg]_RegSet } func mkregtab() *_RegTab { - return &_RegTab { - p: make([]_RegSet, 0, 16), - m: make(map[Reg]_RegSet, len(ArchRegs)), - } + return &_RegTab{ + p: make([]_RegSet, 0, 16), + m: make(map[Reg]_RegSet, len(ArchRegs)), + } } func (self *_RegTab) reset() { - for k, v := range self.m { - self.p = append(self.p, v) - delete(self.m, k) - rt.MapClear(v) - } + for k, v := range self.m { + self.p = append(self.p, v) + delete(self.m, k) + rt.MapClear(v) + } } func (self *_RegTab) pairs() (r []_RegTabPair) { - r = make([]_RegTabPair, 0, len(self.m)) - for rr, rs := range self.m { r = append(r, _RegTabPair { rr, rs }) } - sort.Slice(r, func(i int, j int) bool { return r[i].rr < r[j].rr }) - return + r = make([]_RegTabPair, 0, len(self.m)) + for rr, rs := range self.m { + r = append(r, _RegTabPair{rr, rs}) + } + sort.Slice(r, func(i int, j int) bool { return r[i].rr < r[j].rr }) + return } func (self *_RegTab) alloc(n int) (rs _RegSet) { - if p := len(self.p); p == 0 { - rs = make(_RegSet, n) - return - } else { - rs, self.p = self.p[p - 1], self.p[:p - 1] - return - } + if p := len(self.p); p == 0 { + rs = make(_RegSet, n) + return + } else { + rs, self.p = self.p[p-1], self.p[:p-1] + return + } } func (self *_RegTab) clone(s _RegSet) (rs _RegSet) { - rs = self.alloc(len(s)) - for r := range s { rs.add(r) } - return + rs = self.alloc(len(s)) + for r := range s { + rs.add(r) + } + return } func (self *_RegTab) mkset(r ...Reg) (rs _RegSet) { - rs = self.alloc(len(r)) - for _, v := range r { rs.add(v) } - return + rs = self.alloc(len(r)) + for _, v := range r { + rs.add(v) + } + return } func (self *_RegTab) mksetp(r []*Reg) (rs _RegSet) { - rs = self.alloc(len(r)) - for _, v := range r { rs.add(*v) } - return + rs = self.alloc(len(r)) + for _, v := range r { + rs.add(*v) + } + return } func (self *_RegTab) relate(k Reg, v Reg) { - if p, ok := self.m[k]; ok { - p.add(v) - } else { - self.m[k] = self.alloc(1).add(v) - } + if p, ok := self.m[k]; ok { + p.add(v) + } else { + self.m[k] = self.alloc(1).add(v) + } } func (self *_RegTab) remove(r Reg) (rs _RegSet) { - rs = self.m[r] - delete(self.m, r) - return + rs = self.m[r] + delete(self.m, r) + return } // RegAlloc performs register allocation on CFG. type RegAlloc struct{} func (self RegAlloc) livein(p *_RegTab, lr map[Pos]_RegSet, bb *BasicBlock, in map[int]_RegSet, out map[int]_RegSet) _RegSet { - var ok bool - var rs _RegSet - var use IrUsages - var def IrDefinitions - - /* check for cached live-in sets */ - if rs, ok = in[bb.Id]; ok { - return p.clone(rs) - } - - /* calculate the live-out set of current block */ - tr := bb.Term - regs := p.clone(self.liveout(p, lr, bb, in, out)) - - /* assume all terminators are non-definitive */ - if _, ok = tr.(IrDefinitions); ok { - panic("regalloc: definitions within terminators") - } - - /* add the terminator usages if any */ - if use, ok = tr.(IrUsages); ok { - regs.union(p.mksetp(use.Usages())) - } - - /* mark live range of the terminator */ - rr := p.clone(regs) - lr[pos(bb, _P_term)] = rr - - /* live(i-1) = use(i) ∪ (live(i) - { def(i) }) */ - for i := len(bb.Ins) - 1; i >= 0; i-- { - if def, ok = bb.Ins[i].(IrDefinitions) ; ok { regs.subtract(p.mksetp(def.Definitions())) } - if use, ok = bb.Ins[i].(IrUsages) ; ok { regs.union(p.mksetp(use.Usages())) } - lr[pos(bb, i)] = p.clone(regs) - } - - /* should not have any Phi nodes */ - if len(bb.Phi) != 0 { - panic("regalloc: unexpected Phi nodes") - } - - /* update the cache */ - in[bb.Id] = p.clone(regs) - return regs + var ok bool + var rs _RegSet + var use IrUsages + var def IrDefinitions + + /* check for cached live-in sets */ + if rs, ok = in[bb.Id]; ok { + return p.clone(rs) + } + + /* calculate the live-out set of current block */ + tr := bb.Term + regs := p.clone(self.liveout(p, lr, bb, in, out)) + + /* assume all terminators are non-definitive */ + if _, ok = tr.(IrDefinitions); ok { + panic("regalloc: definitions within terminators") + } + + /* add the terminator usages if any */ + if use, ok = tr.(IrUsages); ok { + regs.union(p.mksetp(use.Usages())) + } + + /* mark live range of the terminator */ + rr := p.clone(regs) + lr[pos(bb, _P_term)] = rr + + /* live(i-1) = use(i) ∪ (live(i) - { def(i) }) */ + for i := len(bb.Ins) - 1; i >= 0; i-- { + if def, ok = bb.Ins[i].(IrDefinitions); ok { + regs.subtract(p.mksetp(def.Definitions())) + } + if use, ok = bb.Ins[i].(IrUsages); ok { + regs.union(p.mksetp(use.Usages())) + } + lr[pos(bb, i)] = p.clone(regs) + } + + /* should not have any Phi nodes */ + if len(bb.Phi) != 0 { + panic("regalloc: unexpected Phi nodes") + } + + /* update the cache */ + in[bb.Id] = p.clone(regs) + return regs } func (self RegAlloc) liveout(p *_RegTab, lr map[Pos]_RegSet, bb *BasicBlock, in map[int]_RegSet, out map[int]_RegSet) _RegSet { - var ok bool - var rr []Reg - var rs _RegSet - - /* check for cached live-out sets */ - if rs, ok = out[bb.Id]; ok { - return rs - } - - /* check for return blocks */ - if rr, ok = IrTryIntoArchReturn(bb.Term); ok { - rs = p.mkset(rr...) - out[bb.Id] = rs - return rs - } - - /* create a new register set */ - rs = p.alloc(0) - it := bb.Term.Successors() - - /* live-out(p) = ∑(live-in(succ(p))) */ - for out[bb.Id] = nil; it.Next(); { - rs.union(self.livein(p, lr, it.Block(), in, out)) - } - - /* update cache */ - out[bb.Id] = rs - return rs + var ok bool + var rr []Reg + var rs _RegSet + + /* check for cached live-out sets */ + if rs, ok = out[bb.Id]; ok { + return rs + } + + /* check for return blocks */ + if rr, ok = IrTryIntoArchReturn(bb.Term); ok { + rs = p.mkset(rr...) + out[bb.Id] = rs + return rs + } + + /* create a new register set */ + rs = p.alloc(0) + it := bb.Term.Successors() + + /* live-out(p) = ∑(live-in(succ(p))) */ + for out[bb.Id] = nil; it.Next(); { + rs.union(self.livein(p, lr, it.Block(), in, out)) + } + + /* update cache */ + out[bb.Id] = rs + return rs } /* try to choose a different color from reloadRegs */ func (self RegAlloc) colorDiffWithReload(rig *simple.UndirectedGraph, reg Reg, reloadReg map[Reg]int, arch []Reg, colormap map[Reg]int) { - sameWithReload := false - for _, c := range reloadReg { - if c == colormap[reg] { - sameWithReload = true - break - } - } - if !sameWithReload { return } - - /* all possible colors */ - colors := make(map[int]struct{}) - for i := range arch { - colors[i] = struct{}{} - } - - /* choose a different color from it's neightbors */ - for r := rig.From(int64(reg)); r.Next(); { - delete(colors, colormap[Reg(r.Node().ID())]) - } - - /* choose a different color from reloadRegs */ - for _, v := range reloadReg { - delete(colors, v) - } - - if len(colors) > 0 { - /* there have some other colors different with reload regs */ - for c := range colors { - colormap[reg] = c - break - } - } + sameWithReload := false + for _, c := range reloadReg { + if c == colormap[reg] { + sameWithReload = true + break + } + } + if !sameWithReload { + return + } + + /* all possible colors */ + colors := make(map[int]struct{}) + for i := range arch { + colors[i] = struct{}{} + } + + /* choose a different color from it's neighbors */ + for r := rig.From(int64(reg)); r.Next(); { + delete(colors, colormap[Reg(r.Node().ID())]) + } + + /* choose a different color from reloadRegs */ + for _, v := range reloadReg { + delete(colors, v) + } + + if len(colors) > 0 { + /* there have some other colors different with reload regs */ + for c := range colors { + colormap[reg] = c + break + } + } } /* try to choose the same color for reloadReg as reg */ func (self RegAlloc) colorSameWithReg(rig *simple.UndirectedGraph, reloadReg Reg, arch []Reg, colormap map[Reg]int, reg Reg) { - /* all possible colors */ - colors := make(map[int]struct{}) - for i := range arch { - colors[i] = struct{}{} - } - - /* choose a different color from it's neightbors */ - for r := rig.From(int64(reloadReg)); r.Next(); { - delete(colors, colormap[Reg(r.Node().ID())]) - } - - /* try to choose the same color with reg */ - if _, ok := colors[colormap[reg]]; ok { - colormap[reloadReg] = colormap[reg] - } + /* all possible colors */ + colors := make(map[int]struct{}) + for i := range arch { + colors[i] = struct{}{} + } + + /* choose a different color from it's neighbors */ + for r := rig.From(int64(reloadReg)); r.Next(); { + delete(colors, colormap[Reg(r.Node().ID())]) + } + + /* try to choose the same color with reg */ + if _, ok := colors[colormap[reg]]; ok { + colormap[reloadReg] = colormap[reg] + } } func (self RegAlloc) Apply(cfg *CFG) { - var rig *simple.UndirectedGraph - var pass int - var arch []Reg - var colormap map[Reg]int - - /* reusable state */ - pool := mkregtab() - regmap := make(map[int]Reg) - livein := make(map[int]_RegSet) - liveout := make(map[int]_RegSet) - liveset := make(map[Pos]_RegSet) - archcolors := make(map[int64]int, len(ArchRegs)) - coalescemap := make(map[Reg]Reg) - invcoalescemap := make(map[Reg][]Reg) - - /* register coalescer */ - coalesce := func(rr []*Reg) { - for _, r := range rr { - if c, ok := coalescemap[*r]; ok { - *r = c - } - } - } - - /* calculate allocatable registers */ - for _, r := range ArchRegs { - if !ArchRegReserved[r] { - arch = append(arch, IrSetArch(Rz, r)) - } - } - - /* allocate colors to the registers */ - for i, r := range arch { - archcolors[int64(r)] = i - } - - /* register precolorer */ - precolor := func(rr []*Reg) { - for _, r := range rr { - if r.Kind() == K_arch { - *r = IrSetArch(Rz, ArchRegs[r.Name()]) - } - } - } - - /* precolor all physical registers */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - ok := false - use := IrUsages(nil) - def := IrDefinitions(nil) - - /* scan all the instructions */ - for _, v := range bb.Ins { - if use, ok = v.(IrUsages) ; ok { precolor(use.Usages()) } - if def, ok = v.(IrDefinitions) ; ok { precolor(def.Definitions()) } - } - - /* scan the terminator */ - if use, ok = bb.Term.(IrUsages); ok { - precolor(use.Usages()) - } - }) - - /* loop until no more retries */ - for { - pool.reset() - rt.MapClear(livein) - rt.MapClear(liveout) - rt.MapClear(liveset) - rt.MapClear(coalescemap) - rt.MapClear(invcoalescemap) - - /* expand operands but not in the first pass */ - if pass != 0 { - new(OperandAlloc).Apply(cfg) - } - - /* Phase 1: Calculate live ranges */ - lr := self.livein(pool, liveset, cfg.Root, livein, liveout) - rig = simple.NewUndirectedGraph() - - /* sanity check: no registers live at the entry point */ - if len(lr) != 0 { - panic("regalloc: live registers at entry: " + lr.String()) - } - - /* Phase 2: Build register interference graph */ - for _, rs := range liveset { - rr := rs.toslice() - nr := len(rr) - - /* special case of a single live register */ - if nr == 1 && rig.Node(int64(rr[0])) == nil { - p, _ := rig.NodeWithID(int64(rr[0])) - rig.AddNode(p) - continue - } - - /* create every edge */ - for i := 0; i < nr - 1; i++ { - for j := i + 1; j < nr; j++ { - p, _ := rig.NodeWithID(int64(rr[i])) - q, _ := rig.NodeWithID(int64(rr[j])) - rig.SetEdge(rig.NewEdge(p, q)) - } - } - } - - /* make sure all physical registers are in the RIG */ - for _, r := range arch { - if p, ok := rig.NodeWithID(int64(r)); ok { - rig.AddNode(p) - } - } - - /* Phase 3: Coalescing registers, find out all the coalescing pairs */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - for _, v := range bb.Ins { - var r0 Reg - var r1 Reg - var rx Reg - var ry Reg - var ok bool - - /* only interested in copy instructions */ - if rx, ry, ok = IrArchTryIntoCopy(v); !ok { - continue - } - - /* locate the coalescing source register */ - if r0, ok = coalescemap[rx]; ok { rx = r0 } - if r1, ok = coalescemap[ry]; ok { ry = r1 } - - /* check if the two registers can be coalesced */ - if rx == ry || rig.HasEdgeBetween(int64(rx), int64(ry)) || (rx.Kind() == K_arch && ry.Kind() == K_arch) { - continue - } - - /* make sure Y is the node with a lower degree */ - if rig.From(int64(rx)).Len() < rig.From(int64(ry)).Len() { - rx, ry = ry, rx - } - - /* all the adjacent nodes of Y */ - p := rig.From(int64(ry)) - ok = true - - /* determain whether it's safe to coalesce using George's heuristic */ - for p.Next() { - if t := p.Node().ID(); len(arch) <= rig.From(t).Len() && !rig.HasEdgeBetween(t, int64(rx)) { - ok = false - break - } - } - - /* check if it can be coalesced */ - if !ok { - continue - } - - /* r0 is the target register, r1 will be coalesced */ - if regorder(rx) < regorder(ry) { - r0 = rx - r1 = ry - } else { - r0 = ry - r1 = rx - } - - /* invcoalescemap's key is the target reg (named r0), it's value is the reg list coalesced to r0 */ - for _, r := range invcoalescemap[r1] { - coalescemap[r] = r0 - } - - /* also mark the inverse relationship */ - invcoalescemap[r0] = append(invcoalescemap[r0], r1) - invcoalescemap[r0] = append(invcoalescemap[r0], invcoalescemap[r1]...) - - /* coalesce r0 and r1 (replace r1 by r0) in the interference graph */ - for p = rig.From(int64(r1)); p.Next(); { - if t := p.Node().ID(); !rig.HasEdgeBetween(t, int64(r0)) { - nt, _ := rig.NodeWithID(t) - n0, _ := rig.NodeWithID(int64(r0)) - rig.SetEdge(rig.NewEdge(n0, nt)) - } - } - - /* mark as coalesced */ - coalescemap[r1] = r0 - rig.RemoveNode(int64(r1)) - } - }) - - /* coalesce if needed */ - if len(coalescemap) != 0 { - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - var ok bool - var use IrUsages - var def IrDefinitions - - /* should not have Phi nodes */ - if len(bb.Phi) != 0 { - panic("regalloc: unexpected Phi nodes") - } - - /* scan instructions */ - for _, v := range bb.Ins { - if use, ok = v.(IrUsages) ; ok { coalesce(use.Usages()) } - if def, ok = v.(IrDefinitions) ; ok { coalesce(def.Definitions()) } - } - - /* scan terminator */ - if use, ok = bb.Term.(IrUsages); ok { - coalesce(use.Usages()) - } - }) - } - - /* remove copies to itself */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - ins := bb.Ins - bb.Ins = bb.Ins[:0] - - /* filter the instructions */ - for _, p := range ins { - if rd, rs, ok := IrArchTryIntoCopy(p); !ok || rd != rs { - bb.Ins = append(bb.Ins, p) - } - } - }) - - /* try again if coalesce occured */ - if len(coalescemap) != 0 { - continue - } - - /* Phase 4: Attempt to color the RIG */ - k, m, _ := coloring.WelshPowell(rig, archcolors) - colormap = *(*map[Reg]int)(unsafe.Pointer(&m)) - - /* check for len(arch)-colorablity */ - if k <= len(arch) { - break - } - - /* the second pass should always be colorable */ - if pass++; pass >= 2 { - panic("regalloc: this CFG may not colorable") - } - - /* calculate the color sets */ - colors := coloring.Sets(m) - colorset := *(*map[int][]Reg)(unsafe.Pointer(&colors)) - - /* Phase 4: Spill excess registers to stack */ - for i, r := range arch { - rr := Rz - ok := false - rs := colorset[i] - - /* remove from color map */ - for _, rr = range rs { - if colormap[rr] != i { - panic("regalloc: color mismatch for register " + rr.String()) - } else if delete(colormap, rr); rr == r { - ok = true - } - } - - /* remove from color set */ - if delete(colorset, i); !ok { - panic("regalloc: invalid coloring: missing register " + r.String()) - } - } - - /* spill those without a physical register */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - var cc int - var rr Reg - var ok bool - var use IrUsages - - /* should not contain Phi nodes */ - if len(bb.Phi) != 0 { - panic("regalloc: unexpected Phi node") - } - - /* allocate buffer for new instructions */ - ins := bb.Ins - bb.Ins = make([]IrNode, 0, len(ins)) - - /* scan every instructions */ - for _, v := range ins { - var s Reg - var t Reg - var r *Reg - var d IrDefinitions - - /* clear the register map */ - for c := range regmap { - delete(regmap, c) - } - - /* reload as needed */ - if use, ok = v.(IrUsages); ok { - for _, r = range use.Usages() { - if cc, ok = colormap[*r]; ok { - if rr, ok = regmap[cc]; ok { - *r = rr - } else { - *r = cfg.CreateRegister(r.Ptr()) - bb.Ins = append(bb.Ins, IrCreateSpill(*r, cc, IrSpillReload)) - regmap[cc] = *r - } - } - } - } - - /* add the instruction itself */ - i := len(bb.Ins) - bb.Ins = append(bb.Ins, v) - - /* no definitions */ - if d, ok = v.(IrDefinitions); !ok { - continue - } - - /* spill as needed */ - for _, r = range d.Definitions() { - if cc, ok = colormap[*r]; ok { - if t, s, ok = IrArchTryIntoCopy(v); !ok { - bb.Ins = append(bb.Ins, IrCreateSpill(*r, cc, IrSpillStore)) - } else { - bb.Ins[i] = IrCreateSpillEx(s, t.Ptr(), cc, IrSpillStore) - } - } - } - } - - /* clear the register map for terminator */ - for c := range regmap { - delete(regmap, c) - } - - /* scan for reloads in the terminator */ - if use, ok = bb.Term.(IrUsages); ok { - for _, r := range use.Usages() { - if cc, ok = colormap[*r]; ok { - if rr, ok = regmap[cc]; ok { - *r = rr - } else { - *r = cfg.CreateRegister(r.Ptr()) - bb.Ins = append(bb.Ins, IrCreateSpill(*r, cc, IrSpillReload)) - regmap[cc] = *r - } - } - } - } - }) - } - - /* finetune color allocation plan */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - reloadRegs := make(map[Reg]int) - spillSlots := make(map[int]Reg) - slotReg := make(map[int]Reg) - - /* process instructions */ - for _, v := range bb.Ins { - if spillIr, ok := v.(*IrSpill); ok && spillIr.Op == IrSpillReload { - if _, ok = reloadRegs[spillIr.R]; !ok { - /* try to assign the same color to reloadReg and spillReg with the same slot */ - if r, ok := spillSlots[spillIr.S.ID()]; ok { - self.colorSameWithReg(rig, spillIr.R, arch, colormap, r) - } else if r, ok := slotReg[spillIr.S.ID()]; ok { - /* try to assign the same color to reloadRegs with the same slot */ - self.colorSameWithReg(rig, spillIr.R, arch, colormap, r) - } else { - /* try to assign different color to reloadRegs with different slots */ - self.colorDiffWithReload(rig, spillIr.R, reloadRegs, arch, colormap) - } - slotReg[spillIr.S.ID()] = spillIr.R - reloadRegs[spillIr.R] = colormap[spillIr.R] - } - } else if ok && spillIr.Op == IrSpillStore { - spillSlots[spillIr.S.ID()] = spillIr.R - } else { - /* try to choose color different from reload regs for defined regs */ - if def, ok := v.(IrDefinitions); ok { - for _, r := range def.Definitions() { - self.colorDiffWithReload(rig, *r, reloadRegs, arch, colormap) - } - } - } - } - }) - - /* register substitution routine */ - replaceregs := func(rr []*Reg) { - for _, r := range rr { - if c, ok := colormap[*r]; ok { - *r = arch[c] - } - } - } - - /* Phase 5: Replace all the virtual registers with physical registers */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - var ok bool - var use IrUsages - var def IrDefinitions - - /* should not contain Phi nodes */ - if len(bb.Phi) != 0 { - panic("regalloc: unexpected Phi node") - } - - /* replace instructions */ - for _, v := range bb.Ins { - if use, ok = v.(IrUsages) ; ok { replaceregs(use.Usages()) } - if def, ok = v.(IrDefinitions) ; ok { replaceregs(def.Definitions()) } - } - - /* replace the terminator */ - if use, ok = bb.Term.(IrUsages); ok { - replaceregs(use.Usages()) - } - }) - - /* remove copies to itself */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - ins := bb.Ins - bb.Ins = bb.Ins[:0] - - /* filter the instructions */ - for _, p := range ins { - if rd, rs, ok := IrArchTryIntoCopy(p); !ok || rd != rs { - bb.Ins = append(bb.Ins, p) - } - } - }) - - /* remove redundant spill and reload instructions where both the register and stack aren't modified */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - /* register to stack slot is a one to one mapping */ - regSlot := make(map[Reg]int) - slotReg := make(map[int]Reg) - ins := bb.Ins - bb.Ins = nil - def := IrDefinitions(nil) - - /* scan every instruction */ - for _, v := range ins { - if spillIr, ok := v.(*IrSpill); ok { - /* if there has been a one-one mapping between the register and stack slot, abandon this spill/reload instruction */ - if s, ok := regSlot[spillIr.R]; ok && s == spillIr.S.ID() { - if r, ok := slotReg[spillIr.S.ID()]; ok && r == spillIr.R { - continue - } - } - - /* delete old mapping relations */ - delete(regSlot, slotReg[spillIr.S.ID()]) - delete(slotReg, regSlot[spillIr.R]) - - /* establish new mapping relations */ - slotReg[spillIr.S.ID()] = spillIr.R - regSlot[spillIr.R] = spillIr.S.ID() - bb.Ins = append(bb.Ins, v) - } else { - if def, ok = v.(IrDefinitions); ok { - for _, r := range def.Definitions() { - /* delete old mapping relations */ - delete(slotReg, regSlot[*r]) - delete(regSlot, *r) - } - } - bb.Ins = append(bb.Ins, v) - } - } - }) - - /* remove redundant spill where the same stack slot is overwritten before loading */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - var redundantIrPos []int - storeStack := make(map[int][]int) - - /* scan every instruction */ - for i, v := range bb.Ins { - if spillIr, ok := v.(*IrSpill) ; ok && spillIr.Op == IrSpillStore { - storeStack[spillIr.S.ID()] = append(storeStack[spillIr.S.ID()], i) - } else if ok && spillIr.Op == IrSpillReload { - if irPos, ok := storeStack[spillIr.S.ID()]; ok { - redundantIrPos = append(redundantIrPos, irPos[0 : len(irPos)-1]...) - delete(storeStack, spillIr.S.ID()) - } - } - } - - for _, irPos := range storeStack { - if len(irPos) > 1 { - redundantIrPos = append(redundantIrPos, irPos[0 : len(irPos)-1]...) - } - } - - /* abandon redundant spill instructions according to their position in the block */ - if len(redundantIrPos) > 0 { - ins := bb.Ins - bb.Ins = nil - sort.Ints(redundantIrPos) - startPos := 0 - for _, p := range redundantIrPos { - bb.Ins = append(bb.Ins, ins[startPos : p]...) - startPos = p + 1 - } - if startPos < len(ins) { - bb.Ins = append(bb.Ins, ins[startPos : ]...) - } - } - }) - - regSliceToSet := func(rr []*Reg) _RegSet { - rs := make(_RegSet, 0) - for _, r := range rr { - rs.add(*r) - } - return rs - } - - /* remove redundant reload where the register isn't used after being reloaded */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - var ok bool - var use IrUsages - var def IrDefinitions - var spillIr *IrSpill - var removePos []int - rs := make(_RegSet, 0) - - /* add the terminator usages if any */ - if use, ok = bb.Term.(IrUsages); ok { rs.union(regSliceToSet(use.Usages())) } - - /* live(i-1) = use(i) ∪ (live(i) - { def(i) }) */ - for i := len(bb.Ins) - 1; i >= 0; i-- { - if def, ok = bb.Ins[i].(IrDefinitions); ok { - if spillIr, ok = bb.Ins[i].(*IrSpill); ok && spillIr.Op == IrSpillReload { - /* if the reloaded reg isn't used afterwards, record its position and then remove it */ - if !rs.contains(spillIr.R) { - removePos = append(removePos, i) - continue - } - } - rs.subtract(regSliceToSet(def.Definitions())) - } - if use, ok = bb.Ins[i].(IrUsages); ok { rs.union(regSliceToSet(use.Usages())) } - } - - if len(removePos) > 0 { - ins := bb.Ins - bb.Ins = nil - sort.Ints(removePos) - startPos := 0 - for _, p := range removePos { - bb.Ins = append(bb.Ins, ins[startPos : p]...) - startPos = p + 1 - } - if startPos < len(ins) { - bb.Ins = append(bb.Ins, ins[startPos : ]...) - } - } - }) -} \ No newline at end of file + var rig *simple.UndirectedGraph + var pass int + var arch []Reg + var colormap map[Reg]int + + /* reusable state */ + pool := mkregtab() + regmap := make(map[int]Reg) + livein := make(map[int]_RegSet) + liveout := make(map[int]_RegSet) + liveset := make(map[Pos]_RegSet) + archcolors := make(map[int64]int, len(ArchRegs)) + coalescemap := make(map[Reg]Reg) + invcoalescemap := make(map[Reg][]Reg) + + /* register coalesce */ + coalesce := func(rr []*Reg) { + for _, r := range rr { + if c, ok := coalescemap[*r]; ok { + *r = c + } + } + } + + /* calculate allocatable registers */ + for _, r := range ArchRegs { + if !ArchRegReserved[r] { + arch = append(arch, IrSetArch(Rz, r)) + } + } + + /* allocate colors to the registers */ + for i, r := range arch { + archcolors[int64(r)] = i + } + + /* register precolorer */ + precolor := func(rr []*Reg) { + for _, r := range rr { + if r.Kind() == K_arch { + *r = IrSetArch(Rz, ArchRegs[r.Name()]) + } + } + } + + /* precolor all physical registers */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + ok := false + use := IrUsages(nil) + def := IrDefinitions(nil) + + /* scan all the instructions */ + for _, v := range bb.Ins { + if use, ok = v.(IrUsages); ok { + precolor(use.Usages()) + } + if def, ok = v.(IrDefinitions); ok { + precolor(def.Definitions()) + } + } + + /* scan the terminator */ + if use, ok = bb.Term.(IrUsages); ok { + precolor(use.Usages()) + } + }) + + /* loop until no more retries */ + for { + pool.reset() + rt.MapClear(livein) + rt.MapClear(liveout) + rt.MapClear(liveset) + rt.MapClear(coalescemap) + rt.MapClear(invcoalescemap) + + /* expand operands but not in the first pass */ + if pass != 0 { + new(OperandAlloc).Apply(cfg) + } + + /* Phase 1: Calculate live ranges */ + lr := self.livein(pool, liveset, cfg.Root, livein, liveout) + rig = simple.NewUndirectedGraph() + + /* sanity check: no registers live at the entry point */ + if len(lr) != 0 { + panic("regalloc: live registers at entry: " + lr.String()) + } + + /* Phase 2: Build register interference graph */ + for _, rs := range liveset { + rr := rs.toslice() + nr := len(rr) + + /* special case of a single live register */ + if nr == 1 && rig.Node(int64(rr[0])) == nil { + p, _ := rig.NodeWithID(int64(rr[0])) + rig.AddNode(p) + continue + } + + /* create every edge */ + for i := 0; i < nr-1; i++ { + for j := i + 1; j < nr; j++ { + p, _ := rig.NodeWithID(int64(rr[i])) + q, _ := rig.NodeWithID(int64(rr[j])) + rig.SetEdge(rig.NewEdge(p, q)) + } + } + } + + /* make sure all physical registers are in the RIG */ + for _, r := range arch { + if p, ok := rig.NodeWithID(int64(r)); ok { + rig.AddNode(p) + } + } + + /* Phase 3: Coalescing registers, find out all the coalescing pairs */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + for _, v := range bb.Ins { + var r0 Reg + var r1 Reg + var rx Reg + var ry Reg + var ok bool + + /* only interested in copy instructions */ + if rx, ry, ok = IrArchTryIntoCopy(v); !ok { + continue + } + + /* locate the coalescing source register */ + if r0, ok = coalescemap[rx]; ok { + rx = r0 + } + if r1, ok = coalescemap[ry]; ok { + ry = r1 + } + + /* check if the two registers can be coalesced */ + if rx == ry || rig.HasEdgeBetween(int64(rx), int64(ry)) || (rx.Kind() == K_arch && ry.Kind() == K_arch) { + continue + } + + /* make sure Y is the node with a lower degree */ + if rig.From(int64(rx)).Len() < rig.From(int64(ry)).Len() { + rx, ry = ry, rx + } + + /* all the adjacent nodes of Y */ + p := rig.From(int64(ry)) + ok = true + + /* determain whether it's safe to coalesce using George's heuristic */ + for p.Next() { + if t := p.Node().ID(); len(arch) <= rig.From(t).Len() && !rig.HasEdgeBetween(t, int64(rx)) { + ok = false + break + } + } + + /* check if it can be coalesced */ + if !ok { + continue + } + + /* r0 is the target register, r1 will be coalesced */ + if regorder(rx) < regorder(ry) { + r0 = rx + r1 = ry + } else { + r0 = ry + r1 = rx + } + + /* invcoalescemap's key is the target reg (named r0), it's value is the reg list coalesced to r0 */ + for _, r := range invcoalescemap[r1] { + coalescemap[r] = r0 + } + + /* also mark the inverse relationship */ + invcoalescemap[r0] = append(invcoalescemap[r0], r1) + invcoalescemap[r0] = append(invcoalescemap[r0], invcoalescemap[r1]...) + + /* coalesce r0 and r1 (replace r1 by r0) in the interference graph */ + for p = rig.From(int64(r1)); p.Next(); { + if t := p.Node().ID(); !rig.HasEdgeBetween(t, int64(r0)) { + nt, _ := rig.NodeWithID(t) + n0, _ := rig.NodeWithID(int64(r0)) + rig.SetEdge(rig.NewEdge(n0, nt)) + } + } + + /* mark as coalesced */ + coalescemap[r1] = r0 + rig.RemoveNode(int64(r1)) + } + }) + + /* coalesce if needed */ + if len(coalescemap) != 0 { + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + var ok bool + var use IrUsages + var def IrDefinitions + + /* should not have Phi nodes */ + if len(bb.Phi) != 0 { + panic("regalloc: unexpected Phi nodes") + } + + /* scan instructions */ + for _, v := range bb.Ins { + if use, ok = v.(IrUsages); ok { + coalesce(use.Usages()) + } + if def, ok = v.(IrDefinitions); ok { + coalesce(def.Definitions()) + } + } + + /* scan terminator */ + if use, ok = bb.Term.(IrUsages); ok { + coalesce(use.Usages()) + } + }) + } + + /* remove copies to itself */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + ins := bb.Ins + bb.Ins = bb.Ins[:0] + + /* filter the instructions */ + for _, p := range ins { + if rd, rs, ok := IrArchTryIntoCopy(p); !ok || rd != rs { + bb.Ins = append(bb.Ins, p) + } + } + }) + + /* try again if coalesce occurred */ + if len(coalescemap) != 0 { + continue + } + + /* Phase 4: Attempt to color the RIG */ + k, m, _ := coloring.WelshPowell(rig, archcolors) + colormap = *(*map[Reg]int)(unsafe.Pointer(&m)) + + /* check for len(arch)-colorablity */ + if k <= len(arch) { + break + } + + /* the second pass should always be colorable */ + if pass++; pass >= 2 { + panic("regalloc: this CFG may not colorable") + } + + /* calculate the color sets */ + colors := coloring.Sets(m) + colorset := *(*map[int][]Reg)(unsafe.Pointer(&colors)) + + /* Phase 4: Spill excess registers to stack */ + for i, r := range arch { + rr := Rz + ok := false + rs := colorset[i] + + /* remove from color map */ + for _, rr = range rs { + if colormap[rr] != i { + panic("regalloc: color mismatch for register " + rr.String()) + } else if delete(colormap, rr); rr == r { + ok = true + } + } + + /* remove from color set */ + if delete(colorset, i); !ok { + panic("regalloc: invalid coloring: missing register " + r.String()) + } + } + + /* spill those without a physical register */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + var cc int + var rr Reg + var ok bool + var use IrUsages + + /* should not contain Phi nodes */ + if len(bb.Phi) != 0 { + panic("regalloc: unexpected Phi node") + } + + /* allocate buffer for new instructions */ + ins := bb.Ins + bb.Ins = make([]IrNode, 0, len(ins)) + + /* scan every instructions */ + for _, v := range ins { + var s Reg + var t Reg + var r *Reg + var d IrDefinitions + + /* clear the register map */ + for c := range regmap { + delete(regmap, c) + } + + /* reload as needed */ + if use, ok = v.(IrUsages); ok { + for _, r = range use.Usages() { + if cc, ok = colormap[*r]; ok { + if rr, ok = regmap[cc]; ok { + *r = rr + } else { + *r = cfg.CreateRegister(r.Ptr()) + bb.Ins = append(bb.Ins, IrCreateSpill(*r, cc, IrSpillReload)) + regmap[cc] = *r + } + } + } + } + + /* add the instruction itself */ + i := len(bb.Ins) + bb.Ins = append(bb.Ins, v) + + /* no definitions */ + if d, ok = v.(IrDefinitions); !ok { + continue + } + + /* spill as needed */ + for _, r = range d.Definitions() { + if cc, ok = colormap[*r]; ok { + if t, s, ok = IrArchTryIntoCopy(v); !ok { + bb.Ins = append(bb.Ins, IrCreateSpill(*r, cc, IrSpillStore)) + } else { + bb.Ins[i] = IrCreateSpillEx(s, t.Ptr(), cc, IrSpillStore) + } + } + } + } + + /* clear the register map for terminator */ + for c := range regmap { + delete(regmap, c) + } + + /* scan for reloads in the terminator */ + if use, ok = bb.Term.(IrUsages); ok { + for _, r := range use.Usages() { + if cc, ok = colormap[*r]; ok { + if rr, ok = regmap[cc]; ok { + *r = rr + } else { + *r = cfg.CreateRegister(r.Ptr()) + bb.Ins = append(bb.Ins, IrCreateSpill(*r, cc, IrSpillReload)) + regmap[cc] = *r + } + } + } + } + }) + } + + /* finetune color allocation plan */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + reloadRegs := make(map[Reg]int) + spillSlots := make(map[int]Reg) + slotReg := make(map[int]Reg) + + /* process instructions */ + for _, v := range bb.Ins { + if spillIr, ok := v.(*IrSpill); ok && spillIr.Op == IrSpillReload { + if _, ok = reloadRegs[spillIr.R]; !ok { + /* try to assign the same color to reloadReg and spillReg with the same slot */ + if r, ok := spillSlots[spillIr.S.ID()]; ok { + self.colorSameWithReg(rig, spillIr.R, arch, colormap, r) + } else if r, ok := slotReg[spillIr.S.ID()]; ok { + /* try to assign the same color to reloadRegs with the same slot */ + self.colorSameWithReg(rig, spillIr.R, arch, colormap, r) + } else { + /* try to assign different color to reloadRegs with different slots */ + self.colorDiffWithReload(rig, spillIr.R, reloadRegs, arch, colormap) + } + slotReg[spillIr.S.ID()] = spillIr.R + reloadRegs[spillIr.R] = colormap[spillIr.R] + } + } else if ok && spillIr.Op == IrSpillStore { + spillSlots[spillIr.S.ID()] = spillIr.R + } else { + /* try to choose color different from reload regs for defined regs */ + if def, ok := v.(IrDefinitions); ok { + for _, r := range def.Definitions() { + self.colorDiffWithReload(rig, *r, reloadRegs, arch, colormap) + } + } + } + } + }) + + /* register substitution routine */ + replaceregs := func(rr []*Reg) { + for _, r := range rr { + if c, ok := colormap[*r]; ok { + *r = arch[c] + } + } + } + + /* Phase 5: Replace all the virtual registers with physical registers */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + var ok bool + var use IrUsages + var def IrDefinitions + + /* should not contain Phi nodes */ + if len(bb.Phi) != 0 { + panic("regalloc: unexpected Phi node") + } + + /* replace instructions */ + for _, v := range bb.Ins { + if use, ok = v.(IrUsages); ok { + replaceregs(use.Usages()) + } + if def, ok = v.(IrDefinitions); ok { + replaceregs(def.Definitions()) + } + } + + /* replace the terminator */ + if use, ok = bb.Term.(IrUsages); ok { + replaceregs(use.Usages()) + } + }) + + /* remove copies to itself */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + ins := bb.Ins + bb.Ins = bb.Ins[:0] + + /* filter the instructions */ + for _, p := range ins { + if rd, rs, ok := IrArchTryIntoCopy(p); !ok || rd != rs { + bb.Ins = append(bb.Ins, p) + } + } + }) + + /* remove redundant spill and reload instructions where both the register and stack aren't modified */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + /* register to stack slot is a one to one mapping */ + regSlot := make(map[Reg]int) + slotReg := make(map[int]Reg) + ins := bb.Ins + bb.Ins = nil + def := IrDefinitions(nil) + + /* scan every instruction */ + for _, v := range ins { + if spillIr, ok := v.(*IrSpill); ok { + /* if there has been a one-one mapping between the register and stack slot, abandon this spill/reload instruction */ + if s, ok := regSlot[spillIr.R]; ok && s == spillIr.S.ID() { + if r, ok := slotReg[spillIr.S.ID()]; ok && r == spillIr.R { + continue + } + } + + /* delete old mapping relations */ + delete(regSlot, slotReg[spillIr.S.ID()]) + delete(slotReg, regSlot[spillIr.R]) + + /* establish new mapping relations */ + slotReg[spillIr.S.ID()] = spillIr.R + regSlot[spillIr.R] = spillIr.S.ID() + bb.Ins = append(bb.Ins, v) + } else { + if def, ok = v.(IrDefinitions); ok { + for _, r := range def.Definitions() { + /* delete old mapping relations */ + delete(slotReg, regSlot[*r]) + delete(regSlot, *r) + } + } + bb.Ins = append(bb.Ins, v) + } + } + }) + + /* remove redundant spill where the same stack slot is overwritten before loading */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + var redundantIrPos []int + storeStack := make(map[int][]int) + + /* scan every instruction */ + for i, v := range bb.Ins { + if spillIr, ok := v.(*IrSpill); ok && spillIr.Op == IrSpillStore { + storeStack[spillIr.S.ID()] = append(storeStack[spillIr.S.ID()], i) + } else if ok && spillIr.Op == IrSpillReload { + if irPos, ok := storeStack[spillIr.S.ID()]; ok { + redundantIrPos = append(redundantIrPos, irPos[0:len(irPos)-1]...) + delete(storeStack, spillIr.S.ID()) + } + } + } + + for _, irPos := range storeStack { + if len(irPos) > 1 { + redundantIrPos = append(redundantIrPos, irPos[0:len(irPos)-1]...) + } + } + + /* abandon redundant spill instructions according to their position in the block */ + if len(redundantIrPos) > 0 { + ins := bb.Ins + bb.Ins = nil + sort.Ints(redundantIrPos) + startPos := 0 + for _, p := range redundantIrPos { + bb.Ins = append(bb.Ins, ins[startPos:p]...) + startPos = p + 1 + } + if startPos < len(ins) { + bb.Ins = append(bb.Ins, ins[startPos:]...) + } + } + }) + + regSliceToSet := func(rr []*Reg) _RegSet { + rs := make(_RegSet, 0) + for _, r := range rr { + rs.add(*r) + } + return rs + } + + /* remove redundant reload where the register isn't used after being reloaded */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + var ok bool + var use IrUsages + var def IrDefinitions + var spillIr *IrSpill + var removePos []int + rs := make(_RegSet, 0) + + /* add the terminator usages if any */ + if use, ok = bb.Term.(IrUsages); ok { + rs.union(regSliceToSet(use.Usages())) + } + + /* live(i-1) = use(i) ∪ (live(i) - { def(i) }) */ + for i := len(bb.Ins) - 1; i >= 0; i-- { + if def, ok = bb.Ins[i].(IrDefinitions); ok { + if spillIr, ok = bb.Ins[i].(*IrSpill); ok && spillIr.Op == IrSpillReload { + /* if the reloaded reg isn't used afterwards, record its position and then remove it */ + if !rs.contains(spillIr.R) { + removePos = append(removePos, i) + continue + } + } + rs.subtract(regSliceToSet(def.Definitions())) + } + if use, ok = bb.Ins[i].(IrUsages); ok { + rs.union(regSliceToSet(use.Usages())) + } + } + + if len(removePos) > 0 { + ins := bb.Ins + bb.Ins = nil + sort.Ints(removePos) + startPos := 0 + for _, p := range removePos { + bb.Ins = append(bb.Ins, ins[startPos:p]...) + startPos = p + 1 + } + if startPos < len(ins) { + bb.Ins = append(bb.Ins, ins[startPos:]...) + } + } + }) +} diff --git a/internal/atm/ssa/pass_rematerialize.go b/internal/atm/ssa/pass_rematerialize.go index c847f99..32b2a74 100644 --- a/internal/atm/ssa/pass_rematerialize.go +++ b/internal/atm/ssa/pass_rematerialize.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,33 +20,33 @@ package ssa type Rematerialize struct{} func (Rematerialize) Apply(cfg *CFG) { - consts := make(map[Reg]_ConstData) - consts[Rz] = constint(0) - consts[Pn] = constptr(nil, Const) + consts := make(map[Reg]_ConstData) + consts[Rz] = constint(0) + consts[Pn] = constptr(nil, Const) - /* Phase 1: Scan all the constants */ - for _, bb := range cfg.PostOrder().Reversed() { - for _, v := range bb.Ins { - if r, x, ok := IrArchTryIntoConstInt(v); ok { - consts[r] = constint(x) - } else if r, p, ok := IrArchTryIntoConstPtr(v); ok { - consts[r] = constptr(p, Volatile) - } - } - } + /* Phase 1: Scan all the constants */ + for _, bb := range cfg.PostOrder().Reversed() { + for _, v := range bb.Ins { + if r, x, ok := IrArchTryIntoConstInt(v); ok { + consts[r] = constint(x) + } else if r, p, ok := IrArchTryIntoConstPtr(v); ok { + consts[r] = constptr(p, Volatile) + } + } + } - /* Phase 2: Replace register copies with consts if possible */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - for i, v := range bb.Ins { - if d, s, ok := IrArchTryIntoCopy(v); ok { - if cc, ok := consts[s]; ok { - if cc.i { - bb.Ins[i] = IrArchConstInt(d, cc.v) - } else { - bb.Ins[i] = IrArchConstPtr(d, cc.p) - } - } - } - } - }) + /* Phase 2: Replace register copies with consts if possible */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + for i, v := range bb.Ins { + if d, s, ok := IrArchTryIntoCopy(v); ok { + if cc, ok := consts[s]; ok { + if cc.i { + bb.Ins[i] = IrArchConstInt(d, cc.v) + } else { + bb.Ins[i] = IrArchConstPtr(d, cc.p) + } + } + } + } + }) } diff --git a/internal/atm/ssa/pass_reorder.go b/internal/atm/ssa/pass_reorder.go index 3d3fd8c..5733dd1 100644 --- a/internal/atm/ssa/pass_reorder.go +++ b/internal/atm/ssa/pass_reorder.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,343 +17,342 @@ package ssa import ( - `fmt` - `sort` + "fmt" + "sort" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) type _ValueId struct { - i int - v IrNode - r bool + i int + v IrNode + r bool } func mkvid(i int, v IrNode) *_ValueId { - return &_ValueId { - i: i, - v: v, - r: true, - } + return &_ValueId{ + i: i, + v: v, + r: true, + } } type _BlockRef struct { - bb *BasicBlock + bb *BasicBlock } func (self *_BlockRef) update(cfg *CFG, bb *BasicBlock) { - u := bb - v := self.bb - - /* move them to the same depth */ - for cfg.Depth[u.Id] != cfg.Depth[v.Id] { - if cfg.Depth[u.Id] > cfg.Depth[v.Id] { - u = cfg.DominatedBy[u.Id] - } else { - v = cfg.DominatedBy[v.Id] - } - } - - /* move both nodes until they meet */ - for u != v { - u = cfg.DominatedBy[u.Id] - v = cfg.DominatedBy[v.Id] - } - - /* sanity check */ - if u != nil { - self.bb = u - } else { - panic("reorder: invalid CFG dominator tree") - } + u := bb + v := self.bb + + /* move them to the same depth */ + for cfg.Depth[u.Id] != cfg.Depth[v.Id] { + if cfg.Depth[u.Id] > cfg.Depth[v.Id] { + u = cfg.DominatedBy[u.Id] + } else { + v = cfg.DominatedBy[v.Id] + } + } + + /* move both nodes until they meet */ + for u != v { + u = cfg.DominatedBy[u.Id] + v = cfg.DominatedBy[v.Id] + } + + /* sanity check */ + if u != nil { + self.bb = u + } else { + panic("reorder: invalid CFG dominator tree") + } } // Reorder moves value closer to it's usage, which reduces register pressure. type Reorder struct{} func (Reorder) isMovable(v IrNode) bool { - var f bool - var u IrUsages - var d IrDefinitions - - /* marked as immovable */ - if _, f = v.(IrImmovable); f { - return false - } - - /* blacklist all instructions that uses physical registers */ - if u, f = v.(IrUsages); f { - for _, r := range u.Usages() { - if r.Kind() == K_arch { - return false - } - } - } - - /* blacklist all instructions that alters physical registers */ - if d, f = v.(IrDefinitions); f { - for _, r := range d.Definitions() { - if r.Kind() == K_arch { - return false - } - } - } - - /* no such registers, all checked ok */ - return true + var f bool + var u IrUsages + var d IrDefinitions + + /* marked as immovable */ + if _, f = v.(IrImmovable); f { + return false + } + + /* blacklist all instructions that uses physical registers */ + if u, f = v.(IrUsages); f { + for _, r := range u.Usages() { + if r.Kind() == K_arch { + return false + } + } + } + + /* blacklist all instructions that alters physical registers */ + if d, f = v.(IrDefinitions); f { + for _, r := range d.Definitions() { + if r.Kind() == K_arch { + return false + } + } + } + + /* no such registers, all checked ok */ + return true } func (self Reorder) moveInterblock(cfg *CFG) { - defs := make(map[Reg]*_BlockRef) - uses := make(map[Pos]*_BlockRef) - move := make(map[*BasicBlock]int) - - /* usage update routine */ - updateUsage := func(r Reg, bb *BasicBlock) { - if m, ok := defs[r]; ok { - if m.bb == nil { - m.bb = bb - } else { - m.update(cfg, bb) - } - } - } - - /* retry until no modifications */ - for move[nil] = 0; len(move) != 0; { - rt.MapClear(defs) - rt.MapClear(move) - rt.MapClear(uses) - - /* Phase 1: Find all movable value definitions */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - for i, v := range bb.Ins { - var f bool - var p *_BlockRef - var d IrDefinitions - - /* value must be movable, and have definitions */ - if d, f = v.(IrDefinitions); !f || !self.isMovable(v) { - continue - } - - /* create a new value movement if needed */ - if p, f = uses[pos(bb, i)]; !f { - p = new(_BlockRef) - uses[pos(bb, i)] = p - } - - /* mark all the non-definition sites */ - for _, r := range d.Definitions() { - if r.Kind() != K_zero { - defs[*r] = p - } - } - } - }) - - /* Phase 2: Identify the earliest usage locations */ - for _, bb := range cfg.PostOrder().Reversed() { - var ok bool - var use IrUsages - - /* search in Phi nodes */ - for _, v := range bb.Phi { - for b, r := range v.V { - updateUsage(*r, b) - } - } - - /* search in instructions */ - for _, v := range bb.Ins { - if use, ok = v.(IrUsages); ok { - for _, r := range use.Usages() { - updateUsage(*r, bb) - } - } - } - - /* search the terminator */ - if use, ok = bb.Term.(IrUsages); ok { - for _, r := range use.Usages() { - updateUsage(*r, bb) - } - } - } - - /* Phase 3: Move value definitions to their usage block */ - for p, m := range uses { - if m.bb != nil && m.bb != p.B { - m.bb.Ins = append(m.bb.Ins, p.B.Ins[p.I]) - move[m.bb] = move[m.bb] + 1 - p.B.Ins[p.I] = new(IrNop) - } - } - - /* Phase 4: Move values to place */ - for bb, i := range move { - v := bb.Ins - n := len(bb.Ins) - bb.Ins = make([]IrNode, n) - copy(bb.Ins[i:], v[:n - i]) - copy(bb.Ins[:i], v[n - i:]) - } - } - - /* Phase 5: Remove all the placeholder NOP instructions */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - ins := bb.Ins - bb.Ins = bb.Ins[:0] - - /* filter out the NOP instructions */ - for _, v := range ins { - if _, ok := v.(*IrNop); !ok { - bb.Ins = append(bb.Ins, v) - } - } - }) + defs := make(map[Reg]*_BlockRef) + uses := make(map[Pos]*_BlockRef) + move := make(map[*BasicBlock]int) + + /* usage update routine */ + updateUsage := func(r Reg, bb *BasicBlock) { + if m, ok := defs[r]; ok { + if m.bb == nil { + m.bb = bb + } else { + m.update(cfg, bb) + } + } + } + + /* retry until no modifications */ + for move[nil] = 0; len(move) != 0; { + rt.MapClear(defs) + rt.MapClear(move) + rt.MapClear(uses) + + /* Phase 1: Find all movable value definitions */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + for i, v := range bb.Ins { + var f bool + var p *_BlockRef + var d IrDefinitions + + /* value must be movable, and have definitions */ + if d, f = v.(IrDefinitions); !f || !self.isMovable(v) { + continue + } + + /* create a new value movement if needed */ + if p, f = uses[pos(bb, i)]; !f { + p = new(_BlockRef) + uses[pos(bb, i)] = p + } + + /* mark all the non-definition sites */ + for _, r := range d.Definitions() { + if r.Kind() != K_zero { + defs[*r] = p + } + } + } + }) + + /* Phase 2: Identify the earliest usage locations */ + for _, bb := range cfg.PostOrder().Reversed() { + var ok bool + var use IrUsages + + /* search in Phi nodes */ + for _, v := range bb.Phi { + for b, r := range v.V { + updateUsage(*r, b) + } + } + + /* search in instructions */ + for _, v := range bb.Ins { + if use, ok = v.(IrUsages); ok { + for _, r := range use.Usages() { + updateUsage(*r, bb) + } + } + } + + /* search the terminator */ + if use, ok = bb.Term.(IrUsages); ok { + for _, r := range use.Usages() { + updateUsage(*r, bb) + } + } + } + + /* Phase 3: Move value definitions to their usage block */ + for p, m := range uses { + if m.bb != nil && m.bb != p.B { + m.bb.Ins = append(m.bb.Ins, p.B.Ins[p.I]) + move[m.bb] = move[m.bb] + 1 + p.B.Ins[p.I] = new(IrNop) + } + } + + /* Phase 4: Move values to place */ + for bb, i := range move { + v := bb.Ins + n := len(bb.Ins) + bb.Ins = make([]IrNode, n) + copy(bb.Ins[i:], v[:n-i]) + copy(bb.Ins[:i], v[n-i:]) + } + } + + /* Phase 5: Remove all the placeholder NOP instructions */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + ins := bb.Ins + bb.Ins = bb.Ins[:0] + + /* filter out the NOP instructions */ + for _, v := range ins { + if _, ok := v.(*IrNop); !ok { + bb.Ins = append(bb.Ins, v) + } + } + }) } func (self Reorder) moveIntrablock(cfg *CFG) { - var rbuf []IrNode - var mbuf []*_ValueId - var vbuf []*_ValueId - var addval func(*_ValueId, bool) - - /* reusable states */ - adds := make(map[int]struct{}) - defs := make(map[Reg]*_ValueId) - - /* topology sorter */ - addval = func(v *_ValueId, depsOnly bool) { - var ok bool - var use IrUsages - var val *_ValueId - - /* check if it's been added */ - if _, ok = adds[v.i]; ok { - return - } - - /* add all the dependencies recursively */ - if use, ok = v.v.(IrUsages); ok { - for _, r := range use.Usages() { - if val, ok = defs[*r]; ok { - addval(val, false) - } - } - } - - /* add the instruction if needed */ - if !depsOnly { - rbuf = append(rbuf, v.v) - adds[v.i] = struct{}{} - } - } - - /* process every block */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - rbuf = rbuf[:0] - mbuf = mbuf[:0] - vbuf = vbuf[:0] - rt.MapClear(adds) - rt.MapClear(defs) - - /* number all instructions */ - for i, v := range bb.Ins { - id := mkvid(i, v) - vbuf = append(vbuf, id) - - /* preserve the order of immovable instructions */ - if !self.isMovable(v) { - mbuf = append(mbuf, id) - } - } - - /* mark all non-Phi definitions in this block */ - for _, v := range vbuf { - if def, ok := v.v.(IrDefinitions); ok { - for _, r := range def.Definitions() { - if _, ok = defs[*r]; ok { - panic(fmt.Sprintf("reorder: multiple definitions for %s in bb_%d", r, bb.Id)) - } else { - defs[*r] = v - } - } - } - } - - /* find all the root nodes */ - for _, v := range vbuf { - if use, ok := v.v.(IrUsages); !ok { - v.r = false - } else { - for _, r := range use.Usages() { - if v, ok = defs[*r]; ok { - v.r = false - } - } - } - } - - /* all the immovable instructions needs to preserve their order */ - for _, v := range mbuf { - addval(v, false) - } - - /* add all the root instructions */ - for _, v := range vbuf { - if v.r { - addval(v, false) - } - } - - /* add remaining instructions */ - for _, v := range vbuf { - if _, ok := adds[v.i]; !ok { - addval(v, false) - } - } - - /* add the terminator */ - addval(mkvid(-1, bb.Term), true) - bb.Ins = append(bb.Ins[:0], rbuf...) - }) + var rbuf []IrNode + var mbuf []*_ValueId + var vbuf []*_ValueId + var addval func(*_ValueId, bool) + + /* reusable states */ + adds := make(map[int]struct{}) + defs := make(map[Reg]*_ValueId) + + /* topology sorter */ + addval = func(v *_ValueId, depsOnly bool) { + var ok bool + var use IrUsages + var val *_ValueId + + /* check if it's been added */ + if _, ok = adds[v.i]; ok { + return + } + + /* add all the dependencies recursively */ + if use, ok = v.v.(IrUsages); ok { + for _, r := range use.Usages() { + if val, ok = defs[*r]; ok { + addval(val, false) + } + } + } + + /* add the instruction if needed */ + if !depsOnly { + rbuf = append(rbuf, v.v) + adds[v.i] = struct{}{} + } + } + + /* process every block */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + rbuf = rbuf[:0] + mbuf = mbuf[:0] + vbuf = vbuf[:0] + rt.MapClear(adds) + rt.MapClear(defs) + + /* number all instructions */ + for i, v := range bb.Ins { + id := mkvid(i, v) + vbuf = append(vbuf, id) + + /* preserve the order of immovable instructions */ + if !self.isMovable(v) { + mbuf = append(mbuf, id) + } + } + + /* mark all non-Phi definitions in this block */ + for _, v := range vbuf { + if def, ok := v.v.(IrDefinitions); ok { + for _, r := range def.Definitions() { + if _, ok = defs[*r]; ok { + panic(fmt.Sprintf("reorder: multiple definitions for %s in bb_%d", r, bb.Id)) + } else { + defs[*r] = v + } + } + } + } + + /* find all the root nodes */ + for _, v := range vbuf { + if use, ok := v.v.(IrUsages); !ok { + v.r = false + } else { + for _, r := range use.Usages() { + if v, ok = defs[*r]; ok { + v.r = false + } + } + } + } + + /* all the immovable instructions needs to preserve their order */ + for _, v := range mbuf { + addval(v, false) + } + + /* add all the root instructions */ + for _, v := range vbuf { + if v.r { + addval(v, false) + } + } + + /* add remaining instructions */ + for _, v := range vbuf { + if _, ok := adds[v.i]; !ok { + addval(v, false) + } + } + + /* add the terminator */ + addval(mkvid(-1, bb.Term), true) + bb.Ins = append(bb.Ins[:0], rbuf...) + }) } func (Reorder) moveArgumentLoad(cfg *CFG) { - var ok bool - var ir []IrNode - var vv *IrLoadArg - - /* extract all the argument loads */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - ins := bb.Ins - bb.Ins = bb.Ins[:0] - - /* scan instructions */ - for _, v := range ins { - if vv, ok = v.(*IrLoadArg); ok { - ir = append(ir, vv) - } else { - bb.Ins = append(bb.Ins, v) - } - } - }) - - /* sort by argument ID */ - sort.Slice(ir, func(i int, j int) bool { - return ir[i].(*IrLoadArg).I < ir[j].(*IrLoadArg).I - }) - - /* prepend to the root node */ - ins := cfg.Root.Ins - cfg.Root.Ins = append(ir, ins...) + var ok bool + var ir []IrNode + var vv *IrLoadArg + + /* extract all the argument loads */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + ins := bb.Ins + bb.Ins = bb.Ins[:0] + + /* scan instructions */ + for _, v := range ins { + if vv, ok = v.(*IrLoadArg); ok { + ir = append(ir, vv) + } else { + bb.Ins = append(bb.Ins, v) + } + } + }) + + /* sort by argument ID */ + sort.Slice(ir, func(i int, j int) bool { + return ir[i].(*IrLoadArg).I < ir[j].(*IrLoadArg).I + }) + + /* prepend to the root node */ + ins := cfg.Root.Ins + cfg.Root.Ins = append(ir, ins...) } func (self Reorder) Apply(cfg *CFG) { - self.moveInterblock(cfg) - self.moveIntrablock(cfg) - self.moveArgumentLoad(cfg) + self.moveInterblock(cfg) + self.moveIntrablock(cfg) + self.moveArgumentLoad(cfg) } - diff --git a/internal/atm/ssa/pass_return_spread.go b/internal/atm/ssa/pass_return_spread.go index 7172796..6dd3ff6 100644 --- a/internal/atm/ssa/pass_return_spread.go +++ b/internal/atm/ssa/pass_return_spread.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,120 +21,120 @@ package ssa type ReturnSpread struct{} func (ReturnSpread) Apply(cfg *CFG) { - more := true - rets := make([]*BasicBlock, 0, 1) - - /* register replacer */ - replaceregs := func(rr map[Reg]Reg, ins IrNode) { - var v Reg - var ok bool - var use IrUsages - var def IrDefinitions - - /* replace register usages */ - if use, ok = ins.(IrUsages); ok { - for _, r := range use.Usages() { - if v, ok = rr[*r]; ok { - *r = v - } - } - } - - /* replace register definitions */ - if def, ok = ins.(IrDefinitions); ok { - for _, r := range def.Definitions() { - if v, ok = rr[*r]; ok { - *r = v - } - } - } - } - - /* loop until no more modifications */ - for more { - more = false - rets = rets[:0] - - /* Phase 1: Find the return blocks that has more than one predecessors */ - for _, bb := range cfg.PostOrder().Reversed() { - if _, ok := bb.Term.(*IrReturn); ok && len(bb.Pred) > 1 { - more = true - rets = append(rets, bb) - } - } - - /* Phase 2: Spread the blocks to it's predecessors */ - for _, bb := range rets { - for _, pred := range bb.Pred { - var ok bool - var sw *IrSwitch - - /* register mappings */ - rr := make(map[Reg]Reg) - nb := len(bb.Phi) + len(bb.Ins) - - /* allocate registers for Phi definitions */ - for _, phi := range bb.Phi { - rr[phi.R] = cfg.CreateRegister(phi.R.Ptr()) - } - - /* allocate registers for instruction definitions */ - for _, ins := range bb.Ins { - if def, ok := ins.(IrDefinitions); ok { - for _, r := range def.Definitions() { - rr[*r] = cfg.CreateRegister(r.Ptr()) - } - } - } - - /* create a new basic block */ - ret := cfg.CreateBlock() - ret.Ins = make([]IrNode, 0, nb) - ret.Pred = []*BasicBlock { pred } - - /* add copy instruction for Phi nodes */ - for _, phi := range bb.Phi { - ret.Ins = append(ret.Ins, IrCopy(rr[phi.R], *phi.V[pred])) - } - - /* copy all instructions */ - for _, ins := range bb.Ins { - ins = ins.Clone() - ret.Ins = append(ret.Ins, ins) - replaceregs(rr, ins) - } - - /* copy the terminator */ - ret.Term = bb.Term.Clone().(IrTerminator) - replaceregs(rr, ret.Term) - - /* link to the predecessor */ - if sw, ok = pred.Term.(*IrSwitch); !ok { - panic("invalid block terminator: " + pred.Term.String()) - } - - /* check for default branch */ - if sw.Ln.To == bb { - sw.Ln.To = ret - continue - } - - /* replace the switch targets */ - for v, b := range sw.Br { - if b.To == bb { - sw.Br[v] = &IrBranch { - To : ret, - Likeliness : b.Likeliness, - } - } - } - } - } - - /* rebuild & cleanup the graph if needed */ - if more { - cfg.Rebuild() - new(BlockMerge).Apply(cfg) - } - } + more := true + rets := make([]*BasicBlock, 0, 1) + + /* register replacer */ + replaceregs := func(rr map[Reg]Reg, ins IrNode) { + var v Reg + var ok bool + var use IrUsages + var def IrDefinitions + + /* replace register usages */ + if use, ok = ins.(IrUsages); ok { + for _, r := range use.Usages() { + if v, ok = rr[*r]; ok { + *r = v + } + } + } + + /* replace register definitions */ + if def, ok = ins.(IrDefinitions); ok { + for _, r := range def.Definitions() { + if v, ok = rr[*r]; ok { + *r = v + } + } + } + } + + /* loop until no more modifications */ + for more { + more = false + rets = rets[:0] + + /* Phase 1: Find the return blocks that has more than one predecessors */ + for _, bb := range cfg.PostOrder().Reversed() { + if _, ok := bb.Term.(*IrReturn); ok && len(bb.Pred) > 1 { + more = true + rets = append(rets, bb) + } + } + + /* Phase 2: Spread the blocks to it's predecessors */ + for _, bb := range rets { + for _, pred := range bb.Pred { + var ok bool + var sw *IrSwitch + + /* register mappings */ + rr := make(map[Reg]Reg) + nb := len(bb.Phi) + len(bb.Ins) + + /* allocate registers for Phi definitions */ + for _, phi := range bb.Phi { + rr[phi.R] = cfg.CreateRegister(phi.R.Ptr()) + } + + /* allocate registers for instruction definitions */ + for _, ins := range bb.Ins { + if def, ok := ins.(IrDefinitions); ok { + for _, r := range def.Definitions() { + rr[*r] = cfg.CreateRegister(r.Ptr()) + } + } + } + + /* create a new basic block */ + ret := cfg.CreateBlock() + ret.Ins = make([]IrNode, 0, nb) + ret.Pred = []*BasicBlock{pred} + + /* add copy instruction for Phi nodes */ + for _, phi := range bb.Phi { + ret.Ins = append(ret.Ins, IrCopy(rr[phi.R], *phi.V[pred])) + } + + /* copy all instructions */ + for _, ins := range bb.Ins { + ins = ins.Clone() + ret.Ins = append(ret.Ins, ins) + replaceregs(rr, ins) + } + + /* copy the terminator */ + ret.Term = bb.Term.Clone().(IrTerminator) + replaceregs(rr, ret.Term) + + /* link to the predecessor */ + if sw, ok = pred.Term.(*IrSwitch); !ok { + panic("invalid block terminator: " + pred.Term.String()) + } + + /* check for default branch */ + if sw.Ln.To == bb { + sw.Ln.To = ret + continue + } + + /* replace the switch targets */ + for v, b := range sw.Br { + if b.To == bb { + sw.Br[v] = &IrBranch{ + To: ret, + Likeliness: b.Likeliness, + } + } + } + } + } + + /* rebuild & cleanup the graph if needed */ + if more { + cfg.Rebuild() + new(BlockMerge).Apply(cfg) + } + } } diff --git a/internal/atm/ssa/pass_splitcritical.go b/internal/atm/ssa/pass_splitcritical.go index 095fa2e..7499fe7 100644 --- a/internal/atm/ssa/pass_splitcritical.go +++ b/internal/atm/ssa/pass_splitcritical.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,8 +17,8 @@ package ssa type _CrEdge struct { - to *BasicBlock - from *BasicBlock + to *BasicBlock + from *BasicBlock } // SplitCritical splits critical edges (those that go from a block with @@ -30,68 +30,68 @@ type _CrEdge struct { type SplitCritical struct{} func (SplitCritical) Apply(cfg *CFG) { - var nb int - var edges []_CrEdge + var nb int + var edges []_CrEdge - /* find all critical edges */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - if len(bb.Pred) > 1 { - for _, p := range bb.Pred { - nb = 0 - tr := p.Term.Successors() + /* find all critical edges */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + if len(bb.Pred) > 1 { + for _, p := range bb.Pred { + nb = 0 + tr := p.Term.Successors() - /* check for successors */ - for nb < 2 && tr.Next() { - nb++ - } + /* check for successors */ + for nb < 2 && tr.Next() { + nb++ + } - /* the predecessor have more than 1 successors, this is a critcal edge */ - if nb > 1 { - edges = append(edges, _CrEdge { - to : bb, - from : p, - }) - } - } - } - }) + /* the predecessor have more than 1 successor, this is a critical edge */ + if nb > 1 { + edges = append(edges, _CrEdge{ + to: bb, + from: p, + }) + } + } + } + }) - /* insert empty block between the edges */ - for _, e := range edges { - bb := cfg.CreateBlock() - bb.Term = IrArchJump(e.to) - bb.Pred = []*BasicBlock { e.from } + /* insert empty block between the edges */ + for _, e := range edges { + bb := cfg.CreateBlock() + bb.Term = IrArchJump(e.to) + bb.Pred = []*BasicBlock{e.from} - /* update the successor */ - for it := e.from.Term.Successors(); it.Next(); { - if it.Block() == e.to { - it.UpdateBlock(bb) - break - } - } + /* update the successor */ + for it := e.from.Term.Successors(); it.Next(); { + if it.Block() == e.to { + it.UpdateBlock(bb) + break + } + } - /* update the predecessor */ - for i, p := range e.to.Pred { - if p == e.from { - e.to.Pred[i] = bb - break - } - } + /* update the predecessor */ + for i, p := range e.to.Pred { + if p == e.from { + e.to.Pred[i] = bb + break + } + } - /* update the Phi nodes */ - for _, p := range e.to.Phi { - for b, r := range p.V { - if b == e.from { - p.V[bb] = r - delete(p.V, b) - break - } - } - } - } + /* update the Phi nodes */ + for _, p := range e.to.Phi { + for b, r := range p.V { + if b == e.from { + p.V[bb] = r + delete(p.V, b) + break + } + } + } + } - /* rebuild the CFG if needed */ - if len(edges) != 0 { - cfg.Rebuild() - } + /* rebuild the CFG if needed */ + if len(edges) != 0 { + cfg.Rebuild() + } } diff --git a/internal/atm/ssa/pass_stack_liveness.go b/internal/atm/ssa/pass_stack_liveness.go index 3f4ad62..addfa17 100644 --- a/internal/atm/ssa/pass_stack_liveness.go +++ b/internal/atm/ssa/pass_stack_liveness.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,119 +17,122 @@ package ssa import ( - `fmt` + "fmt" ) // StackLiveness calculates the liveness of each stack slot. type StackLiveness struct{} func (self StackLiveness) livein(lr map[Pos]SlotSet, bb *BasicBlock, in map[int]SlotSet, out map[int]SlotSet) SlotSet { - var ok bool - var ss SlotSet - var sp *IrSpill - - /* check for cached live-in sets */ - if ss, ok = in[bb.Id]; ok { - return ss.clone() - } - - /* calculate the live-out set of current block */ - tr := bb.Term - sl := self.liveout(lr, bb, in, out).clone() - - /* assume all terminators are non-definitive */ - if _, ok = tr.(IrDefinitions); ok { - panic("regalloc: definitions within terminators") - } - - /* mark live range of the terminator */ - rr := sl.clone() - lr[pos(bb, _P_term)] = rr - - /* live(i-1) = use(i) ∪ (live(i) - { def(i) }) */ - for i := len(bb.Ins) - 1; i >= 0; i-- { - ins := bb.Ins[i] - sp, ok = ins.(*IrSpill) - - /* only account for pointer slots */ - if !ok || !sp.S.IsPtr() { - lr[pos(bb, i)] = sl.clone() - continue - } - - /* handle spill operations */ - switch sp.Op { - default: { - panic("stackmap: invalid spill Op") - } - - /* store operaion marks the value is alive since here */ - case IrSpillStore: { - if sl.remove(sp.S) { - lr[pos(bb, i)] = sl.clone() - } else { - panic(fmt.Sprintf("stackmap: killing non-existing value %s at %s:%d. ins = %s", sp.S, bb, i, bb.Ins[i])) - } - } - - /* load operaion marks the value is alive until here */ - case IrSpillReload: { - if sl.add(sp.S) { - lr[pos(bb, i)] = sl.clone() - } - } - } - } - - /* should not have any Phi nodes */ - if len(bb.Phi) != 0 { - panic("regalloc: unexpected Phi nodes") - } - - /* update the cache */ - in[bb.Id] = sl.clone() - return sl + var ok bool + var ss SlotSet + var sp *IrSpill + + /* check for cached live-in sets */ + if ss, ok = in[bb.Id]; ok { + return ss.clone() + } + + /* calculate the live-out set of current block */ + tr := bb.Term + sl := self.liveout(lr, bb, in, out).clone() + + /* assume all terminators are non-definitive */ + if _, ok = tr.(IrDefinitions); ok { + panic("regalloc: definitions within terminators") + } + + /* mark live range of the terminator */ + rr := sl.clone() + lr[pos(bb, _P_term)] = rr + + /* live(i-1) = use(i) ∪ (live(i) - { def(i) }) */ + for i := len(bb.Ins) - 1; i >= 0; i-- { + ins := bb.Ins[i] + sp, ok = ins.(*IrSpill) + + /* only account for pointer slots */ + if !ok || !sp.S.IsPtr() { + lr[pos(bb, i)] = sl.clone() + continue + } + + /* handle spill operations */ + switch sp.Op { + default: + { + panic("stackmap: invalid spill Op") + } + + /* store operation marks the value is alive since here */ + case IrSpillStore: + { + if sl.remove(sp.S) { + lr[pos(bb, i)] = sl.clone() + } else { + panic(fmt.Sprintf("stackmap: killing non-existing value %s at %s:%d. ins = %s", sp.S, bb, i, bb.Ins[i])) + } + } + + /* load operation marks the value is alive until here */ + case IrSpillReload: + { + if sl.add(sp.S) { + lr[pos(bb, i)] = sl.clone() + } + } + } + } + + /* should not have any Phi nodes */ + if len(bb.Phi) != 0 { + panic("regalloc: unexpected Phi nodes") + } + + /* update the cache */ + in[bb.Id] = sl.clone() + return sl } func (self StackLiveness) liveout(lr map[Pos]SlotSet, bb *BasicBlock, in map[int]SlotSet, out map[int]SlotSet) SlotSet { - var ok bool - var ss SlotSet - var it IrSuccessors - - /* check for cached live-out sets */ - if ss, ok = out[bb.Id]; ok { - return ss - } - - /* check for return blocks */ - if _, ok = IrTryIntoArchReturn(bb.Term); ok { - out[bb.Id] = make(SlotSet) - return ss - } - - /* create a new register set */ - ss = make(SlotSet) - it = bb.Term.Successors() - - /* live-out(p) = ∑(live-in(succ(p))) */ - for out[bb.Id] = nil; it.Next(); { - for sl := range self.livein(lr, it.Block(), in, out) { - ss.add(sl) - } - } - - /* update cache */ - out[bb.Id] = ss - return ss + var ok bool + var ss SlotSet + var it IrSuccessors + + /* check for cached live-out sets */ + if ss, ok = out[bb.Id]; ok { + return ss + } + + /* check for return blocks */ + if _, ok = IrTryIntoArchReturn(bb.Term); ok { + out[bb.Id] = make(SlotSet) + return ss + } + + /* create a new register set */ + ss = make(SlotSet) + it = bb.Term.Successors() + + /* live-out(p) = ∑(live-in(succ(p))) */ + for out[bb.Id] = nil; it.Next(); { + for sl := range self.livein(lr, it.Block(), in, out) { + ss.add(sl) + } + } + + /* update cache */ + out[bb.Id] = ss + return ss } func (self StackLiveness) liveness(cfg *CFG) { - for ss := range self.livein(cfg.Func.Liveness, cfg.Root, make(map[int]SlotSet), make(map[int]SlotSet)) { - panic("stackliveness: live slot at entry: " + ss.String()) - } + for ss := range self.livein(cfg.Func.Liveness, cfg.Root, make(map[int]SlotSet), make(map[int]SlotSet)) { + panic("stackliveness: live slot at entry: " + ss.String()) + } } func (self StackLiveness) Apply(cfg *CFG) { - cfg.Func.Liveness = make(map[Pos]SlotSet) - self.liveness(cfg) + cfg.Func.Liveness = make(map[Pos]SlotSet) + self.liveness(cfg) } diff --git a/internal/atm/ssa/pass_tdce.go b/internal/atm/ssa/pass_tdce.go index 4aaa225..36f7bc9 100644 --- a/internal/atm/ssa/pass_tdce.go +++ b/internal/atm/ssa/pass_tdce.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,142 +16,142 @@ package ssa -// TDCE removes trivial dead-code such as unused register definations from CFG. +// TDCE removes trivial dead-code such as unused register definitions from CFG. type TDCE struct{} func (TDCE) Apply(cfg *CFG) { - for { - done := true - decl := make(map[Reg]struct{}) - - /* Phase 1: Mark all the definations */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - var ok bool - var defs IrDefinitions - - /* mark all definations in Phi nodes */ - for _, v := range bb.Phi { - for _, r := range v.Definitions() { - decl[*r] = struct{}{} - } - } - - /* mark all definations in instructions if any */ - for _, v := range bb.Ins { - if defs, ok = v.(IrDefinitions); ok { - for _, r := range defs.Definitions() { - decl[*r] = struct{}{} - } - } - } - - /* mark all definations in terminators if any */ - if defs, ok = bb.Term.(IrDefinitions); ok { - for _, r := range defs.Definitions() { - decl[*r] = struct{}{} - } - } - }) - - /* Phase 2: Find all register usages */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - var ok bool - var use IrUsages - - /* mark all usages in Phi nodes */ - for _, v := range bb.Phi { - for _, r := range v.Usages() { - delete(decl, *r) - } - } - - /* mark all usages in instructions if any */ - for _, v := range bb.Ins { - if use, ok = v.(IrUsages); ok { - for _, r := range use.Usages() { - delete(decl, *r) - } - } - } - - /* mark usages in the terminator if any */ - if use, ok = bb.Term.(IrUsages); ok { - for _, r := range use.Usages() { - delete(decl, *r) - } - } - }) - - /* Phase 3: Remove all unused declarations */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - var ok bool - var defs IrDefinitions - - /* replace unused Phi assigments with zero registers */ - for _, v := range bb.Phi { - for _, r := range v.Definitions() { - if _, ok = decl[*r]; ok && r.Kind() != K_zero { - *r, done = r.Zero(), false - } - } - } - - /* replace unused instruction assigments with zero registers */ - for _, v := range bb.Ins { - if defs, ok = v.(IrDefinitions); ok { - for _, r := range defs.Definitions() { - if _, ok = decl[*r]; ok && r.Kind() != K_zero { - *r, done = r.Zero(), false - } - } - } - } - - /* replace unused terminator assigments with zero registers */ - if defs, ok = bb.Term.(IrDefinitions); ok { - for _, r := range defs.Definitions() { - if _, ok = decl[*r]; ok && r.Kind() != K_zero { - *r, done = r.Zero(), false - } - } - } - }) - - /* Phase 4: Remove the entire defination if it's all zeros */ - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - phi, ins := bb.Phi, bb.Ins - bb.Phi, bb.Ins = bb.Phi[:0], bb.Ins[:0] - - /* remove Phi nodes that don't have any effects */ - for _, v := range phi { - for _, r := range v.Definitions() { - if r.Kind() != K_zero { - bb.Phi = append(bb.Phi, v) - break - } - } - } - - /* remove instructions that don't have any effects */ - for _, v := range ins { - if _, ok := v.(IrImpure); ok { - bb.Ins = append(bb.Ins, v) - } else if d, ok := v.(IrDefinitions); !ok { - bb.Ins = append(bb.Ins, v) - } else { - for _, r := range d.Definitions() { - if r.Kind() != K_zero { - bb.Ins = append(bb.Ins, v) - break - } - } - } - } - }) - - /* no more modifications */ - if done { - break - } - } + for { + done := true + decl := make(map[Reg]struct{}) + + /* Phase 1: Mark all the definitions */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + var ok bool + var defs IrDefinitions + + /* mark all definitions in Phi nodes */ + for _, v := range bb.Phi { + for _, r := range v.Definitions() { + decl[*r] = struct{}{} + } + } + + /* mark all definitions in instructions if any */ + for _, v := range bb.Ins { + if defs, ok = v.(IrDefinitions); ok { + for _, r := range defs.Definitions() { + decl[*r] = struct{}{} + } + } + } + + /* mark all definitions in terminators if any */ + if defs, ok = bb.Term.(IrDefinitions); ok { + for _, r := range defs.Definitions() { + decl[*r] = struct{}{} + } + } + }) + + /* Phase 2: Find all register usages */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + var ok bool + var use IrUsages + + /* mark all usages in Phi nodes */ + for _, v := range bb.Phi { + for _, r := range v.Usages() { + delete(decl, *r) + } + } + + /* mark all usages in instructions if any */ + for _, v := range bb.Ins { + if use, ok = v.(IrUsages); ok { + for _, r := range use.Usages() { + delete(decl, *r) + } + } + } + + /* mark usages in the terminator if any */ + if use, ok = bb.Term.(IrUsages); ok { + for _, r := range use.Usages() { + delete(decl, *r) + } + } + }) + + /* Phase 3: Remove all unused declarations */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + var ok bool + var defs IrDefinitions + + /* replace unused Phi assignments with zero registers */ + for _, v := range bb.Phi { + for _, r := range v.Definitions() { + if _, ok = decl[*r]; ok && r.Kind() != K_zero { + *r, done = r.Zero(), false + } + } + } + + /* replace unused instruction assignments with zero registers */ + for _, v := range bb.Ins { + if defs, ok = v.(IrDefinitions); ok { + for _, r := range defs.Definitions() { + if _, ok = decl[*r]; ok && r.Kind() != K_zero { + *r, done = r.Zero(), false + } + } + } + } + + /* replace unused terminator assignments with zero registers */ + if defs, ok = bb.Term.(IrDefinitions); ok { + for _, r := range defs.Definitions() { + if _, ok = decl[*r]; ok && r.Kind() != K_zero { + *r, done = r.Zero(), false + } + } + } + }) + + /* Phase 4: Remove the entire definition if it's all zeros */ + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + phi, ins := bb.Phi, bb.Ins + bb.Phi, bb.Ins = bb.Phi[:0], bb.Ins[:0] + + /* remove Phi nodes that don't have any effects */ + for _, v := range phi { + for _, r := range v.Definitions() { + if r.Kind() != K_zero { + bb.Phi = append(bb.Phi, v) + break + } + } + } + + /* remove instructions that don't have any effects */ + for _, v := range ins { + if _, ok := v.(IrImpure); ok { + bb.Ins = append(bb.Ins, v) + } else if d, ok := v.(IrDefinitions); !ok { + bb.Ins = append(bb.Ins, v) + } else { + for _, r := range d.Definitions() { + if r.Kind() != K_zero { + bb.Ins = append(bb.Ins, v) + break + } + } + } + } + }) + + /* no more modifications */ + if done { + break + } + } } diff --git a/internal/atm/ssa/pass_zeroreg.go b/internal/atm/ssa/pass_zeroreg.go index be440d3..4891673 100644 --- a/internal/atm/ssa/pass_zeroreg.go +++ b/internal/atm/ssa/pass_zeroreg.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ package ssa import ( - `runtime` + "runtime" ) // ZeroReg replaces read to %z or %nil to a register that was @@ -26,83 +26,83 @@ import ( type ZeroReg struct{} func (ZeroReg) replace(cfg *CFG) { - cfg.PostOrder().ForEach(func(bb *BasicBlock) { - var ok bool - var rr *Reg - var use IrUsages + cfg.PostOrder().ForEach(func(bb *BasicBlock) { + var ok bool + var rr *Reg + var use IrUsages - /* create the instruction buffer */ - ins := bb.Ins - bb.Ins = make([]IrNode, 0, len(ins)) + /* create the instruction buffer */ + ins := bb.Ins + bb.Ins = make([]IrNode, 0, len(ins)) - /* zero register replacer */ - replacez := func(v IrUsages, ins *[]IrNode, tail IrNode) { - var z Reg - var r *Reg + /* zero register replacer */ + replacez := func(v IrUsages, ins *[]IrNode, tail IrNode) { + var z Reg + var r *Reg - /* insert an zeroing instruction if needed */ - for _, r = range v.Usages() { - if r.Kind() == K_zero { - z = cfg.CreateRegister(false) - *ins = append(*ins, IrArchZero(z)) - break - } - } + /* insert an zeroing instruction if needed */ + for _, r = range v.Usages() { + if r.Kind() == K_zero { + z = cfg.CreateRegister(false) + *ins = append(*ins, IrArchZero(z)) + break + } + } - /* substitute all the zero register usages */ - for _, r = range v.Usages() { - if r.Kind() == K_zero { - *r = z - } - } + /* substitute all the zero register usages */ + for _, r = range v.Usages() { + if r.Kind() == K_zero { + *r = z + } + } - /* add the instruction if needed */ - if tail != nil { - *ins = append(*ins, tail) - } - } + /* add the instruction if needed */ + if tail != nil { + *ins = append(*ins, tail) + } + } - /* scan all the Phi nodes */ - for _, p := range bb.Pred { - var z Reg - var v *IrPhi + /* scan all the Phi nodes */ + for _, p := range bb.Pred { + var z Reg + var v *IrPhi - /* insert an zeroing instruction to it's predecessor if needed */ - for _, v = range bb.Phi { - if v.V[p].Kind() == K_zero { - z = cfg.CreateRegister(false) - p.Ins = append(p.Ins, IrArchZero(z)) - break - } - } + /* insert an zeroing instruction to it's predecessor if needed */ + for _, v = range bb.Phi { + if v.V[p].Kind() == K_zero { + z = cfg.CreateRegister(false) + p.Ins = append(p.Ins, IrArchZero(z)) + break + } + } - /* substitute all the zero register usages */ - for _, v = range bb.Phi { - if rr = v.V[p]; rr.Kind() == K_zero { - *rr = z - } - } - } + /* substitute all the zero register usages */ + for _, v = range bb.Phi { + if rr = v.V[p]; rr.Kind() == K_zero { + *rr = z + } + } + } - /* scan all the instructions */ - for _, v := range ins { - if use, ok = v.(IrUsages); ok { - replacez(use, &bb.Ins, v) - } else { - bb.Ins = append(bb.Ins, v) - } - } + /* scan all the instructions */ + for _, v := range ins { + if use, ok = v.(IrUsages); ok { + replacez(use, &bb.Ins, v) + } else { + bb.Ins = append(bb.Ins, v) + } + } - /* scan the terminator */ - if use, ok = bb.Term.(IrUsages); ok { - replacez(use, &bb.Ins, nil) - } - }) + /* scan the terminator */ + if use, ok = bb.Term.(IrUsages); ok { + replacez(use, &bb.Ins, nil) + } + }) } //goland:noinspection GoBoolExpressions func (self ZeroReg) Apply(cfg *CFG) { - if runtime.GOARCH == "amd64" { - self.replace(cfg) - } + if runtime.GOARCH == "amd64" { + self.replace(cfg) + } } diff --git a/internal/atm/ssa/phi.go b/internal/atm/ssa/phi.go index 8374c36..bacc24b 100644 --- a/internal/atm/ssa/phi.go +++ b/internal/atm/ssa/phi.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,137 +17,137 @@ package ssa import ( - `sort` + "sort" - `github.com/oleiade/lane` + "github.com/oleiade/lane" ) type _PhiDesc struct { - r Reg - b []*BasicBlock + r Reg + b []*BasicBlock } func appendReg(buf map[Reg]bool, r Reg) map[Reg]bool { - if buf == nil { - return map[Reg]bool { r: true } - } else { - buf[r] = true - return buf - } + if buf == nil { + return map[Reg]bool{r: true} + } else { + buf[r] = true + return buf + } } func appendBlock(buf map[int]*BasicBlock, bb *BasicBlock) map[int]*BasicBlock { - if buf == nil { - return map[int]*BasicBlock { bb.Id: bb } - } else { - buf[bb.Id] = bb - return buf - } + if buf == nil { + return map[int]*BasicBlock{bb.Id: bb} + } else { + buf[bb.Id] = bb + return buf + } } func insertPhiNodes(cfg *CFG) { - q := lane.NewQueue() - phi := make(map[Reg]map[int]bool) - orig := make(map[int]map[Reg]bool) - defs := make(map[Reg]map[int]*BasicBlock) - - /* find out all the variable origins */ - for q.Enqueue(cfg.Root); !q.Empty(); { - p := q.Dequeue().(*BasicBlock) - addImmediateDominated(cfg.DominatorOf, p, q) - - /* mark all the definition sites */ - for _, ins := range p.Ins { - if def, ok := ins.(IrDefinitions); ok { - for _, d := range def.Definitions() { - if k := d.Kind(); k != K_zero { - orig[p.Id] = appendReg(orig[p.Id], *d) - } - } - } - } - } - - /* find out all the variable defination sites */ - for q.Enqueue(cfg.Root); !q.Empty(); { - p := q.Dequeue().(*BasicBlock) - addImmediateDominated(cfg.DominatorOf, p, q) - - /* mark all the defination sites */ - for def := range orig[p.Id] { - defs[def] = appendBlock(defs[def], p) - } - } - - /* reserve buffer for Phi descriptors */ - nb := len(defs) - pd := make([]_PhiDesc, nb) - - /* dump the descriptors */ - for r, v := range defs { - n := len(v) - b := make([]*BasicBlock, 0, n) - - /* dump the blocks */ - for _, p := range v { - b = append(b, p) - } - - /* sort blocks by ID */ - sort.Slice(b, func(i int, j int) bool { - return b[i].Id < b[j].Id - }) - - /* add the descriptor */ - pd = append(pd, _PhiDesc { - r: r, - b: b, - }) - } - - /* sort descriptors by register */ - sort.Slice(pd, func(i int, j int) bool { - return pd[i].r < pd[j].r - }) - - /* insert Phi node for every variable */ - for _, p := range pd { - for len(p.b) != 0 { - n := p.b[0] - p.b = p.b[1:] - - /* insert Phi nodes */ - for _, y := range cfg.DominanceFrontier[n.Id] { - if rem := phi[p.r]; !rem[y.Id] { - id := y.Id - src := make(map[*BasicBlock]*Reg) - - /* mark as processed */ - if rem != nil { - rem[id] = true - } else { - phi[p.r] = map[int]bool { id: true } - } - - /* build the Phi node args */ - for _, pred := range y.Pred { - src[pred] = new(Reg) - *src[pred] = p.r - } - - /* insert a new Phi node */ - y.Phi = append(y.Phi, &IrPhi { - R: p.r, - V: src, - }) - - /* a node may contain both an ordinary definition and a - * Phi node for the same variable */ - if !orig[y.Id][p.r] { - p.b = append(p.b, y) - } - } - } - } - } + q := lane.NewQueue() + phi := make(map[Reg]map[int]bool) + orig := make(map[int]map[Reg]bool) + defs := make(map[Reg]map[int]*BasicBlock) + + /* find out all the variable origins */ + for q.Enqueue(cfg.Root); !q.Empty(); { + p := q.Dequeue().(*BasicBlock) + addImmediateDominated(cfg.DominatorOf, p, q) + + /* mark all the definition sites */ + for _, ins := range p.Ins { + if def, ok := ins.(IrDefinitions); ok { + for _, d := range def.Definitions() { + if k := d.Kind(); k != K_zero { + orig[p.Id] = appendReg(orig[p.Id], *d) + } + } + } + } + } + + /* find out all the variable definition sites */ + for q.Enqueue(cfg.Root); !q.Empty(); { + p := q.Dequeue().(*BasicBlock) + addImmediateDominated(cfg.DominatorOf, p, q) + + /* mark all the definition sites */ + for def := range orig[p.Id] { + defs[def] = appendBlock(defs[def], p) + } + } + + /* reserve buffer for Phi descriptors */ + nb := len(defs) + pd := make([]_PhiDesc, nb) + + /* dump the descriptors */ + for r, v := range defs { + n := len(v) + b := make([]*BasicBlock, 0, n) + + /* dump the blocks */ + for _, p := range v { + b = append(b, p) + } + + /* sort blocks by ID */ + sort.Slice(b, func(i int, j int) bool { + return b[i].Id < b[j].Id + }) + + /* add the descriptor */ + pd = append(pd, _PhiDesc{ + r: r, + b: b, + }) + } + + /* sort descriptors by register */ + sort.Slice(pd, func(i int, j int) bool { + return pd[i].r < pd[j].r + }) + + /* insert Phi node for every variable */ + for _, p := range pd { + for len(p.b) != 0 { + n := p.b[0] + p.b = p.b[1:] + + /* insert Phi nodes */ + for _, y := range cfg.DominanceFrontier[n.Id] { + if rem := phi[p.r]; !rem[y.Id] { + id := y.Id + src := make(map[*BasicBlock]*Reg) + + /* mark as processed */ + if rem != nil { + rem[id] = true + } else { + phi[p.r] = map[int]bool{id: true} + } + + /* build the Phi node args */ + for _, pred := range y.Pred { + src[pred] = new(Reg) + *src[pred] = p.r + } + + /* insert a new Phi node */ + y.Phi = append(y.Phi, &IrPhi{ + R: p.r, + V: src, + }) + + /* a node may contain both an ordinary definition and a + * Phi node for the same variable */ + if !orig[y.Id][p.r] { + p.b = append(p.b, y) + } + } + } + } + } } diff --git a/internal/atm/ssa/pos.go b/internal/atm/ssa/pos.go index 739b4b3..22ea34e 100644 --- a/internal/atm/ssa/pos.go +++ b/internal/atm/ssa/pos.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,31 +17,31 @@ package ssa import ( - `fmt` - `math` + "fmt" + "math" ) const ( - _P_term = math.MaxUint32 + _P_term = math.MaxUint32 ) type Pos struct { - B *BasicBlock - I int + B *BasicBlock + I int } func pos(bb *BasicBlock, i int) Pos { - return Pos { bb, i } + return Pos{bb, i} } func (self Pos) String() string { - if self.I == _P_term { - return fmt.Sprintf("bb_%d.term", self.B.Id) - } else { - return fmt.Sprintf("bb_%d.ins[%d]", self.B.Id, self.I) - } + if self.I == _P_term { + return fmt.Sprintf("bb_%d.term", self.B.Id) + } else { + return fmt.Sprintf("bb_%d.ins[%d]", self.B.Id, self.I) + } } func (self Pos) isPriorTo(other Pos) bool { - return self.B.Id < other.B.Id || (self.I < other.I && self.B.Id == other.B.Id) + return self.B.Id < other.B.Id || (self.I < other.I && self.B.Id == other.B.Id) } diff --git a/internal/atm/ssa/rename.go b/internal/atm/ssa/rename.go index 1ea7180..dc13e46 100644 --- a/internal/atm/ssa/rename.go +++ b/internal/atm/ssa/rename.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,179 +17,179 @@ package ssa import ( - `github.com/oleiade/lane` + "github.com/oleiade/lane" ) type _Renamer struct { - count map[Reg]int - stack map[Reg][]int + count map[Reg]int + stack map[Reg][]int } func newRenamer() _Renamer { - return _Renamer { - count: make(map[Reg]int), - stack: make(map[Reg][]int), - } + return _Renamer{ + count: make(map[Reg]int), + stack: make(map[Reg][]int), + } } func (self _Renamer) popr(r Reg) { - if n := len(self.stack[r]); n != 0 { - self.stack[r] = self.stack[r][:n - 1] - } + if n := len(self.stack[r]); n != 0 { + self.stack[r] = self.stack[r][:n-1] + } } func (self _Renamer) topr(r Reg) int { - if n := len(self.stack[r]); n == 0 { - return 0 - } else { - return self.stack[r][n - 1] - } + if n := len(self.stack[r]); n == 0 { + return 0 + } else { + return self.stack[r][n-1] + } } func (self _Renamer) pushr(r Reg) (i int) { - i = self.count[r] - self.count[r] = i + 1 - self.stack[r] = append(self.stack[r], i) - return + i = self.count[r] + self.count[r] = i + 1 + self.stack[r] = append(self.stack[r], i) + return } func (self _Renamer) renameuses(ins IrNode) { - if u, ok := ins.(IrUsages); ok { - for _, a := range u.Usages() { - *a = a.Derive(self.topr(*a)) - } - } + if u, ok := ins.(IrUsages); ok { + for _, a := range u.Usages() { + *a = a.Derive(self.topr(*a)) + } + } } func (self _Renamer) renamedefs(ins IrNode, buf *[]Reg) { - if s, ok := ins.(IrDefinitions); ok { - for _, def := range s.Definitions() { - *buf = append(*buf, *def) - *def = def.Derive(self.pushr(*def)) - } - } + if s, ok := ins.(IrDefinitions); ok { + for _, def := range s.Definitions() { + *buf = append(*buf, *def) + *def = def.Derive(self.pushr(*def)) + } + } } func (self _Renamer) renameblock(cfg *CFG, bb *BasicBlock) { - var r Reg - var d []Reg - var n IrNode - - /* rename Phi nodes */ - for _, n = range bb.Phi { - self.renamedefs(n, &d) - } - - /* rename body */ - for _, n = range bb.Ins { - self.renameuses(n) - self.renamedefs(n, &d) - } - - /* get the successor iterator */ - tr := bb.Term - it := tr.Successors() - - /* rename terminators */ - self.renameuses(tr) - self.renamedefs(tr, &d) - - /* rename all the Phi node of it's successors */ - for it.Next() { - for _, phi := range it.Block().Phi { - r = *phi.V[bb] - phi.V[bb] = regnewref(r.Derive(self.topr(r))) - } - } - - /* rename all it's children in the dominator tree */ - for _, p := range cfg.DominatorOf[bb.Id] { - self.renameblock(cfg, p) - } - - /* pop the definations */ - for _, s := range d { - self.popr(s) - } + var r Reg + var d []Reg + var n IrNode + + /* rename Phi nodes */ + for _, n = range bb.Phi { + self.renamedefs(n, &d) + } + + /* rename body */ + for _, n = range bb.Ins { + self.renameuses(n) + self.renamedefs(n, &d) + } + + /* get the successor iterator */ + tr := bb.Term + it := tr.Successors() + + /* rename terminators */ + self.renameuses(tr) + self.renamedefs(tr, &d) + + /* rename all the Phi node of it's successors */ + for it.Next() { + for _, phi := range it.Block().Phi { + r = *phi.V[bb] + phi.V[bb] = regnewref(r.Derive(self.topr(r))) + } + } + + /* rename all it's children in the dominator tree */ + for _, p := range cfg.DominatorOf[bb.Id] { + self.renameblock(cfg, p) + } + + /* pop the definitions */ + for _, s := range d { + self.popr(s) + } } func renameRegisters(cfg *CFG) { - newRenamer().renameblock(cfg, cfg.Root) - normalizeRegisters(cfg) + newRenamer().renameblock(cfg, cfg.Root) + normalizeRegisters(cfg) } func assignRegisters(rr []*Reg, rm map[Reg]Reg, cfg *CFG) { - for _, r := range rr { - if r.Kind() != K_zero { - if v, ok := rm[*r]; ok { - panic("register redefined: " + r.String()) - } else { - v = r.Normalize(cfg.allocreg()) - *r, rm[*r] = v, v - } - } - } + for _, r := range rr { + if r.Kind() != K_zero { + if v, ok := rm[*r]; ok { + panic("register redefined: " + r.String()) + } else { + v = r.Normalize(cfg.allocreg()) + *r, rm[*r] = v, v + } + } + } } func replaceRegisters(rr []*Reg, rm map[Reg]Reg) { - for _, r := range rr { - if r.Kind() != K_zero { - if v, ok := rm[*r]; ok { - *r = v - } else { - panic("use of undefined register: " + r.String()) - } - } - } + for _, r := range rr { + if r.Kind() != K_zero { + if v, ok := rm[*r]; ok { + *r = v + } else { + panic("use of undefined register: " + r.String()) + } + } + } } func normalizeRegisters(cfg *CFG) { - q := lane.NewQueue() - r := make(map[Reg]Reg) - - /* find all the register definations */ - for q.Enqueue(cfg.Root); !q.Empty(); { - p := q.Dequeue().(*BasicBlock) - addImmediateDominated(cfg.DominatorOf, p, q) - - /* assign Phi nodes */ - for _, n := range p.Phi { - assignRegisters(n.Definitions(), r, cfg) - } - - /* assign instructions */ - for _, n := range p.Ins { - if d, ok := n.(IrDefinitions); ok { - assignRegisters(d.Definitions(), r, cfg) - } - } - - /* assign terminators */ - if d, ok := p.Term.(IrDefinitions); ok { - assignRegisters(d.Definitions(), r, cfg) - } - } - - /* normalize each block */ - for q.Enqueue(cfg.Root); !q.Empty(); { - p := q.Dequeue().(*BasicBlock) - addImmediateDominated(cfg.DominatorOf, p, q) - - /* replace Phi nodes */ - for _, n := range p.Phi { - replaceRegisters(n.Usages(), r) - } - - /* replace instructions */ - for _, n := range p.Ins { - if u, ok := n.(IrUsages); ok { - replaceRegisters(u.Usages(), r) - } - } - - /* replace terminators */ - if u, ok := p.Term.(IrUsages); ok { - replaceRegisters(u.Usages(), r) - } - } + q := lane.NewQueue() + r := make(map[Reg]Reg) + + /* find all the register definitions */ + for q.Enqueue(cfg.Root); !q.Empty(); { + p := q.Dequeue().(*BasicBlock) + addImmediateDominated(cfg.DominatorOf, p, q) + + /* assign Phi nodes */ + for _, n := range p.Phi { + assignRegisters(n.Definitions(), r, cfg) + } + + /* assign instructions */ + for _, n := range p.Ins { + if d, ok := n.(IrDefinitions); ok { + assignRegisters(d.Definitions(), r, cfg) + } + } + + /* assign terminators */ + if d, ok := p.Term.(IrDefinitions); ok { + assignRegisters(d.Definitions(), r, cfg) + } + } + + /* normalize each block */ + for q.Enqueue(cfg.Root); !q.Empty(); { + p := q.Dequeue().(*BasicBlock) + addImmediateDominated(cfg.DominatorOf, p, q) + + /* replace Phi nodes */ + for _, n := range p.Phi { + replaceRegisters(n.Usages(), r) + } + + /* replace instructions */ + for _, n := range p.Ins { + if u, ok := n.(IrUsages); ok { + replaceRegisters(u.Usages(), r) + } + } + + /* replace terminators */ + if u, ok := p.Term.(IrUsages); ok { + replaceRegisters(u.Usages(), r) + } + } } diff --git a/internal/atm/ssa/slotset.go b/internal/atm/ssa/slotset.go index e8a3ac3..a627f84 100644 --- a/internal/atm/ssa/slotset.go +++ b/internal/atm/ssa/slotset.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,62 +17,64 @@ package ssa import ( - `fmt` - `sort` - `strings` + "fmt" + "sort" + "strings" ) type ( - SlotSet map[IrSpillSlot]struct{} + SlotSet map[IrSpillSlot]struct{} ) func (self SlotSet) add(r IrSpillSlot) bool { - if _, ok := self[r]; ok { - return false - } else { - self[r] = struct{}{} - return true - } + if _, ok := self[r]; ok { + return false + } else { + self[r] = struct{}{} + return true + } } func (self SlotSet) clone() (rs SlotSet) { - rs = make(SlotSet, len(self)) - for r := range self { rs.add(r) } - return + rs = make(SlotSet, len(self)) + for r := range self { + rs.add(r) + } + return } func (self SlotSet) remove(r IrSpillSlot) bool { - if _, ok := self[r]; !ok { - return false - } else { - delete(self, r) - return true - } + if _, ok := self[r]; !ok { + return false + } else { + delete(self, r) + return true + } } func (self SlotSet) String() string { - nb := len(self) - rs := make([]string, 0, nb) - rr := make([]IrSpillSlot, 0, nb) + nb := len(self) + rs := make([]string, 0, nb) + rr := make([]IrSpillSlot, 0, nb) - /* extract all slot */ - for r := range self { - rr = append(rr, r) - } + /* extract all slot */ + for r := range self { + rr = append(rr, r) + } - /* sort by slot ID */ - sort.Slice(rr, func(i int, j int) bool { - return rr[i] < rr[j] - }) + /* sort by slot ID */ + sort.Slice(rr, func(i int, j int) bool { + return rr[i] < rr[j] + }) - /* convert every slot */ - for _, r := range rr { - rs = append(rs, r.String()) - } + /* convert every slot */ + for _, r := range rr { + rs = append(rs, r.String()) + } - /* join them together */ - return fmt.Sprintf( - "{%s}", - strings.Join(rs, ", "), - ) + /* join them together */ + return fmt.Sprintf( + "{%s}", + strings.Join(rs, ", "), + ) } diff --git a/internal/atm/ssa/utils.go b/internal/atm/ssa/utils.go index a0ed908..f2e5668 100644 --- a/internal/atm/ssa/utils.go +++ b/internal/atm/ssa/utils.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,132 +17,146 @@ package ssa import ( - `math` - `strings` - `unsafe` + "math" + "strings" + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/oleiade/lane` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/oleiade/lane" ) func isu8(v int64) bool { - return v >= 0 && v <= math.MaxUint8 + return v >= 0 && v <= math.MaxUint8 } func isi32(v int64) bool { - return v >= math.MinInt32 && v <= math.MaxInt32 + return v >= math.MinInt32 && v <= math.MaxInt32 } func isp32(p unsafe.Pointer) bool { - return uintptr(p) <= math.MaxInt32 + return uintptr(p) <= math.MaxInt32 } func ri2reg(ri uint8) Reg { - if ri & hir.ArgPointer == 0 { - return Rv(hir.GenericRegister(ri & hir.ArgMask)) - } else { - return Rv(hir.PointerRegister(ri & hir.ArgMask)) - } + if ri&hir.ArgPointer == 0 { + return Rv(hir.GenericRegister(ri & hir.ArgMask)) + } else { + return Rv(hir.PointerRegister(ri & hir.ArgMask)) + } } func ri2regz(ri []uint8) Reg { - switch len(ri) { - case 0 : return Rz - case 1 : return ri2reg(ri[0]) - default : panic("invalid register count") - } + switch len(ri) { + case 0: + return Rz + case 1: + return ri2reg(ri[0]) + default: + panic("invalid register count") + } } func ri2regs(ri []uint8) []Reg { - ret := make([]Reg, len(ri)) - for i, r := range ri { ret[i] = ri2reg(r) } - return ret + ret := make([]Reg, len(ri)) + for i, r := range ri { + ret[i] = ri2reg(r) + } + return ret } func minint(a int, b int) int { - if a < b { - return a - } else { - return b - } + if a < b { + return a + } else { + return b + } } func cmpu64(a uint64, b uint64) int { - if a < b { - return -1 - } else if a > b { - return 1 - } else { - return 0 - } + if a < b { + return -1 + } else if a > b { + return 1 + } else { + return 0 + } } func addptr(p unsafe.Pointer, i int64) unsafe.Pointer { - return unsafe.Pointer(uintptr(p) + uintptr(i)) + return unsafe.Pointer(uintptr(p) + uintptr(i)) } func int2bool(v int) bool { - return v != 0 + return v != 0 } func bool2int(v bool) int { - if v { - return 1 - } else { - return 0 - } + if v { + return 1 + } else { + return 0 + } } func memsizec(n uint8) rune { - switch n { - case 1 : return 'b' - case 2 : return 'w' - case 4 : return 'l' - case 8 : return 'q' - default : panic("unreachable") - } + switch n { + case 1: + return 'b' + case 2: + return 'w' + case 4: + return 'l' + case 8: + return 'q' + default: + panic("unreachable") + } } func stacknew(v interface{}) (r *lane.Stack) { - r = lane.NewStack() - r.Push(v) - return + r = lane.NewStack() + r.Push(v) + return } func regorder(v Reg) int { - if v.Kind() != K_arch { - return int(v) - } else { - return math.MinInt64 - } + if v.Kind() != K_arch { + return int(v) + } else { + return math.MinInt64 + } } func regnewref(v Reg) (r *Reg) { - r = new(Reg) - *r = v - return + r = new(Reg) + *r = v + return } func regsliceref(v []Reg) (r []*Reg) { - r = make([]*Reg, len(v)) - for i := range v { r[i] = &v[i] } - return + r = make([]*Reg, len(v)) + for i := range v { + r[i] = &v[i] + } + return } func regslicerepr(v []Reg) string { - r := make([]string, 0, len(v)) - for _, x := range v { r = append(r, x.String()) } - return strings.Join(r, ", ") + r := make([]string, 0, len(v)) + for _, x := range v { + r = append(r, x.String()) + } + return strings.Join(r, ", ") } func regsliceclone(v []Reg) (r []Reg) { - r = make([]Reg, len(v)) - copy(r, v) - return + r = make([]Reg, len(v)) + copy(r, v) + return } func blockreverse(s []*BasicBlock) { - for i, j := 0, len(s) - 1; i < j; i, j = i + 1, j - 1 { - s[i], s[j] = s[j], s[i] - } + for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { + s[i], s[j] = s[j], s[i] + } } diff --git a/internal/binary/decoder/alloc.go b/internal/binary/decoder/alloc.go index fa28b44..41372ca 100644 --- a/internal/binary/decoder/alloc.go +++ b/internal/binary/decoder/alloc.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,10 +17,10 @@ package decoder import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" ) //go:noescape @@ -34,6 +34,6 @@ func makemap(t *rt.GoMapType, hint int, h *rt.GoMap) *rt.GoMap func mallocgc(size uintptr, typ *rt.GoType, needzero bool) unsafe.Pointer var ( - F_makemap = hir.RegisterGCall(makemap, emu_gcall_makemap) - F_mallocgc = hir.RegisterGCall(mallocgc, emu_gcall_mallocgc) + F_makemap = hir.RegisterGCall(makemap, emu_gcall_makemap) + F_mallocgc = hir.RegisterGCall(mallocgc, emu_gcall_mallocgc) ) diff --git a/internal/binary/decoder/alloc_emu.go b/internal/binary/decoder/alloc_emu.go index e5f2dd4..1eaf2c6 100644 --- a/internal/binary/decoder/alloc_emu.go +++ b/internal/binary/decoder/alloc_emu.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,24 +17,24 @@ package decoder import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" ) func emu_gcall_makemap(ctx hir.CallContext) { - if !ctx.Verify("*i*", "*") { - panic("invalid makemap call") - } else { - ctx.Rp(0, unsafe.Pointer(makemap((*rt.GoMapType)(ctx.Ap(0)), int(ctx.Au(1)), (*rt.GoMap)(ctx.Ap(2))))) - } + if !ctx.Verify("*i*", "*") { + panic("invalid makemap call") + } else { + ctx.Rp(0, unsafe.Pointer(makemap((*rt.GoMapType)(ctx.Ap(0)), int(ctx.Au(1)), (*rt.GoMap)(ctx.Ap(2))))) + } } func emu_gcall_mallocgc(ctx hir.CallContext) { - if !ctx.Verify("i*i", "*") { - panic("invalid mallocgc call") - } else { - ctx.Rp(0, mallocgc(uintptr(ctx.Au(0)), (*rt.GoType)(ctx.Ap(1)), ctx.Au(2) != 0)) - } + if !ctx.Verify("i*i", "*") { + panic("invalid mallocgc call") + } else { + ctx.Rp(0, mallocgc(uintptr(ctx.Au(0)), (*rt.GoType)(ctx.Ap(1)), ctx.Au(2) != 0)) + } } diff --git a/internal/binary/decoder/asm.s b/internal/binary/decoder/asm.s index a8c6cc1..9c99547 100644 --- a/internal/binary/decoder/asm.s +++ b/internal/binary/decoder/asm.s @@ -1,4 +1,4 @@ -// Copyright 2022 ByteDance Inc. +// Copyright 2022 CloudWeGo Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/internal/binary/decoder/bitmap.go b/internal/binary/decoder/bitmap.go index 52e424b..890ea04 100644 --- a/internal/binary/decoder/bitmap.go +++ b/internal/binary/decoder/bitmap.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,49 +17,49 @@ package decoder import ( - `sync` + "sync" - `github.com/cloudwego/frugal/internal/atm/hir` + "github.com/cloudwego/frugal/internal/atm/hir" ) const ( - MaxField = 65536 - MaxBitmap = MaxField / 64 + MaxField = 65536 + MaxBitmap = MaxField / 64 ) var ( - bitmapPool sync.Pool + bitmapPool sync.Pool ) var ( - F_newFieldBitmap = hir.RegisterGCall(newFieldBitmap, emu_gcall_newFieldBitmap) - F_FieldBitmap_Free = hir.RegisterGCall((*FieldBitmap).Free, emu_gcall_FieldBitmap_Free) + F_newFieldBitmap = hir.RegisterGCall(newFieldBitmap, emu_gcall_newFieldBitmap) + F_FieldBitmap_Free = hir.RegisterGCall((*FieldBitmap).Free, emu_gcall_FieldBitmap_Free) ) type ( - FieldBitmap [MaxBitmap]int64 + FieldBitmap [MaxBitmap]int64 ) func newFieldBitmap() *FieldBitmap { - if v := bitmapPool.Get(); v != nil { - return v.(*FieldBitmap) - } else { - return new(FieldBitmap) - } + if v := bitmapPool.Get(); v != nil { + return v.(*FieldBitmap) + } else { + return new(FieldBitmap) + } } func (self *FieldBitmap) Free() { - bitmapPool.Put(self) + bitmapPool.Put(self) } func (self *FieldBitmap) Clear() { - *self = FieldBitmap{} + *self = FieldBitmap{} } func (self *FieldBitmap) Append(i int) { - if i < MaxField { - self[i / 64] |= 1 << (i % 64) - } else { - panic("field index too large") - } + if i < MaxField { + self[i/64] |= 1 << (i % 64) + } else { + panic("field index too large") + } } diff --git a/internal/binary/decoder/bitmap_emu.go b/internal/binary/decoder/bitmap_emu.go index 73d9e7f..e64fba2 100644 --- a/internal/binary/decoder/bitmap_emu.go +++ b/internal/binary/decoder/bitmap_emu.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,23 +17,23 @@ package decoder import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` + "github.com/cloudwego/frugal/internal/atm/hir" ) func emu_gcall_newFieldBitmap(ctx hir.CallContext) { - if !ctx.Verify("", "*") { - panic("invalid newFieldBitmap call") - } else { - ctx.Rp(0, unsafe.Pointer(newFieldBitmap())) - } + if !ctx.Verify("", "*") { + panic("invalid newFieldBitmap call") + } else { + ctx.Rp(0, unsafe.Pointer(newFieldBitmap())) + } } func emu_gcall_FieldBitmap_Free(ctx hir.CallContext) { - if !ctx.Verify("*", "") { - panic("invalid (*FieldBitmap).Free call") - } else { - (*FieldBitmap)(ctx.Ap(0)).Free() - } + if !ctx.Verify("*", "") { + panic("invalid (*FieldBitmap).Free call") + } else { + (*FieldBitmap)(ctx.Ap(0)).Free() + } } diff --git a/internal/binary/decoder/compiler.go b/internal/binary/decoder/compiler.go index 8724a6e..ddbae44 100644 --- a/internal/binary/decoder/compiler.go +++ b/internal/binary/decoder/compiler.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,504 +17,582 @@ package decoder import ( - `fmt` - `reflect` - `sort` - `strconv` - `strings` - `unsafe` - - `github.com/cloudwego/frugal/internal/binary/defs` - `github.com/cloudwego/frugal/internal/opts` - `github.com/cloudwego/frugal/internal/rt` - `github.com/cloudwego/frugal/internal/utils` + "fmt" + "reflect" + "sort" + "strconv" + "strings" + "unsafe" + + "github.com/cloudwego/frugal/internal/binary/defs" + "github.com/cloudwego/frugal/internal/opts" + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/frugal/internal/utils" ) type Instr struct { - Op OpCode - Tx defs.Tag - Id uint16 - To int - Iv int64 - Sw *int - Vt *rt.GoType - Fn unsafe.Pointer + Op OpCode + Tx defs.Tag + Id uint16 + To int + Iv int64 + Sw *int + Vt *rt.GoType + Fn unsafe.Pointer } func (self Instr) stab() string { - t := self.IntSeq() - s := make([]string, 0, self.Iv) - - /* convert to strings */ - for i, v := range t { - if v >= 0 { - s = append(s, fmt.Sprintf("%4ccase %d: L_%d\n", ' ', i, v)) - } - } - - /* join them together */ - return fmt.Sprintf( - "{\n%s}", - strings.Join(s, ""), - ) + t := self.IntSeq() + s := make([]string, 0, self.Iv) + + /* convert to strings */ + for i, v := range t { + if v >= 0 { + s = append(s, fmt.Sprintf("%4ccase %d: L_%d\n", ' ', i, v)) + } + } + + /* join them together */ + return fmt.Sprintf( + "{\n%s}", + strings.Join(s, ""), + ) } func (self Instr) rtab() string { - t := self.IntSeq() - s := make([]string, 0, self.Iv) + t := self.IntSeq() + s := make([]string, 0, self.Iv) - /* convert to strings */ - for _, v := range t { - s = append(s, strconv.Itoa(v)) - } + /* convert to strings */ + for _, v := range t { + s = append(s, strconv.Itoa(v)) + } - /* join with ',' */ - return strings.Join(s, ", ") + /* join with ',' */ + return strings.Join(s, ", ") } func (self Instr) IntSeq() (p []int) { - (*rt.GoSlice)(unsafe.Pointer(&p)).Cap = int(self.Iv) - (*rt.GoSlice)(unsafe.Pointer(&p)).Len = int(self.Iv) - (*rt.GoSlice)(unsafe.Pointer(&p)).Ptr = unsafe.Pointer(self.Sw) - return + (*rt.GoSlice)(unsafe.Pointer(&p)).Cap = int(self.Iv) + (*rt.GoSlice)(unsafe.Pointer(&p)).Len = int(self.Iv) + (*rt.GoSlice)(unsafe.Pointer(&p)).Ptr = unsafe.Pointer(self.Sw) + return } func (self Instr) Disassemble() string { - switch self.Op { - case OP_int : fallthrough - case OP_size : fallthrough - case OP_seek : fallthrough - case OP_struct_mark_tag : return fmt.Sprintf("%-18s%d", self.Op, self.Iv) - case OP_type : return fmt.Sprintf("%-18s%d", self.Op, self.Tx) - case OP_deref : fallthrough - case OP_map_alloc : fallthrough - case OP_map_set_i8 : fallthrough - case OP_map_set_i16 : fallthrough - case OP_map_set_i32 : fallthrough - case OP_map_set_i64 : fallthrough - case OP_map_set_str : fallthrough - case OP_map_set_enum : fallthrough - case OP_map_set_pointer : fallthrough - case OP_list_alloc : fallthrough - case OP_construct : fallthrough - case OP_defer : return fmt.Sprintf("%-18s%s", self.Op, self.Vt) - case OP_ctr_is_zero : fallthrough - case OP_struct_is_stop : fallthrough - case OP_goto : return fmt.Sprintf("%-18sL_%d", self.Op, self.To) - case OP_struct_bitmap : fallthrough - case OP_struct_require : return fmt.Sprintf("%-18s%s", self.Op, self.rtab()) - case OP_struct_switch : return fmt.Sprintf("%-18s%s", self.Op, self.stab()) - case OP_struct_check_type : return fmt.Sprintf("%-18s%d, L_%d", self.Op, self.Tx, self.To) - case OP_initialize : return fmt.Sprintf("%-18s*%p [%s]", self.Op, self.Fn, rt.FuncName(self.Fn)) - default : return self.Op.String() - } + switch self.Op { + case OP_int: + fallthrough + case OP_size: + fallthrough + case OP_seek: + fallthrough + case OP_struct_mark_tag: + return fmt.Sprintf("%-18s%d", self.Op, self.Iv) + case OP_type: + return fmt.Sprintf("%-18s%d", self.Op, self.Tx) + case OP_deref: + fallthrough + case OP_map_alloc: + fallthrough + case OP_map_set_i8: + fallthrough + case OP_map_set_i16: + fallthrough + case OP_map_set_i32: + fallthrough + case OP_map_set_i64: + fallthrough + case OP_map_set_str: + fallthrough + case OP_map_set_enum: + fallthrough + case OP_map_set_pointer: + fallthrough + case OP_list_alloc: + fallthrough + case OP_construct: + fallthrough + case OP_defer: + return fmt.Sprintf("%-18s%s", self.Op, self.Vt) + case OP_ctr_is_zero: + fallthrough + case OP_struct_is_stop: + fallthrough + case OP_goto: + return fmt.Sprintf("%-18sL_%d", self.Op, self.To) + case OP_struct_bitmap: + fallthrough + case OP_struct_require: + return fmt.Sprintf("%-18s%s", self.Op, self.rtab()) + case OP_struct_switch: + return fmt.Sprintf("%-18s%s", self.Op, self.stab()) + case OP_struct_check_type: + return fmt.Sprintf("%-18s%d, L_%d", self.Op, self.Tx, self.To) + case OP_initialize: + return fmt.Sprintf("%-18s*%p [%s]", self.Op, self.Fn, rt.FuncName(self.Fn)) + default: + return self.Op.String() + } } func mkins(op OpCode, dt defs.Tag, id uint16, to int, iv int64, sw []int, vt reflect.Type, fn unsafe.Pointer) Instr { - return Instr { - Op: op, - Tx: dt, - Id: id, - To: to, - Fn: fn, - Vt: rt.UnpackType(vt), - Iv: int64(len(sw)) | iv, - Sw: (*int)((*rt.GoSlice)(unsafe.Pointer(&sw)).Ptr), - } + return Instr{ + Op: op, + Tx: dt, + Id: id, + To: to, + Fn: fn, + Vt: rt.UnpackType(vt), + Iv: int64(len(sw)) | iv, + Sw: (*int)((*rt.GoSlice)(unsafe.Pointer(&sw)).Ptr), + } } type ( - Program []Instr + Program []Instr ) func (self Program) pc() int { - return len(self) + return len(self) } func (self Program) pin(i int) { - self[i].To = self.pc() + self[i].To = self.pc() } func (self Program) use(n int) { - if n >= defs.StackSize { - panic("type nesting too deep") - } + if n >= defs.StackSize { + panic("type nesting too deep") + } } -func (self *Program) ins(iv Instr) { *self = append(*self, iv) } -func (self *Program) add(op OpCode) { self.ins(mkins(op, 0, 0, 0, 0, nil, nil, nil)) } -func (self *Program) jmp(op OpCode, to int) { self.ins(mkins(op, 0, 0, to, 0, nil, nil, nil)) } -func (self *Program) i64(op OpCode, iv int64) { self.ins(mkins(op, 0, 0, 0, iv, nil, nil, nil)) } -func (self *Program) tab(op OpCode, tv []int) { self.ins(mkins(op, 0, 0, 0, 0, tv, nil, nil)) } -func (self *Program) tag(op OpCode, vt defs.Tag) { self.ins(mkins(op, vt, 0, 0, 0, nil, nil, nil)) } -func (self *Program) rtt(op OpCode, vt reflect.Type) { self.ins(mkins(op, 0, 0, 0, 0, nil, vt, nil)) } -func (self *Program) jsr(op OpCode, fn unsafe.Pointer) { self.ins(mkins(op, 0, 0, 0, 0, nil, nil, fn)) } -func (self *Program) jcc(op OpCode, vt defs.Tag, to int) { self.ins(mkins(op, vt, 0, to, 0, nil, nil, nil)) } -func (self *Program) req(op OpCode, vt reflect.Type, fv []int) { self.ins(mkins(op, 0, 0, 0, 0, fv, vt, nil)) } +func (self *Program) ins(iv Instr) { *self = append(*self, iv) } +func (self *Program) add(op OpCode) { self.ins(mkins(op, 0, 0, 0, 0, nil, nil, nil)) } +func (self *Program) jmp(op OpCode, to int) { self.ins(mkins(op, 0, 0, to, 0, nil, nil, nil)) } +func (self *Program) i64(op OpCode, iv int64) { self.ins(mkins(op, 0, 0, 0, iv, nil, nil, nil)) } +func (self *Program) tab(op OpCode, tv []int) { self.ins(mkins(op, 0, 0, 0, 0, tv, nil, nil)) } +func (self *Program) tag(op OpCode, vt defs.Tag) { self.ins(mkins(op, vt, 0, 0, 0, nil, nil, nil)) } +func (self *Program) rtt(op OpCode, vt reflect.Type) { self.ins(mkins(op, 0, 0, 0, 0, nil, vt, nil)) } +func (self *Program) jsr(op OpCode, fn unsafe.Pointer) { self.ins(mkins(op, 0, 0, 0, 0, nil, nil, fn)) } +func (self *Program) jcc(op OpCode, vt defs.Tag, to int) { + self.ins(mkins(op, vt, 0, to, 0, nil, nil, nil)) +} +func (self *Program) req(op OpCode, vt reflect.Type, fv []int) { + self.ins(mkins(op, 0, 0, 0, 0, fv, vt, nil)) +} func (self Program) Free() { - freeProgram(self) + freeProgram(self) } func (self Program) Disassemble() string { - nb := len(self) - tab := make([]bool, nb + 1) - ret := make([]string, 0, nb + 1) - - /* prescan to get all the labels */ - for _, ins := range self { - if _OpBranches[ins.Op] { - if ins.Op != OP_struct_switch { - tab[ins.To] = true - } else { - for _, v := range ins.IntSeq() { - if v >= 0 { - tab[v] = true - } - } - } - } - } - - /* disassemble each instruction */ - for i, ins := range self { - ln := "" - ds := ins.Disassemble() - - /* check for label reference */ - if tab[i] { - ret = append(ret, fmt.Sprintf("L_%d:", i)) - } - - /* indent each line */ - for _, ln = range strings.Split(ds, "\n") { - ret = append(ret, " " + ln) - } - } - - /* add the last label, if needed */ - if tab[nb] { - ret = append(ret, fmt.Sprintf("L_%d:", nb)) - } - - /* add an "end" indicator, and join all the strings */ - return strings.Join(append(ret, " end"), "\n") + nb := len(self) + tab := make([]bool, nb+1) + ret := make([]string, 0, nb+1) + + /* prescan to get all the labels */ + for _, ins := range self { + if _OpBranches[ins.Op] { + if ins.Op != OP_struct_switch { + tab[ins.To] = true + } else { + for _, v := range ins.IntSeq() { + if v >= 0 { + tab[v] = true + } + } + } + } + } + + /* disassemble each instruction */ + for i, ins := range self { + ln := "" + ds := ins.Disassemble() + + /* check for label reference */ + if tab[i] { + ret = append(ret, fmt.Sprintf("L_%d:", i)) + } + + /* indent each line */ + for _, ln = range strings.Split(ds, "\n") { + ret = append(ret, " "+ln) + } + } + + /* add the last label, if needed */ + if tab[nb] { + ret = append(ret, fmt.Sprintf("L_%d:", nb)) + } + + /* add an "end" indicator, and join all the strings */ + return strings.Join(append(ret, " end"), "\n") } type Compiler struct { - o opts.Options - t map[reflect.Type]bool - d map[reflect.Type]struct{} + o opts.Options + t map[reflect.Type]bool + d map[reflect.Type]struct{} } func CreateCompiler() *Compiler { - return newCompiler() + return newCompiler() } func (self *Compiler) rescue(ep *error) { - if val := recover(); val != nil { - if err, ok := val.(error); ok { - *ep = err - } else { - panic(val) - } - } + if val := recover(); val != nil { + if err, ok := val.(error); ok { + *ep = err + } else { + panic(val) + } + } } func (self *Compiler) compileDef(p *Program, vt *defs.Type) { - p.rtt(OP_defer, vt.S) - self.d[vt.S] = struct{}{} + p.rtt(OP_defer, vt.S) + self.d[vt.S] = struct{}{} } func (self *Compiler) compileOne(p *Program, sp int, vt *defs.Type) { - if vt.T == defs.T_pointer { - self.compilePtr(p, sp, vt) - } else if vt.T != defs.T_struct { - self.compileRec(p, sp, vt) - } else if _, ok := self.t[vt.S]; !ok && self.o.CanInline(sp, p.pc()) { - self.compileTag(p, sp, vt) - } else { - self.compileDef(p, vt) - } + if vt.T == defs.T_pointer { + self.compilePtr(p, sp, vt) + } else if vt.T != defs.T_struct { + self.compileRec(p, sp, vt) + } else if _, ok := self.t[vt.S]; !ok && self.o.CanInline(sp, p.pc()) { + self.compileTag(p, sp, vt) + } else { + self.compileDef(p, vt) + } } func (self *Compiler) compileTag(p *Program, sp int, vt *defs.Type) { - self.t[vt.S] = true - self.compileRec(p, sp, vt) - delete(self.t, vt.S) + self.t[vt.S] = true + self.compileRec(p, sp, vt) + delete(self.t, vt.S) } func (self *Compiler) compileRec(p *Program, sp int, vt *defs.Type) { - switch vt.T { - case defs.T_bool : p.i64(OP_size, 1); p.i64(OP_int, 1) - case defs.T_i8 : p.i64(OP_size, 1); p.i64(OP_int, 1) - case defs.T_i16 : p.i64(OP_size, 2); p.i64(OP_int, 2) - case defs.T_i32 : p.i64(OP_size, 4); p.i64(OP_int, 4) - case defs.T_i64 : p.i64(OP_size, 8); p.i64(OP_int, 8) - case defs.T_double : p.i64(OP_size, 8); p.i64(OP_int, 8) - case defs.T_string : p.i64(OP_size, 4); p.add(OP_str) - case defs.T_binary : p.i64(OP_size, 4); p.add(OP_bin) - case defs.T_enum : p.i64(OP_size, 4); p.add(OP_enum) - case defs.T_struct : self.compileStruct (p, sp, vt) - case defs.T_map : self.compileMap (p, sp, vt) - case defs.T_set : self.compileSetList (p, sp, vt.V) - case defs.T_list : self.compileSetList (p, sp, vt.V) - default : panic("unreachable") - } + switch vt.T { + case defs.T_bool: + p.i64(OP_size, 1) + p.i64(OP_int, 1) + case defs.T_i8: + p.i64(OP_size, 1) + p.i64(OP_int, 1) + case defs.T_i16: + p.i64(OP_size, 2) + p.i64(OP_int, 2) + case defs.T_i32: + p.i64(OP_size, 4) + p.i64(OP_int, 4) + case defs.T_i64: + p.i64(OP_size, 8) + p.i64(OP_int, 8) + case defs.T_double: + p.i64(OP_size, 8) + p.i64(OP_int, 8) + case defs.T_string: + p.i64(OP_size, 4) + p.add(OP_str) + case defs.T_binary: + p.i64(OP_size, 4) + p.add(OP_bin) + case defs.T_enum: + p.i64(OP_size, 4) + p.add(OP_enum) + case defs.T_struct: + self.compileStruct(p, sp, vt) + case defs.T_map: + self.compileMap(p, sp, vt) + case defs.T_set: + self.compileSetList(p, sp, vt.V) + case defs.T_list: + self.compileSetList(p, sp, vt.V) + default: + panic("unreachable") + } } func (self *Compiler) compilePtr(p *Program, sp int, vt *defs.Type) { - p.use(sp) - p.add(OP_make_state) - p.rtt(OP_deref, vt.V.S) - self.compileOne(p, sp + 1, vt.V) - p.add(OP_drop_state) + p.use(sp) + p.add(OP_make_state) + p.rtt(OP_deref, vt.V.S) + self.compileOne(p, sp+1, vt.V) + p.add(OP_drop_state) } func (self *Compiler) compileMap(p *Program, sp int, vt *defs.Type) { - p.use(sp) - p.i64(OP_size, 6) - p.tag(OP_type, vt.K.Tag()) - p.tag(OP_type, vt.V.Tag()) - p.add(OP_make_state) - p.add(OP_ctr_load) - p.rtt(OP_map_alloc, vt.S) - i := p.pc() - p.add(OP_ctr_is_zero) - self.compileKey(p, sp + 1, vt) - self.compileOne(p, sp + 1, vt.V) - p.add(OP_ctr_decr) - p.jmp(OP_goto, i) - p.pin(i) - p.add(OP_map_close) - p.add(OP_drop_state) + p.use(sp) + p.i64(OP_size, 6) + p.tag(OP_type, vt.K.Tag()) + p.tag(OP_type, vt.V.Tag()) + p.add(OP_make_state) + p.add(OP_ctr_load) + p.rtt(OP_map_alloc, vt.S) + i := p.pc() + p.add(OP_ctr_is_zero) + self.compileKey(p, sp+1, vt) + self.compileOne(p, sp+1, vt.V) + p.add(OP_ctr_decr) + p.jmp(OP_goto, i) + p.pin(i) + p.add(OP_map_close) + p.add(OP_drop_state) } func (self *Compiler) compileKey(p *Program, sp int, vt *defs.Type) { - switch vt.K.T { - case defs.T_bool : p.i64(OP_size, 1); p.rtt(OP_map_set_i8, vt.S) - case defs.T_i8 : p.i64(OP_size, 1); p.rtt(OP_map_set_i8, vt.S) - case defs.T_double : p.i64(OP_size, 8); p.rtt(OP_map_set_i64, vt.S) - case defs.T_i16 : p.i64(OP_size, 2); p.rtt(OP_map_set_i16, vt.S) - case defs.T_i32 : p.i64(OP_size, 4); p.rtt(OP_map_set_i32, vt.S) - case defs.T_i64 : p.i64(OP_size, 8); p.rtt(OP_map_set_i64, vt.S) - case defs.T_binary : p.i64(OP_size, 4); p.rtt(OP_map_set_str, vt.S) - case defs.T_string : p.i64(OP_size, 4); p.rtt(OP_map_set_str, vt.S) - case defs.T_enum : p.i64(OP_size, 4); p.rtt(OP_map_set_enum, vt.S) - case defs.T_pointer : self.compileKeyPtr(p, sp, vt) - default : panic("unreachable") - } + switch vt.K.T { + case defs.T_bool: + p.i64(OP_size, 1) + p.rtt(OP_map_set_i8, vt.S) + case defs.T_i8: + p.i64(OP_size, 1) + p.rtt(OP_map_set_i8, vt.S) + case defs.T_double: + p.i64(OP_size, 8) + p.rtt(OP_map_set_i64, vt.S) + case defs.T_i16: + p.i64(OP_size, 2) + p.rtt(OP_map_set_i16, vt.S) + case defs.T_i32: + p.i64(OP_size, 4) + p.rtt(OP_map_set_i32, vt.S) + case defs.T_i64: + p.i64(OP_size, 8) + p.rtt(OP_map_set_i64, vt.S) + case defs.T_binary: + p.i64(OP_size, 4) + p.rtt(OP_map_set_str, vt.S) + case defs.T_string: + p.i64(OP_size, 4) + p.rtt(OP_map_set_str, vt.S) + case defs.T_enum: + p.i64(OP_size, 4) + p.rtt(OP_map_set_enum, vt.S) + case defs.T_pointer: + self.compileKeyPtr(p, sp, vt) + default: + panic("unreachable") + } } func (self *Compiler) compileNoCopy(p *Program, sp int, vt *defs.Type) { - switch { - default: { - panic("invalid nocopy type: " + vt.String()) - } - - /* simple strings */ - case vt.T == defs.T_string: { - p.i64(OP_size, 4) - p.add(OP_str_nocopy) - } - - /* simple binaries */ - case vt.T == defs.T_binary: { - p.i64(OP_size, 4) - p.add(OP_bin_nocopy) - } - - /* string pointers */ - case vt.T == defs.T_pointer && vt.V.T == defs.T_string: { - p.use(sp) - p.add(OP_make_state) - p.rtt(OP_deref, vt.V.S) - p.i64(OP_size, 4) - p.add(OP_str_nocopy) - p.add(OP_drop_state) - } - - /* binary pointers */ - case vt.T == defs.T_pointer && vt.V.T == defs.T_binary: { - p.use(sp) - p.add(OP_make_state) - p.rtt(OP_deref, vt.V.S) - p.i64(OP_size, 4) - p.add(OP_bin_nocopy) - p.add(OP_drop_state) - } - } + switch { + default: + { + panic("invalid nocopy type: " + vt.String()) + } + + /* simple strings */ + case vt.T == defs.T_string: + { + p.i64(OP_size, 4) + p.add(OP_str_nocopy) + } + + /* simple binaries */ + case vt.T == defs.T_binary: + { + p.i64(OP_size, 4) + p.add(OP_bin_nocopy) + } + + /* string pointers */ + case vt.T == defs.T_pointer && vt.V.T == defs.T_string: + { + p.use(sp) + p.add(OP_make_state) + p.rtt(OP_deref, vt.V.S) + p.i64(OP_size, 4) + p.add(OP_str_nocopy) + p.add(OP_drop_state) + } + + /* binary pointers */ + case vt.T == defs.T_pointer && vt.V.T == defs.T_binary: + { + p.use(sp) + p.add(OP_make_state) + p.rtt(OP_deref, vt.V.S) + p.i64(OP_size, 4) + p.add(OP_bin_nocopy) + p.add(OP_drop_state) + } + } } func (self *Compiler) compileKeyPtr(p *Program, sp int, vt *defs.Type) { - pt := vt.K - st := pt.V - - /* must be a struct */ - if st.T != defs.T_struct { - panic("map key cannot be non-struct pointers") - } - - /* construct a new object */ - p.rtt(OP_construct, st.S) - self.compileOne(p, sp, st) - p.rtt(OP_map_set_pointer, vt.S) + pt := vt.K + st := pt.V + + /* must be a struct */ + if st.T != defs.T_struct { + panic("map key cannot be non-struct pointers") + } + + /* construct a new object */ + p.rtt(OP_construct, st.S) + self.compileOne(p, sp, st) + p.rtt(OP_map_set_pointer, vt.S) } func (self *Compiler) compileStruct(p *Program, sp int, vt *defs.Type) { - var fid int - var err error - var req []int - var fvs []defs.Field - var ifn unsafe.Pointer - - /* resolve the fields */ - if fvs, err = defs.ResolveFields(vt.S); err != nil { - panic(err) - } - - /* empty struct */ - if len(fvs) == 0 { - p.add(OP_struct_ignore) - return - } - - /* find the default initializer */ - if ifn, err = defs.GetDefaultInitializer(vt.S); err != nil { - panic(err) - } - - /* call the initializer if any */ - if ifn != nil { - p.jsr(OP_initialize, ifn) - } - - /* find the maximum field IDs */ - for _, fv := range fvs { - if fid = utils.MaxInt(fid, int(fv.ID)); fv.Spec == defs.Required { - req = append(req, int(fv.ID)) - } - } - - /* save the current state */ - p.use(sp) - p.add(OP_make_state) - - /* allocate bitmap for required fields, if needed */ - if sort.Ints(req); len(req) != 0 { - p.tab(OP_struct_bitmap, req) - } - - /* switch jump buffer */ - i := p.pc() - s := make([]int, fid + 1) - - /* set the default branch */ - for v := range s { - s[v] = -1 - } - - /* dispatch the next field */ - p.i64(OP_size, 1) - p.add(OP_struct_read_type) - j := p.pc() - p.add(OP_struct_is_stop) - p.i64(OP_size, 2) - p.tab(OP_struct_switch, s) - k := p.pc() - p.add(OP_struct_skip) - p.jmp(OP_goto, i) - - /* assemble every field */ - for _, fv := range fvs { - s[fv.ID] = p.pc() - p.jcc(OP_struct_check_type, fv.Type.Tag(), k) - - /* mark the field as seen, if needed */ - if fv.Spec == defs.Required { - p.i64(OP_struct_mark_tag, int64(fv.ID)) - } - - /* seek to the field */ - off := int64(fv.F) - p.i64(OP_seek, off) - - /* check for no-copy strings */ - if fv.Opts & defs.NoCopy == 0 { - self.compileOne(p, sp + 1, fv.Type) - } else if fv.Type.Tag() == defs.T_string { - self.compileNoCopy(p, sp + 1, fv.Type) - } else { - panic(`"nocopy" is only applicable to "string" or "binary" types`) - } - - /* seek back to the beginning */ - p.i64(OP_seek, -off) - p.jmp(OP_goto, i) - } - - /* no required fields */ - if p.pin(j); len(req) == 0 { - p.add(OP_drop_state) - return - } - - /* check all the required fields */ - p.req(OP_struct_require, vt.S, req) - p.add(OP_drop_state) + var fid int + var err error + var req []int + var fvs []defs.Field + var ifn unsafe.Pointer + + /* resolve the fields */ + if fvs, err = defs.ResolveFields(vt.S); err != nil { + panic(err) + } + + /* empty struct */ + if len(fvs) == 0 { + p.add(OP_struct_ignore) + return + } + + /* find the default initializer */ + if ifn, err = defs.GetDefaultInitializer(vt.S); err != nil { + panic(err) + } + + /* call the initializer if any */ + if ifn != nil { + p.jsr(OP_initialize, ifn) + } + + /* find the maximum field IDs */ + for _, fv := range fvs { + if fid = utils.MaxInt(fid, int(fv.ID)); fv.Spec == defs.Required { + req = append(req, int(fv.ID)) + } + } + + /* save the current state */ + p.use(sp) + p.add(OP_make_state) + + /* allocate bitmap for required fields, if needed */ + if sort.Ints(req); len(req) != 0 { + p.tab(OP_struct_bitmap, req) + } + + /* switch jump buffer */ + i := p.pc() + s := make([]int, fid+1) + + /* set the default branch */ + for v := range s { + s[v] = -1 + } + + /* dispatch the next field */ + p.i64(OP_size, 1) + p.add(OP_struct_read_type) + j := p.pc() + p.add(OP_struct_is_stop) + p.i64(OP_size, 2) + p.tab(OP_struct_switch, s) + k := p.pc() + p.add(OP_struct_skip) + p.jmp(OP_goto, i) + + /* assemble every field */ + for _, fv := range fvs { + s[fv.ID] = p.pc() + p.jcc(OP_struct_check_type, fv.Type.Tag(), k) + + /* mark the field as seen, if needed */ + if fv.Spec == defs.Required { + p.i64(OP_struct_mark_tag, int64(fv.ID)) + } + + /* seek to the field */ + off := int64(fv.F) + p.i64(OP_seek, off) + + /* check for no-copy strings */ + if fv.Opts&defs.NoCopy == 0 { + self.compileOne(p, sp+1, fv.Type) + } else if fv.Type.Tag() == defs.T_string { + self.compileNoCopy(p, sp+1, fv.Type) + } else { + panic(`"nocopy" is only applicable to "string" or "binary" types`) + } + + /* seek back to the beginning */ + p.i64(OP_seek, -off) + p.jmp(OP_goto, i) + } + + /* no required fields */ + if p.pin(j); len(req) == 0 { + p.add(OP_drop_state) + return + } + + /* check all the required fields */ + p.req(OP_struct_require, vt.S, req) + p.add(OP_drop_state) } func (self *Compiler) compileSetList(p *Program, sp int, et *defs.Type) { - p.use(sp) - p.i64(OP_size, 5) - p.tag(OP_type, et.Tag()) - p.add(OP_make_state) - p.add(OP_ctr_load) - p.rtt(OP_list_alloc, et.S) - i := p.pc() - p.add(OP_ctr_is_zero) - j := p.pc() - self.compileOne(p, sp + 1, et) - p.add(OP_ctr_decr) - k := p.pc() - p.add(OP_ctr_is_zero) - p.i64(OP_seek, int64(et.S.Size())) - p.jmp(OP_goto, j) - p.pin(i) - p.pin(k) - p.add(OP_drop_state) + p.use(sp) + p.i64(OP_size, 5) + p.tag(OP_type, et.Tag()) + p.add(OP_make_state) + p.add(OP_ctr_load) + p.rtt(OP_list_alloc, et.S) + i := p.pc() + p.add(OP_ctr_is_zero) + j := p.pc() + self.compileOne(p, sp+1, et) + p.add(OP_ctr_decr) + k := p.pc() + p.add(OP_ctr_is_zero) + p.i64(OP_seek, int64(et.S.Size())) + p.jmp(OP_goto, j) + p.pin(i) + p.pin(k) + p.add(OP_drop_state) } func (self *Compiler) Free() { - freeCompiler(self) + freeCompiler(self) } func (self *Compiler) Apply(o opts.Options) *Compiler { - self.o = o - return self + self.o = o + return self } func (self *Compiler) Compile(vt reflect.Type) (_ Program, err error) { - ret := newProgram() - vtp := (*defs.Type)(nil) - - /* parse the type */ - if vtp, err = defs.ParseType(vt, ""); err != nil { - return nil, err - } - - /* catch the exceptions, and free the type */ - defer self.rescue(&err) - defer vtp.Free() - - /* compile the actual type */ - self.compileOne(&ret, 0, vtp) - ret.add(OP_halt) - return Optimize(ret), nil + ret := newProgram() + vtp := (*defs.Type)(nil) + + /* parse the type */ + if vtp, err = defs.ParseType(vt, ""); err != nil { + return nil, err + } + + /* catch the exceptions, and free the type */ + defer self.rescue(&err) + defer vtp.Free() + + /* compile the actual type */ + self.compileOne(&ret, 0, vtp) + ret.add(OP_halt) + return Optimize(ret), nil } func (self *Compiler) CompileAndFree(vt reflect.Type) (ret Program, err error) { - ret, err = self.Compile(vt) - self.Free() - return + ret, err = self.Compile(vt) + self.Free() + return } diff --git a/internal/binary/decoder/compiler_test.go b/internal/binary/decoder/compiler_test.go index 6f9ca16..6e8d7fd 100644 --- a/internal/binary/decoder/compiler_test.go +++ b/internal/binary/decoder/compiler_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,62 +17,62 @@ package decoder import ( - `reflect` - `testing` + "reflect" + "testing" - `github.com/stretchr/testify/require` + "github.com/stretchr/testify/require" ) type CompilerTest struct { - A bool `frugal:"0,default,bool"` - B int8 `frugal:"1,default,i8"` - C float64 `frugal:"2,default,double"` - D int16 `frugal:"3,default,i16"` - E int32 `frugal:"4,default,i32"` - F int64 `frugal:"5,default,i64"` - G string `frugal:"6,default,string"` - H *CompilerTest `frugal:"7,default,CompilerTest"` - I *CompilerTestSubStruct `frugal:"8,default,CompilerTestSubStruct"` - J map[string]int `frugal:"9,default,map"` - K []string `frugal:"10,default,set"` - L []string `frugal:"11,default,list"` - M []byte `frugal:"12,default,binary"` - N []int8 `frugal:"13,default,set"` - O []int8 `frugal:"14,default,list"` - P int64 `frugal:"16,required,i64"` + A bool `frugal:"0,default,bool"` + B int8 `frugal:"1,default,i8"` + C float64 `frugal:"2,default,double"` + D int16 `frugal:"3,default,i16"` + E int32 `frugal:"4,default,i32"` + F int64 `frugal:"5,default,i64"` + G string `frugal:"6,default,string"` + H *CompilerTest `frugal:"7,default,CompilerTest"` + I *CompilerTestSubStruct `frugal:"8,default,CompilerTestSubStruct"` + J map[string]int `frugal:"9,default,map"` + K []string `frugal:"10,default,set"` + L []string `frugal:"11,default,list"` + M []byte `frugal:"12,default,binary"` + N []int8 `frugal:"13,default,set"` + O []int8 `frugal:"14,default,list"` + P int64 `frugal:"16,required,i64"` } func (self *CompilerTest) InitDefault() { - *self = CompilerTest{} + *self = CompilerTest{} } type CompilerTestSubStruct struct { - X int `frugal:"0,default,i64"` - Y *CompilerTestSubStruct `frugal:"1,default,CompilerTestSubStruct"` - Z map[*CompilerTest]*CompilerTestSubStruct `frugal:"2,default,map"` + X int `frugal:"0,default,i64"` + Y *CompilerTestSubStruct `frugal:"1,default,CompilerTestSubStruct"` + Z map[*CompilerTest]*CompilerTestSubStruct `frugal:"2,default,map"` } func (self *CompilerTestSubStruct) InitDefault() { - *self = CompilerTestSubStruct{} + *self = CompilerTestSubStruct{} } func TestCompiler_Compile(t *testing.T) { - p, err := CreateCompiler().Compile(reflect.TypeOf(CompilerTest{})) - require.NoError(t, err) - println(p.Disassemble()) + p, err := CreateCompiler().Compile(reflect.TypeOf(CompilerTest{})) + require.NoError(t, err) + println(p.Disassemble()) } type NoCopyStringTestStruct struct { - A string `frugal:"1,default,string"` - B string `frugal:"2,default,string,nocopy"` - C *string `frugal:"3,optional,string,nocopy"` - D []byte `frugal:"4,default,binary"` - E []byte `frugal:"5,default,binary,nocopy"` - F *[]byte `frugal:"6,optional,binary,nocopy"` + A string `frugal:"1,default,string"` + B string `frugal:"2,default,string,nocopy"` + C *string `frugal:"3,optional,string,nocopy"` + D []byte `frugal:"4,default,binary"` + E []byte `frugal:"5,default,binary,nocopy"` + F *[]byte `frugal:"6,optional,binary,nocopy"` } func TestCompiler_NoCopyString(t *testing.T) { - p, err := CreateCompiler().Compile(reflect.TypeOf(NoCopyStringTestStruct{})) - require.NoError(t, err) - println(p.Disassemble()) + p, err := CreateCompiler().Compile(reflect.TypeOf(NoCopyStringTestStruct{})) + require.NoError(t, err) + println(p.Disassemble()) } diff --git a/internal/binary/decoder/decoder.go b/internal/binary/decoder/decoder.go index 82ee5e6..5f4cff3 100644 --- a/internal/binary/decoder/decoder.go +++ b/internal/binary/decoder/decoder.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,146 +17,146 @@ package decoder import ( - `reflect` - `sync/atomic` - `unsafe` + "reflect" + "sync/atomic" + "unsafe" - `github.com/cloudwego/frugal/internal/opts` - `github.com/cloudwego/frugal/internal/rt` - `github.com/cloudwego/frugal/internal/utils` + "github.com/cloudwego/frugal/internal/opts" + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/frugal/internal/utils" ) -type Decoder func ( - buf unsafe.Pointer, - nb int, - i int, - p unsafe.Pointer, - rs *RuntimeState, - st int, +type Decoder func( + buf unsafe.Pointer, + nb int, + i int, + p unsafe.Pointer, + rs *RuntimeState, + st int, ) (int, error) var ( - HitCount uint64 = 0 - MissCount uint64 = 0 - TypeCount uint64 = 0 + HitCount uint64 = 0 + MissCount uint64 = 0 + TypeCount uint64 = 0 ) var ( - programCache = utils.CreateProgramCache() + programCache = utils.CreateProgramCache() ) func decode(vt *rt.GoType, buf unsafe.Pointer, nb int, i int, p unsafe.Pointer, rs *RuntimeState, st int) (int, error) { - if dec, err := resolve(vt); err != nil { - return 0, err - } else { - return dec(buf, nb, i, p, rs, st) - } + if dec, err := resolve(vt); err != nil { + return 0, err + } else { + return dec(buf, nb, i, p, rs, st) + } } func resolve(vt *rt.GoType) (Decoder, error) { - var err error - var val interface{} - - /* fast-path: type is cached */ - if val = programCache.Get(vt); val != nil { - atomic.AddUint64(&HitCount, 1) - return val.(Decoder), nil - } - - /* record the cache miss, and compile the type */ - atomic.AddUint64(&MissCount, 1) - val, err = programCache.Compute(vt, compile) - - /* check for errors */ - if err != nil { - return nil, err - } - - /* record the successful compilation */ - atomic.AddUint64(&TypeCount, 1) - return val.(Decoder), nil + var err error + var val interface{} + + /* fast-path: type is cached */ + if val = programCache.Get(vt); val != nil { + atomic.AddUint64(&HitCount, 1) + return val.(Decoder), nil + } + + /* record the cache miss, and compile the type */ + atomic.AddUint64(&MissCount, 1) + val, err = programCache.Compute(vt, compile) + + /* check for errors */ + if err != nil { + return nil, err + } + + /* record the successful compilation */ + atomic.AddUint64(&TypeCount, 1) + return val.(Decoder), nil } func compile(vt *rt.GoType) (interface{}, error) { - if pp, err := CreateCompiler().CompileAndFree(vt.Pack()); err != nil { - return nil, err - } else { - return Link(Translate(pp)), nil - } + if pp, err := CreateCompiler().CompileAndFree(vt.Pack()); err != nil { + return nil, err + } else { + return Link(Translate(pp)), nil + } } func mkcompile(ty map[reflect.Type]struct{}, opts opts.Options) func(*rt.GoType) (interface{}, error) { - return func(vt *rt.GoType) (interface{}, error) { - cc := CreateCompiler() - pp, err := cc.Apply(opts).Compile(vt.Pack()) - - /* add all the deferred types */ - for t := range cc.d { - ty[t] = struct{}{} - } - - /* translate and link the program */ - if err != nil { - return nil, err - } else { - return Link(Translate(pp)), nil - } - } + return func(vt *rt.GoType) (interface{}, error) { + cc := CreateCompiler() + pp, err := cc.Apply(opts).Compile(vt.Pack()) + + /* add all the deferred types */ + for t := range cc.d { + ty[t] = struct{}{} + } + + /* translate and link the program */ + if err != nil { + return nil, err + } else { + return Link(Translate(pp)), nil + } + } } type DecodeError struct { - vt *rt.GoType + vt *rt.GoType } func (self DecodeError) Error() string { - if self.vt == nil { - return "frugal: unmarshal to nil interface" - } else if self.vt.Kind() == reflect.Ptr { - return "frugal: unmarshal to nil " + self.vt.String() - } else { - return "frugal: unmarshal to non-pointer " + self.vt.String() - } + if self.vt == nil { + return "frugal: unmarshal to nil interface" + } else if self.vt.Kind() == reflect.Ptr { + return "frugal: unmarshal to nil " + self.vt.String() + } else { + return "frugal: unmarshal to non-pointer " + self.vt.String() + } } func Pretouch(vt *rt.GoType, opts opts.Options) (map[reflect.Type]struct{}, error) { - var err error - var ret map[reflect.Type]struct{} - - /* check for cached types */ - if programCache.Get(vt) != nil { - return nil, nil - } - - /* compile & load the type */ - ret = make(map[reflect.Type]struct{}) - _, err = programCache.Compute(vt, mkcompile(ret, opts)) - - /* check for errors */ - if err != nil { - return nil, err - } - - /* add the type count */ - atomic.AddUint64(&TypeCount, 1) - return ret, nil + var err error + var ret map[reflect.Type]struct{} + + /* check for cached types */ + if programCache.Get(vt) != nil { + return nil, nil + } + + /* compile & load the type */ + ret = make(map[reflect.Type]struct{}) + _, err = programCache.Compute(vt, mkcompile(ret, opts)) + + /* check for errors */ + if err != nil { + return nil, err + } + + /* add the type count */ + atomic.AddUint64(&TypeCount, 1) + return ret, nil } func DecodeObject(buf []byte, val interface{}) (ret int, err error) { - vv := rt.UnpackEface(val) - vt := vv.Type - - /* check for nil interface */ - if vt == nil || vv.Value == nil || vt.Kind() != reflect.Ptr { - return 0, DecodeError { vt } - } - - /* create a new runtime state */ - et := rt.PtrElem(vt) - st := newRuntimeState() - sl := (*rt.GoSlice)(unsafe.Pointer(&buf)) - - /* call the encoder, and return the runtime state into pool */ - ret, err = decode(et, sl.Ptr, sl.Len, 0, vv.Value, st, 0) - freeRuntimeState(st) - return + vv := rt.UnpackEface(val) + vt := vv.Type + + /* check for nil interface */ + if vt == nil || vv.Value == nil || vt.Kind() != reflect.Ptr { + return 0, DecodeError{vt} + } + + /* create a new runtime state */ + et := rt.PtrElem(vt) + st := newRuntimeState() + sl := (*rt.GoSlice)(unsafe.Pointer(&buf)) + + /* call the encoder, and return the runtime state into pool */ + ret, err = decode(et, sl.Ptr, sl.Len, 0, vv.Value, st, 0) + freeRuntimeState(st) + return } diff --git a/internal/binary/decoder/decoder_test.go b/internal/binary/decoder/decoder_test.go index 05f8fc6..64abdfd 100644 --- a/internal/binary/decoder/decoder_test.go +++ b/internal/binary/decoder/decoder_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,154 +17,154 @@ package decoder import ( - `testing` - `unsafe` + "testing" + "unsafe" - `github.com/cloudwego/frugal/internal/rt` - `github.com/davecgh/go-spew/spew` - `github.com/stretchr/testify/require` + "github.com/cloudwego/frugal/internal/rt" + "github.com/davecgh/go-spew/spew" + "github.com/stretchr/testify/require" ) func TestDecoder_Decode(t *testing.T) { - var v TranslatorTestStruct - rs := new(RuntimeState) - buf := []byte { - 0x02, 0x00, 0x00, 0x01, 0x03, 0x00, 0x01, 0x12, 0x04, 0x00, 0x02, 0x40, 0x28, 0xae, 0x14, 0x7a, - 0xe1, 0x47, 0xae, 0x06, 0x00, 0x03, 0x34, 0x56, 0x08, 0x00, 0x04, 0x12, 0x34, 0x56, 0x78, 0x0a, - 0x00, 0x05, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0x0b, 0x00, 0x06, 0x00, 0x00, 0x00, - 0x0c, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x0b, 0x00, 0x07, - 0x00, 0x00, 0x00, 0x0e, 0x74, 0x65, 0x73, 0x74, 0x62, 0x79, 0x74, 0x65, 0x62, 0x75, 0x66, 0x66, - 0x65, 0x72, 0x0f, 0x00, 0x08, 0x08, 0x00, 0x00, 0x00, 0x05, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, - 0x77, 0x88, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x05, 0x0d, 0x00, - 0x09, 0x0b, 0x0b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x61, 0x73, 0x64, 0x66, 0x00, - 0x00, 0x00, 0x04, 0x71, 0x77, 0x65, 0x72, 0x00, 0x00, 0x00, 0x04, 0x7a, 0x78, 0x63, 0x76, 0x00, - 0x00, 0x00, 0x04, 0x68, 0x6a, 0x6b, 0x6c, 0x0d, 0x00, 0x41, 0x0b, 0x0c, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x03, 0x66, 0x6f, 0x6f, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x01, 0xff, 0x04, - 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x03, 0x00, 0x00, 0x08, - 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x0b, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x0f, - 0x00, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x09, 0x0b, 0x0b, 0x00, 0x00, 0x00, 0x00, - 0x0d, 0x00, 0x41, 0x0b, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - } - sl := (*rt.GoSlice)(unsafe.Pointer(&buf)) - pos, err := decode(rt.UnpackEface(v).Type, sl.Ptr, sl.Len, 0, unsafe.Pointer(&v), rs, 0) - require.NoError(t, err) - require.Equal(t, len(buf), pos) - require.Equal(t, TranslatorTestStruct { - A: true, - B: 0x12, - C: 12.34, - D: 0x3456, - E: 0x12345678, - F: 0x66778899aabbccdd, - G: "hello, world", - H: []byte("testbytebuffer"), - I: []int32{0x11223344, 0x55667788, 3, 4, 5}, - J: map[string]string{"asdf": "qwer", "zxcv": "hjkl"}, - K: map[string]*TranslatorTestStruct{ - "foo": { - B: -1, - H: []uint8{}, - I: []int32{}, - J: map[string]string{}, - K: map[string]*TranslatorTestStruct{}, - }, - }, - }, v) - spew.Dump(v) + var v TranslatorTestStruct + rs := new(RuntimeState) + buf := []byte{ + 0x02, 0x00, 0x00, 0x01, 0x03, 0x00, 0x01, 0x12, 0x04, 0x00, 0x02, 0x40, 0x28, 0xae, 0x14, 0x7a, + 0xe1, 0x47, 0xae, 0x06, 0x00, 0x03, 0x34, 0x56, 0x08, 0x00, 0x04, 0x12, 0x34, 0x56, 0x78, 0x0a, + 0x00, 0x05, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0x0b, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x0c, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x0b, 0x00, 0x07, + 0x00, 0x00, 0x00, 0x0e, 0x74, 0x65, 0x73, 0x74, 0x62, 0x79, 0x74, 0x65, 0x62, 0x75, 0x66, 0x66, + 0x65, 0x72, 0x0f, 0x00, 0x08, 0x08, 0x00, 0x00, 0x00, 0x05, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, + 0x77, 0x88, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x05, 0x0d, 0x00, + 0x09, 0x0b, 0x0b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x61, 0x73, 0x64, 0x66, 0x00, + 0x00, 0x00, 0x04, 0x71, 0x77, 0x65, 0x72, 0x00, 0x00, 0x00, 0x04, 0x7a, 0x78, 0x63, 0x76, 0x00, + 0x00, 0x00, 0x04, 0x68, 0x6a, 0x6b, 0x6c, 0x0d, 0x00, 0x41, 0x0b, 0x0c, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x03, 0x66, 0x6f, 0x6f, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x01, 0xff, 0x04, + 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x03, 0x00, 0x00, 0x08, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x0b, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x0f, + 0x00, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x09, 0x0b, 0x0b, 0x00, 0x00, 0x00, 0x00, + 0x0d, 0x00, 0x41, 0x0b, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + } + sl := (*rt.GoSlice)(unsafe.Pointer(&buf)) + pos, err := decode(rt.UnpackEface(v).Type, sl.Ptr, sl.Len, 0, unsafe.Pointer(&v), rs, 0) + require.NoError(t, err) + require.Equal(t, len(buf), pos) + require.Equal(t, TranslatorTestStruct{ + A: true, + B: 0x12, + C: 12.34, + D: 0x3456, + E: 0x12345678, + F: 0x66778899aabbccdd, + G: "hello, world", + H: []byte("testbytebuffer"), + I: []int32{0x11223344, 0x55667788, 3, 4, 5}, + J: map[string]string{"asdf": "qwer", "zxcv": "hjkl"}, + K: map[string]*TranslatorTestStruct{ + "foo": { + B: -1, + H: []uint8{}, + I: []int32{}, + J: map[string]string{}, + K: map[string]*TranslatorTestStruct{}, + }, + }, + }, v) + spew.Dump(v) } type TestSimple struct { - A bool `frugal:"0,default,bool"` + A bool `frugal:"0,default,bool"` } func TestDecoder_Simple(t *testing.T) { - var v TestSimple - rs := new(RuntimeState) - buf := []byte { - 0x02, 0x00, 0x00, 0x01, 0x03, 0x00, 0x01, 0x12, 0x04, 0x00, 0x02, 0x40, 0x28, 0xae, 0x14, 0x7a, - 0xe1, 0x47, 0xae, 0x06, 0x00, 0x03, 0x34, 0x56, 0x08, 0x00, 0x04, 0x12, 0x34, 0x56, 0x78, 0x0a, - 0x00, 0x05, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0x0b, 0x00, 0x06, 0x00, 0x00, 0x00, - 0x0c, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x0b, 0x00, 0x07, - 0x00, 0x00, 0x00, 0x0e, 0x74, 0x65, 0x73, 0x74, 0x62, 0x79, 0x74, 0x65, 0x62, 0x75, 0x66, 0x66, - 0x65, 0x72, 0x0f, 0x00, 0x08, 0x08, 0x00, 0x00, 0x00, 0x05, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, - 0x77, 0x88, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x05, 0x0d, 0x00, - 0x09, 0x0b, 0x0b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x61, 0x73, 0x64, 0x66, 0x00, - 0x00, 0x00, 0x04, 0x71, 0x77, 0x65, 0x72, 0x00, 0x00, 0x00, 0x04, 0x7a, 0x78, 0x63, 0x76, 0x00, - 0x00, 0x00, 0x04, 0x68, 0x6a, 0x6b, 0x6c, 0x0d, 0x00, 0x41, 0x0b, 0x0c, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x03, 0x66, 0x6f, 0x6f, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x01, 0xff, 0x04, - 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x03, 0x00, 0x00, 0x08, - 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x0b, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x0f, - 0x00, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x09, 0x0b, 0x0b, 0x00, 0x00, 0x00, 0x00, - 0x0d, 0x00, 0x41, 0x0b, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - } - sl := (*rt.GoSlice)(unsafe.Pointer(&buf)) - pos, err := decode(rt.UnpackEface(v).Type, sl.Ptr, sl.Len, 0, unsafe.Pointer(&v), rs, 0) - require.NoError(t, err) - require.Equal(t, len(buf), pos) + var v TestSimple + rs := new(RuntimeState) + buf := []byte{ + 0x02, 0x00, 0x00, 0x01, 0x03, 0x00, 0x01, 0x12, 0x04, 0x00, 0x02, 0x40, 0x28, 0xae, 0x14, 0x7a, + 0xe1, 0x47, 0xae, 0x06, 0x00, 0x03, 0x34, 0x56, 0x08, 0x00, 0x04, 0x12, 0x34, 0x56, 0x78, 0x0a, + 0x00, 0x05, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0x0b, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x0c, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x0b, 0x00, 0x07, + 0x00, 0x00, 0x00, 0x0e, 0x74, 0x65, 0x73, 0x74, 0x62, 0x79, 0x74, 0x65, 0x62, 0x75, 0x66, 0x66, + 0x65, 0x72, 0x0f, 0x00, 0x08, 0x08, 0x00, 0x00, 0x00, 0x05, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, + 0x77, 0x88, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x05, 0x0d, 0x00, + 0x09, 0x0b, 0x0b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x61, 0x73, 0x64, 0x66, 0x00, + 0x00, 0x00, 0x04, 0x71, 0x77, 0x65, 0x72, 0x00, 0x00, 0x00, 0x04, 0x7a, 0x78, 0x63, 0x76, 0x00, + 0x00, 0x00, 0x04, 0x68, 0x6a, 0x6b, 0x6c, 0x0d, 0x00, 0x41, 0x0b, 0x0c, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x03, 0x66, 0x6f, 0x6f, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x01, 0xff, 0x04, + 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x03, 0x00, 0x00, 0x08, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x0b, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x0f, + 0x00, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x09, 0x0b, 0x0b, 0x00, 0x00, 0x00, 0x00, + 0x0d, 0x00, 0x41, 0x0b, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + } + sl := (*rt.GoSlice)(unsafe.Pointer(&buf)) + pos, err := decode(rt.UnpackEface(v).Type, sl.Ptr, sl.Len, 0, unsafe.Pointer(&v), rs, 0) + require.NoError(t, err) + require.Equal(t, len(buf), pos) } type TestWithDefaultValue struct { - A int64 `frugal:"0,default,i64"` - B int64 `frugal:"1,default,i64"` - C int64 `frugal:"2,default,i64"` + A int64 `frugal:"0,default,i64"` + B int64 `frugal:"1,default,i64"` + C int64 `frugal:"2,default,i64"` } func (self *TestWithDefaultValue) InitDefault() { - *self = TestWithDefaultValue { B: 100 } + *self = TestWithDefaultValue{B: 100} } func TestDecoder_WithDefaultValue(t *testing.T) { - var v TestWithDefaultValue - rs := new(RuntimeState) - buf := []byte { 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00 } - sl := (*rt.GoSlice)(unsafe.Pointer(&buf)) - pos, err := decode(rt.UnpackEface(v).Type, sl.Ptr, sl.Len, 0, unsafe.Pointer(&v), rs, 0) - require.NoError(t, err) - require.Equal(t, len(buf), pos) - spew.Dump(v) + var v TestWithDefaultValue + rs := new(RuntimeState) + buf := []byte{0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00} + sl := (*rt.GoSlice)(unsafe.Pointer(&buf)) + pos, err := decode(rt.UnpackEface(v).Type, sl.Ptr, sl.Len, 0, unsafe.Pointer(&v), rs, 0) + require.NoError(t, err) + require.Equal(t, len(buf), pos) + spew.Dump(v) } type TestNoCopyString struct { - A string `frugal:"1,default,string"` - B string `frugal:"2,default,string,nocopy"` - C *string `frugal:"3,optional,string,nocopy"` - D []byte `frugal:"4,default,binary"` - E []byte `frugal:"5,default,binary,nocopy"` - F *[]byte `frugal:"6,optional,binary,nocopy"` + A string `frugal:"1,default,string"` + B string `frugal:"2,default,string,nocopy"` + C *string `frugal:"3,optional,string,nocopy"` + D []byte `frugal:"4,default,binary"` + E []byte `frugal:"5,default,binary,nocopy"` + F *[]byte `frugal:"6,optional,binary,nocopy"` } func TestDecoder_NoCopyString(t *testing.T) { - var v TestNoCopyString - rs := new(RuntimeState) - buf := []byte { - 0x0b, 0, 1, 0, 0, 0, 5, 't', 'e', 's', 't', '1', - 0x0b, 0, 2, 0, 0, 0, 5, 't', 'e', 's', 't', '2', - 0x0b, 0, 3, 0, 0, 0, 5, 't', 'e', 's', 't', '3', - 0x0b, 0, 4, 0, 0, 0, 5, 't', 'e', 's', 't', '4', - 0x0b, 0, 5, 0, 0, 0, 5, 't', 'e', 's', 't', '5', - 0x0b, 0, 6, 0, 0, 0, 5, 't', 'e', 's', 't', '6', - 0x00, - } - sl := (*rt.GoSlice)(unsafe.Pointer(&buf)) - pos, err := decode(rt.UnpackEface(v).Type, sl.Ptr, sl.Len, 0, unsafe.Pointer(&v), rs, 0) - require.NoError(t, err) - require.Equal(t, len(buf), pos) - require.Equal(t, TestNoCopyString { - A: "test1", - B: "test2", - C: &(&struct{v string}{"test3"}).v, - D: []byte("test4"), - E: []byte("test5"), - F: &(&struct{v []byte}{[]byte("test6")}).v, - }, v) - println("buf: source =", &buf[0]) - println("v.A: normal =", *(*unsafe.Pointer)(unsafe.Pointer(&v.A))) - println("v.B: nocopy =", *(*unsafe.Pointer)(unsafe.Pointer(&v.B))) - println("v.C: nocopy =", *(*unsafe.Pointer)(unsafe.Pointer(v.C))) - println("v.D: normal =", &v.D[0]) - println("v.E: nocopy =", &v.E[0]) - println("v.F: nocopy =", &(*v.F)[0]) - spew.Dump(v) + var v TestNoCopyString + rs := new(RuntimeState) + buf := []byte{ + 0x0b, 0, 1, 0, 0, 0, 5, 't', 'e', 's', 't', '1', + 0x0b, 0, 2, 0, 0, 0, 5, 't', 'e', 's', 't', '2', + 0x0b, 0, 3, 0, 0, 0, 5, 't', 'e', 's', 't', '3', + 0x0b, 0, 4, 0, 0, 0, 5, 't', 'e', 's', 't', '4', + 0x0b, 0, 5, 0, 0, 0, 5, 't', 'e', 's', 't', '5', + 0x0b, 0, 6, 0, 0, 0, 5, 't', 'e', 's', 't', '6', + 0x00, + } + sl := (*rt.GoSlice)(unsafe.Pointer(&buf)) + pos, err := decode(rt.UnpackEface(v).Type, sl.Ptr, sl.Len, 0, unsafe.Pointer(&v), rs, 0) + require.NoError(t, err) + require.Equal(t, len(buf), pos) + require.Equal(t, TestNoCopyString{ + A: "test1", + B: "test2", + C: &(&struct{ v string }{"test3"}).v, + D: []byte("test4"), + E: []byte("test5"), + F: &(&struct{ v []byte }{[]byte("test6")}).v, + }, v) + println("buf: source =", &buf[0]) + println("v.A: normal =", *(*unsafe.Pointer)(unsafe.Pointer(&v.A))) + println("v.B: nocopy =", *(*unsafe.Pointer)(unsafe.Pointer(&v.B))) + println("v.C: nocopy =", *(*unsafe.Pointer)(unsafe.Pointer(v.C))) + println("v.D: normal =", &v.D[0]) + println("v.E: nocopy =", &v.E[0]) + println("v.F: nocopy =", &(*v.F)[0]) + spew.Dump(v) } diff --git a/internal/binary/decoder/errors.go b/internal/binary/decoder/errors.go index 3fa665f..6621ad8 100644 --- a/internal/binary/decoder/errors.go +++ b/internal/binary/decoder/errors.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,41 +17,45 @@ package decoder import ( - `fmt` - `math/bits` + "fmt" + "math/bits" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" ) //go:nosplit func error_eof(n int) error { - return fmt.Errorf("frugal: unexpected EOF: %d bytes short", n) + return fmt.Errorf("frugal: unexpected EOF: %d bytes short", n) } //go:nosplit func error_skip(e int) error { - switch e { - case ETAG : return fmt.Errorf("frugal: error when skipping fields: -1 (invalid tag)") - case EEOF : return fmt.Errorf("frugal: error when skipping fields: -2 (unexpected EOF)") - case ESTACK : return fmt.Errorf("frugal: error when skipping fields: -3 (value nesting too deep)") - default : return fmt.Errorf("frugal: error when skipping fields: %d (unknown error)", e) - } + switch e { + case ETAG: + return fmt.Errorf("frugal: error when skipping fields: -1 (invalid tag)") + case EEOF: + return fmt.Errorf("frugal: error when skipping fields: -2 (unexpected EOF)") + case ESTACK: + return fmt.Errorf("frugal: error when skipping fields: -3 (value nesting too deep)") + default: + return fmt.Errorf("frugal: error when skipping fields: %d (unknown error)", e) + } } //go:nosplit func error_type(e uint8, t uint8) error { - return fmt.Errorf("frugal: type mismatch: %d expected, got %d", e, t) + return fmt.Errorf("frugal: type mismatch: %d expected, got %d", e, t) } //go:nosplit func error_missing(t *rt.GoType, i int, m uint64) error { - return fmt.Errorf("frugal: missing required field %d for type %s", i * 64 + bits.TrailingZeros64(m), t) + return fmt.Errorf("frugal: missing required field %d for type %s", i*64+bits.TrailingZeros64(m), t) } var ( - F_error_eof = hir.RegisterGCall(error_eof, emu_gcall_error_eof) - F_error_skip = hir.RegisterGCall(error_skip, emu_gcall_error_skip) - F_error_type = hir.RegisterGCall(error_type, emu_gcall_error_type) - F_error_missing = hir.RegisterGCall(error_missing, emu_gcall_error_missing) + F_error_eof = hir.RegisterGCall(error_eof, emu_gcall_error_eof) + F_error_skip = hir.RegisterGCall(error_skip, emu_gcall_error_skip) + F_error_type = hir.RegisterGCall(error_type, emu_gcall_error_type) + F_error_missing = hir.RegisterGCall(error_missing, emu_gcall_error_missing) ) diff --git a/internal/binary/decoder/errors_emu.go b/internal/binary/decoder/errors_emu.go index b0e7846..f52da67 100644 --- a/internal/binary/decoder/errors_emu.go +++ b/internal/binary/decoder/errors_emu.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,46 +17,46 @@ package decoder import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" ) func emu_seterr(ctx hir.CallContext, i int, err error) { - vv := (*rt.GoIface)(unsafe.Pointer(&err)) - ctx.Rp(i, unsafe.Pointer(vv.Itab)) - ctx.Rp(i + 1, vv.Value) + vv := (*rt.GoIface)(unsafe.Pointer(&err)) + ctx.Rp(i, unsafe.Pointer(vv.Itab)) + ctx.Rp(i+1, vv.Value) } func emu_gcall_error_eof(ctx hir.CallContext) { - if !ctx.Verify("i", "**") { - panic("invalid error_eof call") - } else { - emu_seterr(ctx, 0, error_eof(int(ctx.Au(0)))) - } + if !ctx.Verify("i", "**") { + panic("invalid error_eof call") + } else { + emu_seterr(ctx, 0, error_eof(int(ctx.Au(0)))) + } } func emu_gcall_error_skip(ctx hir.CallContext) { - if !ctx.Verify("i", "**") { - panic("invalid error_skip call") - } else { - emu_seterr(ctx, 0, error_skip(int(ctx.Au(0)))) - } + if !ctx.Verify("i", "**") { + panic("invalid error_skip call") + } else { + emu_seterr(ctx, 0, error_skip(int(ctx.Au(0)))) + } } func emu_gcall_error_type(ctx hir.CallContext) { - if !ctx.Verify("ii", "**") { - panic("invalid error_type call") - } else { - emu_seterr(ctx, 0, error_type(uint8(ctx.Au(0)), uint8(ctx.Au(1)))) - } + if !ctx.Verify("ii", "**") { + panic("invalid error_type call") + } else { + emu_seterr(ctx, 0, error_type(uint8(ctx.Au(0)), uint8(ctx.Au(1)))) + } } func emu_gcall_error_missing(ctx hir.CallContext) { - if !ctx.Verify("*ii", "**") { - panic("invalid error_skip call") - } else { - emu_seterr(ctx, 0, error_missing((*rt.GoType)(ctx.Ap(0)), int(ctx.Au(1)), ctx.Au(2))) - } + if !ctx.Verify("*ii", "**") { + panic("invalid error_skip call") + } else { + emu_seterr(ctx, 0, error_missing((*rt.GoType)(ctx.Ap(0)), int(ctx.Au(1)), ctx.Au(2))) + } } diff --git a/internal/binary/decoder/initfn.go b/internal/binary/decoder/initfn.go index ed0359d..c998719 100644 --- a/internal/binary/decoder/initfn.go +++ b/internal/binary/decoder/initfn.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,57 +17,57 @@ package decoder import ( - `fmt` - `sync` - `unsafe` + "fmt" + "sync" + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" ) var ( - initFnLock = new(sync.RWMutex) - initFnCache = make(map[unsafe.Pointer]*hir.CallHandle) + initFnLock = new(sync.RWMutex) + initFnCache = make(map[unsafe.Pointer]*hir.CallHandle) ) func toInitFn(fp unsafe.Pointer) (fn func(unsafe.Pointer)) { - *(*unsafe.Pointer)(unsafe.Pointer(&fn)) = unsafe.Pointer(&fp) - return + *(*unsafe.Pointer)(unsafe.Pointer(&fn)) = unsafe.Pointer(&fp) + return } func addInitFn(fp unsafe.Pointer) *hir.CallHandle { - var ok bool - var fn *hir.CallHandle + var ok bool + var fn *hir.CallHandle - /* check function cache */ - initFnLock.RLock() - fn, ok = initFnCache[fp] - initFnLock.RUnlock() + /* check function cache */ + initFnLock.RLock() + fn, ok = initFnCache[fp] + initFnLock.RUnlock() - /* exists, use the cached value */ - if ok { - return fn - } + /* exists, use the cached value */ + if ok { + return fn + } - /* lock in write mode */ - initFnLock.Lock() - defer initFnLock.Unlock() + /* lock in write mode */ + initFnLock.Lock() + defer initFnLock.Unlock() - /* double check */ - if fn, ok = initFnCache[fp]; ok { - return fn - } + /* double check */ + if fn, ok = initFnCache[fp]; ok { + return fn + } - /* still not exists, register a new function */ - fn = hir.RegisterGCall(toInitFn(fp), func(ctx hir.CallContext) { - if !ctx.Verify("*", "") { - panic(fmt.Sprintf("invalid %s call", rt.FuncName(fp))) - } else { - toInitFn(fp)(ctx.Ap(0)) - } - }) + /* still not exists, register a new function */ + fn = hir.RegisterGCall(toInitFn(fp), func(ctx hir.CallContext) { + if !ctx.Verify("*", "") { + panic(fmt.Sprintf("invalid %s call", rt.FuncName(fp))) + } else { + toInitFn(fp)(ctx.Ap(0)) + } + }) - /* update the cache */ - initFnCache[fp] = fn - return fn + /* update the cache */ + initFnCache[fp] = fn + return fn } diff --git a/internal/binary/decoder/linker.go b/internal/binary/decoder/linker.go index 2f096a5..8058acb 100644 --- a/internal/binary/decoder/linker.go +++ b/internal/binary/decoder/linker.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,31 +17,31 @@ package decoder import ( - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/utils` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/utils" ) type Linker interface { - Link(p hir.Program) Decoder + Link(p hir.Program) Decoder } var ( - linker Linker - F_decode *hir.CallHandle + linker Linker + F_decode *hir.CallHandle ) func init() { - F_decode = hir.RegisterGCall(decode, emu_gcall_decode) + F_decode = hir.RegisterGCall(decode, emu_gcall_decode) } func Link(p hir.Program) Decoder { - if linker == nil || utils.ForceEmulator { - return link_emu(p) - } else { - return linker.Link(p) - } + if linker == nil || utils.ForceEmulator { + return link_emu(p) + } else { + return linker.Link(p) + } } func SetLinker(v Linker) { - linker = v + linker = v } diff --git a/internal/binary/decoder/linker_amd64.go b/internal/binary/decoder/linker_amd64.go index 8900f6f..ad5d563 100644 --- a/internal/binary/decoder/linker_amd64.go +++ b/internal/binary/decoder/linker_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,27 +17,27 @@ package decoder import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/atm/pgen` - `github.com/cloudwego/frugal/internal/loader` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/atm/pgen" + "github.com/cloudwego/frugal/internal/loader" ) type ( - LinkerAMD64 struct{} + LinkerAMD64 struct{} ) const ( - _NativeStackSize = _stack__do_skip + _NativeStackSize = _stack__do_skip ) func init() { - SetLinker(new(LinkerAMD64)) + SetLinker(new(LinkerAMD64)) } func (LinkerAMD64) Link(p hir.Program) Decoder { - fn := pgen.CreateCodeGen((Decoder)(nil)).Generate(p, _NativeStackSize) - fp := loader.Loader(fn.Code).Load("decoder", fn.Frame) - return *(*Decoder)(unsafe.Pointer(&fp)) + fn := pgen.CreateCodeGen((Decoder)(nil)).Generate(p, _NativeStackSize) + fp := loader.Loader(fn.Code).Load("decoder", fn.Frame) + return *(*Decoder)(unsafe.Pointer(&fp)) } diff --git a/internal/binary/decoder/linker_arm64.go b/internal/binary/decoder/linker_arm64.go index 6ac7ab0..3a57df0 100644 --- a/internal/binary/decoder/linker_arm64.go +++ b/internal/binary/decoder/linker_arm64.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 ByteDance Inc. + * Copyright 2024 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/internal/binary/decoder/linker_emu.go b/internal/binary/decoder/linker_emu.go index 9a93428..4df80ea 100644 --- a/internal/binary/decoder/linker_emu.go +++ b/internal/binary/decoder/linker_emu.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,55 +17,55 @@ package decoder import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/atm/emu` - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/emu" + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" ) func link_emu(prog hir.Program) Decoder { - return func(buf unsafe.Pointer, nb int, i int, p unsafe.Pointer, rs *RuntimeState, st int) (pos int, err error) { - ctx := emu.LoadProgram(prog) - ret := (*rt.GoIface)(unsafe.Pointer(&err)) - ctx.Ap(0, buf) - ctx.Au(1, uint64(nb)) - ctx.Au(2, uint64(i)) - ctx.Ap(3, p) - ctx.Ap(4, unsafe.Pointer(rs)) - ctx.Au(5, uint64(st)) - ctx.Run() - pos = int(ctx.Ru(0)) - ret.Itab = (*rt.GoItab)(ctx.Rp(1)) - ret.Value = ctx.Rp(2) - ctx.Free() - return - } + return func(buf unsafe.Pointer, nb int, i int, p unsafe.Pointer, rs *RuntimeState, st int) (pos int, err error) { + ctx := emu.LoadProgram(prog) + ret := (*rt.GoIface)(unsafe.Pointer(&err)) + ctx.Ap(0, buf) + ctx.Au(1, uint64(nb)) + ctx.Au(2, uint64(i)) + ctx.Ap(3, p) + ctx.Ap(4, unsafe.Pointer(rs)) + ctx.Au(5, uint64(st)) + ctx.Run() + pos = int(ctx.Ru(0)) + ret.Itab = (*rt.GoItab)(ctx.Rp(1)) + ret.Value = ctx.Rp(2) + ctx.Free() + return + } } func emu_decode(ctx hir.CallContext) (int, error) { - return decode( - (*rt.GoType)(ctx.Ap(0)), - ctx.Ap(1), - int(ctx.Au(2)), - int(ctx.Au(3)), - ctx.Ap(4), - (*RuntimeState)(ctx.Ap(5)), - int(ctx.Au(6)), - ) + return decode( + (*rt.GoType)(ctx.Ap(0)), + ctx.Ap(1), + int(ctx.Au(2)), + int(ctx.Au(3)), + ctx.Ap(4), + (*RuntimeState)(ctx.Ap(5)), + int(ctx.Au(6)), + ) } func emu_mkreturn(ctx hir.CallContext) func(int, error) { - return func(ret int, err error) { - ctx.Ru(0, uint64(ret)) - emu_seterr(ctx, 1, err) - } + return func(ret int, err error) { + ctx.Ru(0, uint64(ret)) + emu_seterr(ctx, 1, err) + } } func emu_gcall_decode(ctx hir.CallContext) { - if !ctx.Verify("**ii**i", "i**") { - panic("invalid decode call") - } else { - emu_mkreturn(ctx)(emu_decode(ctx)) - } -} \ No newline at end of file + if !ctx.Verify("**ii**i", "i**") { + panic("invalid decode call") + } else { + emu_mkreturn(ctx)(emu_decode(ctx)) + } +} diff --git a/internal/binary/decoder/mapassign.go b/internal/binary/decoder/mapassign.go index 2dc098e..db65d16 100644 --- a/internal/binary/decoder/mapassign.go +++ b/internal/binary/decoder/mapassign.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,10 +17,10 @@ package decoder import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" ) //go:noescape @@ -49,9 +49,9 @@ func mapassign_faststr(t *rt.GoMapType, h *rt.GoMap, s string) unsafe.Pointer func mapassign_fast64ptr(t *rt.GoMapType, h *rt.GoMap, key unsafe.Pointer) unsafe.Pointer var ( - F_mapassign = hir.RegisterGCall(mapassign, emu_gcall_mapassign) - F_mapassign_fast32 = hir.RegisterGCall(mapassign_fast32, emu_gcall_mapassign_fast32) - F_mapassign_fast64 = hir.RegisterGCall(mapassign_fast64, emu_gcall_mapassign_fast64) - F_mapassign_faststr = hir.RegisterGCall(mapassign_faststr, emu_gcall_mapassign_faststr) - F_mapassign_fast64ptr = hir.RegisterGCall(mapassign_fast64ptr, emu_gcall_mapassign_fast64ptr) + F_mapassign = hir.RegisterGCall(mapassign, emu_gcall_mapassign) + F_mapassign_fast32 = hir.RegisterGCall(mapassign_fast32, emu_gcall_mapassign_fast32) + F_mapassign_fast64 = hir.RegisterGCall(mapassign_fast64, emu_gcall_mapassign_fast64) + F_mapassign_faststr = hir.RegisterGCall(mapassign_faststr, emu_gcall_mapassign_faststr) + F_mapassign_fast64ptr = hir.RegisterGCall(mapassign_fast64ptr, emu_gcall_mapassign_fast64ptr) ) diff --git a/internal/binary/decoder/mapassign_emu.go b/internal/binary/decoder/mapassign_emu.go index 9572f0f..bb10ec2 100644 --- a/internal/binary/decoder/mapassign_emu.go +++ b/internal/binary/decoder/mapassign_emu.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,54 +17,54 @@ package decoder import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" ) func emu_string(ctx hir.CallContext, i int) (v string) { - (*rt.GoString)(unsafe.Pointer(&v)).Ptr = ctx.Ap(i) - (*rt.GoString)(unsafe.Pointer(&v)).Len = int(ctx.Au(i + 1)) - return + (*rt.GoString)(unsafe.Pointer(&v)).Ptr = ctx.Ap(i) + (*rt.GoString)(unsafe.Pointer(&v)).Len = int(ctx.Au(i + 1)) + return } func emu_gcall_mapassign(ctx hir.CallContext) { - if !ctx.Verify("***", "*") { - panic("invalid mapassign call") - } else { - ctx.Rp(0, mapassign((*rt.GoMapType)(ctx.Ap(0)), (*rt.GoMap)(ctx.Ap(1)), ctx.Ap(2))) - } + if !ctx.Verify("***", "*") { + panic("invalid mapassign call") + } else { + ctx.Rp(0, mapassign((*rt.GoMapType)(ctx.Ap(0)), (*rt.GoMap)(ctx.Ap(1)), ctx.Ap(2))) + } } func emu_gcall_mapassign_fast32(ctx hir.CallContext) { - if !ctx.Verify("**i", "*") { - panic("invalid mapassign_fast32 call") - } else { - ctx.Rp(0, mapassign_fast32((*rt.GoMapType)(ctx.Ap(0)), (*rt.GoMap)(ctx.Ap(1)), uint32(ctx.Au(2)))) - } + if !ctx.Verify("**i", "*") { + panic("invalid mapassign_fast32 call") + } else { + ctx.Rp(0, mapassign_fast32((*rt.GoMapType)(ctx.Ap(0)), (*rt.GoMap)(ctx.Ap(1)), uint32(ctx.Au(2)))) + } } func emu_gcall_mapassign_fast64(ctx hir.CallContext) { - if !ctx.Verify("**i", "*") { - panic("invalid mapassign_fast64 call") - } else { - ctx.Rp(0, mapassign_fast64((*rt.GoMapType)(ctx.Ap(0)), (*rt.GoMap)(ctx.Ap(1)), ctx.Au(2))) - } + if !ctx.Verify("**i", "*") { + panic("invalid mapassign_fast64 call") + } else { + ctx.Rp(0, mapassign_fast64((*rt.GoMapType)(ctx.Ap(0)), (*rt.GoMap)(ctx.Ap(1)), ctx.Au(2))) + } } func emu_gcall_mapassign_faststr(ctx hir.CallContext) { - if !ctx.Verify("***i", "*") { - panic("invalid mapassign_faststr call") - } else { - ctx.Rp(0, mapassign_faststr((*rt.GoMapType)(ctx.Ap(0)), (*rt.GoMap)(ctx.Ap(1)), emu_string(ctx, 2))) - } + if !ctx.Verify("***i", "*") { + panic("invalid mapassign_faststr call") + } else { + ctx.Rp(0, mapassign_faststr((*rt.GoMapType)(ctx.Ap(0)), (*rt.GoMap)(ctx.Ap(1)), emu_string(ctx, 2))) + } } func emu_gcall_mapassign_fast64ptr(ctx hir.CallContext) { - if !ctx.Verify("***", "*") { - panic("invalid mapassign_fast64 call") - } else { - ctx.Rp(0, mapassign_fast64ptr((*rt.GoMapType)(ctx.Ap(0)), (*rt.GoMap)(ctx.Ap(1)), ctx.Ap(2))) - } + if !ctx.Verify("***", "*") { + panic("invalid mapassign_fast64 call") + } else { + ctx.Rp(0, mapassign_fast64ptr((*rt.GoMapType)(ctx.Ap(0)), (*rt.GoMap)(ctx.Ap(1)), ctx.Ap(2))) + } } diff --git a/internal/binary/decoder/native_amd64.go b/internal/binary/decoder/native_amd64.go index 9b797d0..7366a91 100644 --- a/internal/binary/decoder/native_amd64.go +++ b/internal/binary/decoder/native_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/internal/binary/decoder/opcode.go b/internal/binary/decoder/opcode.go index 078a119..033031e 100644 --- a/internal/binary/decoder/opcode.go +++ b/internal/binary/decoder/opcode.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,107 +17,107 @@ package decoder import ( - `fmt` + "fmt" ) type OpCode uint8 const ( - OP_int OpCode = iota - OP_str - OP_str_nocopy - OP_bin - OP_bin_nocopy - OP_enum - OP_size - OP_type - OP_seek - OP_deref - OP_ctr_load - OP_ctr_decr - OP_ctr_is_zero - OP_map_alloc - OP_map_close - OP_map_set_i8 - OP_map_set_i16 - OP_map_set_i32 - OP_map_set_i64 - OP_map_set_str - OP_map_set_enum - OP_map_set_pointer - OP_list_alloc - OP_struct_skip - OP_struct_ignore - OP_struct_bitmap - OP_struct_switch - OP_struct_require - OP_struct_is_stop - OP_struct_mark_tag - OP_struct_read_type - OP_struct_check_type - OP_make_state - OP_drop_state - OP_construct - OP_initialize - OP_defer - OP_goto - OP_halt + OP_int OpCode = iota + OP_str + OP_str_nocopy + OP_bin + OP_bin_nocopy + OP_enum + OP_size + OP_type + OP_seek + OP_deref + OP_ctr_load + OP_ctr_decr + OP_ctr_is_zero + OP_map_alloc + OP_map_close + OP_map_set_i8 + OP_map_set_i16 + OP_map_set_i32 + OP_map_set_i64 + OP_map_set_str + OP_map_set_enum + OP_map_set_pointer + OP_list_alloc + OP_struct_skip + OP_struct_ignore + OP_struct_bitmap + OP_struct_switch + OP_struct_require + OP_struct_is_stop + OP_struct_mark_tag + OP_struct_read_type + OP_struct_check_type + OP_make_state + OP_drop_state + OP_construct + OP_initialize + OP_defer + OP_goto + OP_halt ) -var _OpNames = [256]string { - OP_int : "int", - OP_str : "str", - OP_str_nocopy : "str_nocopy", - OP_bin : "bin", - OP_bin_nocopy : "bin_nocopy", - OP_enum : "enum", - OP_size : "size", - OP_type : "type", - OP_seek : "seek", - OP_deref : "deref", - OP_ctr_load : "ctr_load", - OP_ctr_decr : "ctr_decr", - OP_ctr_is_zero : "ctr_is_zero", - OP_map_alloc : "map_alloc", - OP_map_close : "map_close", - OP_map_set_i8 : "map_set_i8", - OP_map_set_i16 : "map_set_i16", - OP_map_set_i32 : "map_set_i32", - OP_map_set_i64 : "map_set_i64", - OP_map_set_str : "map_set_str", - OP_map_set_enum : "map_set_enum", - OP_map_set_pointer : "map_set_pointer", - OP_list_alloc : "list_alloc", - OP_struct_skip : "struct_skip", - OP_struct_ignore : "struct_ignore", - OP_struct_bitmap : "struct_bitmap", - OP_struct_switch : "struct_switch", - OP_struct_require : "struct_require", - OP_struct_is_stop : "struct_is_stop", - OP_struct_mark_tag : "struct_mark_tag", - OP_struct_read_type : "struct_read_type", - OP_struct_check_type : "struct_check_type", - OP_make_state : "make_state", - OP_drop_state : "drop_state", - OP_construct : "construct", - OP_initialize : "initialize", - OP_defer : "defer", - OP_goto : "goto", - OP_halt : "halt", +var _OpNames = [256]string{ + OP_int: "int", + OP_str: "str", + OP_str_nocopy: "str_nocopy", + OP_bin: "bin", + OP_bin_nocopy: "bin_nocopy", + OP_enum: "enum", + OP_size: "size", + OP_type: "type", + OP_seek: "seek", + OP_deref: "deref", + OP_ctr_load: "ctr_load", + OP_ctr_decr: "ctr_decr", + OP_ctr_is_zero: "ctr_is_zero", + OP_map_alloc: "map_alloc", + OP_map_close: "map_close", + OP_map_set_i8: "map_set_i8", + OP_map_set_i16: "map_set_i16", + OP_map_set_i32: "map_set_i32", + OP_map_set_i64: "map_set_i64", + OP_map_set_str: "map_set_str", + OP_map_set_enum: "map_set_enum", + OP_map_set_pointer: "map_set_pointer", + OP_list_alloc: "list_alloc", + OP_struct_skip: "struct_skip", + OP_struct_ignore: "struct_ignore", + OP_struct_bitmap: "struct_bitmap", + OP_struct_switch: "struct_switch", + OP_struct_require: "struct_require", + OP_struct_is_stop: "struct_is_stop", + OP_struct_mark_tag: "struct_mark_tag", + OP_struct_read_type: "struct_read_type", + OP_struct_check_type: "struct_check_type", + OP_make_state: "make_state", + OP_drop_state: "drop_state", + OP_construct: "construct", + OP_initialize: "initialize", + OP_defer: "defer", + OP_goto: "goto", + OP_halt: "halt", } -var _OpBranches = [256]bool { - OP_ctr_is_zero : true, - OP_struct_switch : true, - OP_struct_is_stop : true, - OP_struct_check_type : true, - OP_goto : true, +var _OpBranches = [256]bool{ + OP_ctr_is_zero: true, + OP_struct_switch: true, + OP_struct_is_stop: true, + OP_struct_check_type: true, + OP_goto: true, } func (self OpCode) String() string { - if _OpNames[self] != "" { - return _OpNames[self] - } else { - return fmt.Sprintf("OpCode(%d)", self) - } + if _OpNames[self] != "" { + return _OpNames[self] + } else { + return fmt.Sprintf("OpCode(%d)", self) + } } diff --git a/internal/binary/decoder/optimizer.go b/internal/binary/decoder/optimizer.go index 0742bbb..562c0d2 100644 --- a/internal/binary/decoder/optimizer.go +++ b/internal/binary/decoder/optimizer.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,293 +17,297 @@ package decoder import ( - `fmt` - `sort` - `strings` + "fmt" + "sort" + "strings" - `github.com/cloudwego/frugal/internal/rt` - `github.com/oleiade/lane` + "github.com/cloudwego/frugal/internal/rt" + "github.com/oleiade/lane" ) type BasicBlock struct { - P Program - Src int - End int - Link []*BasicBlock + P Program + Src int + End int + Link []*BasicBlock } func (self *BasicBlock) Len() int { - return self.End - self.Src + return self.End - self.Src } func (self *BasicBlock) Free() { - q := lane.NewQueue() - m := make(map[*BasicBlock]struct{}) - - /* traverse the graph with BFS */ - for q.Enqueue(self); !q.Empty(); { - v := q.Dequeue() - p := v.(*BasicBlock) - - /* add branch to queue */ - for _, b := range p.Link { - q.Enqueue(b) - } - - /* clear branch, and add to free list */ - m[p] = struct{}{} - p.Link = p.Link[:0] - } - - /* reset and free all the nodes */ - for p := range m { - freeBasicBlock(p) - } + q := lane.NewQueue() + m := make(map[*BasicBlock]struct{}) + + /* traverse the graph with BFS */ + for q.Enqueue(self); !q.Empty(); { + v := q.Dequeue() + p := v.(*BasicBlock) + + /* add branch to queue */ + for _, b := range p.Link { + q.Enqueue(b) + } + + /* clear branch, and add to free list */ + m[p] = struct{}{} + p.Link = p.Link[:0] + } + + /* reset and free all the nodes */ + for p := range m { + freeBasicBlock(p) + } } func (self *BasicBlock) String() string { - n := self.End - self.Src - v := make([]string, n + 1) + n := self.End - self.Src + v := make([]string, n+1) - /* dump every instructions */ - for i := self.Src; i < self.End; i++ { - v[i - self.Src + 1] = " " + self.P[i].Disassemble() - } + /* dump every instructions */ + for i := self.Src; i < self.End; i++ { + v[i-self.Src+1] = " " + self.P[i].Disassemble() + } - /* add the entry label */ - v[0] = fmt.Sprintf("L_%d:", self.Src) - return strings.Join(v, "\n") + /* add the entry label */ + v[0] = fmt.Sprintf("L_%d:", self.Src) + return strings.Join(v, "\n") } type GraphBuilder struct { - Pin map[int]bool - Graph map[int]*BasicBlock + Pin map[int]bool + Graph map[int]*BasicBlock } func (self *GraphBuilder) scan(p Program) { - for _, v := range p { - if _OpBranches[v.Op] { - self.Pin[v.To] = true - } - } + for _, v := range p { + if _OpBranches[v.Op] { + self.Pin[v.To] = true + } + } } func (self *GraphBuilder) block(p Program, i int, bb *BasicBlock) { - bb.Src = i - bb.End = i - - /* traverse down until it hits a branch instruction */ - for i < len(p) && !_OpBranches[p[i].Op] { - i++ - bb.End++ - - /* hit a merge point, merge with existing block */ - if self.Pin[i] { - bb.Link = append(bb.Link, self.branch(p, i)) - return - } - } - - /* end of basic block */ - if i == len(p) { - return - } - - /* also include the branch instruction */ - if bb.End++; p[i].Op != OP_struct_switch { - bb.Link = append(bb.Link, self.branch(p, p[i].To)) - } else { - for _, v := range p[i].IntSeq() { - if v >= 0 { - bb.Link = append(bb.Link, self.branch(p, v)) - } - } - } - - /* GOTO instruction doesn't technically "branch", anything - * sits between it and the next branch target are unreachable. */ - if p[i].Op != OP_goto { - bb.Link = append(bb.Link, self.branch(p, i + 1)) - } + bb.Src = i + bb.End = i + + /* traverse down until it hits a branch instruction */ + for i < len(p) && !_OpBranches[p[i].Op] { + i++ + bb.End++ + + /* hit a merge point, merge with existing block */ + if self.Pin[i] { + bb.Link = append(bb.Link, self.branch(p, i)) + return + } + } + + /* end of basic block */ + if i == len(p) { + return + } + + /* also include the branch instruction */ + if bb.End++; p[i].Op != OP_struct_switch { + bb.Link = append(bb.Link, self.branch(p, p[i].To)) + } else { + for _, v := range p[i].IntSeq() { + if v >= 0 { + bb.Link = append(bb.Link, self.branch(p, v)) + } + } + } + + /* GOTO instruction doesn't technically "branch", anything + * sits between it and the next branch target are unreachable. */ + if p[i].Op != OP_goto { + bb.Link = append(bb.Link, self.branch(p, i+1)) + } } func (self *GraphBuilder) branch(p Program, i int) *BasicBlock { - var ok bool - var bb *BasicBlock - - /* check for existing basic blocks */ - if bb, ok = self.Graph[i]; ok { - return bb - } - - /* create a new block */ - bb = newBasicBlock() - bb.P, bb.Link = p, bb.Link[:0] - - /* process the new block */ - self.Graph[i] = bb - self.block(p, i, bb) - return bb + var ok bool + var bb *BasicBlock + + /* check for existing basic blocks */ + if bb, ok = self.Graph[i]; ok { + return bb + } + + /* create a new block */ + bb = newBasicBlock() + bb.P, bb.Link = p, bb.Link[:0] + + /* process the new block */ + self.Graph[i] = bb + self.block(p, i, bb) + return bb } func (self *GraphBuilder) Free() { - rt.MapClear(self.Pin) - rt.MapClear(self.Graph) - freeGraphBuilder(self) + rt.MapClear(self.Pin) + rt.MapClear(self.Graph) + freeGraphBuilder(self) } func (self *GraphBuilder) Build(p Program) *BasicBlock { - self.scan(p) - return self.branch(p, 0) + self.scan(p) + return self.branch(p, 0) } func (self *GraphBuilder) BuildAndFree(p Program) (bb *BasicBlock) { - bb = self.Build(p) - self.Free() - return + bb = self.Build(p) + self.Free() + return } type _OptimizerState struct { - buf []*BasicBlock - refs map[int]int - mask map[*BasicBlock]bool + buf []*BasicBlock + refs map[int]int + mask map[*BasicBlock]bool } func (self *_OptimizerState) visit(bb *BasicBlock) bool { - var mm bool - var ok bool - - /* check for duplication */ - if mm, ok = self.mask[bb]; mm && ok { - return false - } - - /* add to block buffer */ - self.buf = append(self.buf, bb) - self.mask[bb] = true - return true + var mm bool + var ok bool + + /* check for duplication */ + if mm, ok = self.mask[bb]; mm && ok { + return false + } + + /* add to block buffer */ + self.buf = append(self.buf, bb) + self.mask[bb] = true + return true } func Optimize(p Program) Program { - acc := 0 - ret := newProgram() - buf := lane.NewQueue() - ctx := newOptimizerState() - cfg := newGraphBuilder().BuildAndFree(p) - - /* travel with BFS */ - for buf.Enqueue(cfg); !buf.Empty(); { - v := buf.Dequeue() - b := v.(*BasicBlock) - - /* check for duplication, and then mark as visited */ - if !ctx.visit(b) { - continue - } - - /* optimize each block */ - for _, pass := range _PassTab { - pass(b) - } - - /* add conditional branches if any */ - for _, q := range b.Link { - buf.Enqueue(q) - } - } - - /* sort the blocks by entry point */ - sort.Slice(ctx.buf, func(i int, j int) bool { - return ctx.buf[i].Src < ctx.buf[j].Src - }) - - /* remap all the branch locations */ - for _, bb := range ctx.buf { - ctx.refs[bb.Src] = acc - acc += bb.End - bb.Src - } - - /* adjust all the branch targets */ - for _, bb := range ctx.buf { - if end := bb.End; bb.Src != end { - if ins := &bb.P[end - 1]; _OpBranches[ins.Op] { - if ins.Op != OP_struct_switch { - ins.To = ctx.refs[ins.To] - } else { - for i, v := range ins.IntSeq() { - if v >= 0 { - ins.IntSeq()[i] = ctx.refs[v] - } - } - } - } - } - } - - /* merge all the basic blocks */ - for _, bb := range ctx.buf { - ret = append(ret, bb.P[bb.Src:bb.End]...) - } - - /* release the original program */ - p.Free() - freeOptimizerState(ctx) - return ret + acc := 0 + ret := newProgram() + buf := lane.NewQueue() + ctx := newOptimizerState() + cfg := newGraphBuilder().BuildAndFree(p) + + /* travel with BFS */ + for buf.Enqueue(cfg); !buf.Empty(); { + v := buf.Dequeue() + b := v.(*BasicBlock) + + /* check for duplication, and then mark as visited */ + if !ctx.visit(b) { + continue + } + + /* optimize each block */ + for _, pass := range _PassTab { + pass(b) + } + + /* add conditional branches if any */ + for _, q := range b.Link { + buf.Enqueue(q) + } + } + + /* sort the blocks by entry point */ + sort.Slice(ctx.buf, func(i int, j int) bool { + return ctx.buf[i].Src < ctx.buf[j].Src + }) + + /* remap all the branch locations */ + for _, bb := range ctx.buf { + ctx.refs[bb.Src] = acc + acc += bb.End - bb.Src + } + + /* adjust all the branch targets */ + for _, bb := range ctx.buf { + if end := bb.End; bb.Src != end { + if ins := &bb.P[end-1]; _OpBranches[ins.Op] { + if ins.Op != OP_struct_switch { + ins.To = ctx.refs[ins.To] + } else { + for i, v := range ins.IntSeq() { + if v >= 0 { + ins.IntSeq()[i] = ctx.refs[v] + } + } + } + } + } + } + + /* merge all the basic blocks */ + for _, bb := range ctx.buf { + ret = append(ret, bb.P[bb.Src:bb.End]...) + } + + /* release the original program */ + p.Free() + freeOptimizerState(ctx) + return ret } -var _PassTab = [...]func(p *BasicBlock) { - _PASS_SeekMerging, - _PASS_NopElimination, - _PASS_Compacting, +var _PassTab = [...]func(p *BasicBlock){ + _PASS_SeekMerging, + _PASS_NopElimination, + _PASS_Compacting, } const ( - _NOP OpCode = 0xff + _NOP OpCode = 0xff ) func init() { - _OpNames[_NOP] = "(nop)" + _OpNames[_NOP] = "(nop)" } // Seek Merging Pass: merges seeking instructions as much as possible. func _PASS_SeekMerging(bb *BasicBlock) { - for i := bb.Src; i < bb.End; i++ { - if p := &bb.P[i]; p.Op == OP_seek { - for r, j := true, i + 1; r && j < bb.End; i, j = i + 1, j + 1 { - switch bb.P[j].Op { - case _NOP : break - case OP_seek : p.Iv += bb.P[j].Iv; bb.P[j].Op = _NOP - default : r = false - } - } - } - } + for i := bb.Src; i < bb.End; i++ { + if p := &bb.P[i]; p.Op == OP_seek { + for r, j := true, i+1; r && j < bb.End; i, j = i+1, j+1 { + switch bb.P[j].Op { + case _NOP: + break + case OP_seek: + p.Iv += bb.P[j].Iv + bb.P[j].Op = _NOP + default: + r = false + } + } + } + } } // NOP Elimination Pass: remove instructions that are effectively NOPs (`seek 0`) func _PASS_NopElimination(bb *BasicBlock) { - for i := bb.Src; i < bb.End; i++ { - if bb.P[i].Iv == 0 && bb.P[i].Op == OP_seek { - bb.P[i].Op = _NOP - } - } + for i := bb.Src; i < bb.End; i++ { + if bb.P[i].Iv == 0 && bb.P[i].Op == OP_seek { + bb.P[i].Op = _NOP + } + } } // Compacting Pass: remove all the placeholder NOP instructions inserted in the previous pass. func _PASS_Compacting(bb *BasicBlock) { - var i int - var j int - - /* copy instructins excluding NOPs */ - for i, j = bb.Src, bb.Src; i < bb.End; i++ { - if bb.P[i].Op != _NOP { - bb.P[j] = bb.P[i] - j++ - } - } - - /* update basic block end if needed */ - if i != j { - bb.End = j - } + var i int + var j int + + /* copy instructins excluding NOPs */ + for i, j = bb.Src, bb.Src; i < bb.End; i++ { + if bb.P[i].Op != _NOP { + bb.P[j] = bb.P[i] + j++ + } + } + + /* update basic block end if needed */ + if i != j { + bb.End = j + } } diff --git a/internal/binary/decoder/pools.go b/internal/binary/decoder/pools.go index 6ca846b..cfe53f1 100644 --- a/internal/binary/decoder/pools.go +++ b/internal/binary/decoder/pools.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,133 +17,133 @@ package decoder import ( - `reflect` - `sync` + "reflect" + "sync" - `github.com/cloudwego/frugal/internal/opts` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/opts" + "github.com/cloudwego/frugal/internal/rt" ) var ( - programPool sync.Pool - compilerPool sync.Pool - basicBlockPool sync.Pool - graphBuilderPool sync.Pool - runtimeStatePool sync.Pool - optimizerStatePool sync.Pool + programPool sync.Pool + compilerPool sync.Pool + basicBlockPool sync.Pool + graphBuilderPool sync.Pool + runtimeStatePool sync.Pool + optimizerStatePool sync.Pool ) func newProgram() Program { - if v := programPool.Get(); v != nil { - return v.(Program)[:0] - } else { - return make(Program, 0, 16) - } + if v := programPool.Get(); v != nil { + return v.(Program)[:0] + } else { + return make(Program, 0, 16) + } } func freeProgram(p Program) { - programPool.Put(p) + programPool.Put(p) } func newCompiler() *Compiler { - if v := compilerPool.Get(); v == nil { - return allocCompiler() - } else { - return resetCompiler(v.(*Compiler)) - } + if v := compilerPool.Get(); v == nil { + return allocCompiler() + } else { + return resetCompiler(v.(*Compiler)) + } } func freeCompiler(p *Compiler) { - compilerPool.Put(p) + compilerPool.Put(p) } func allocCompiler() *Compiler { - return &Compiler { - o: opts.GetDefaultOptions(), - t: make(map[reflect.Type]bool), - d: make(map[reflect.Type]struct{}), - } + return &Compiler{ + o: opts.GetDefaultOptions(), + t: make(map[reflect.Type]bool), + d: make(map[reflect.Type]struct{}), + } } func resetCompiler(p *Compiler) *Compiler { - p.o = opts.GetDefaultOptions() - rt.MapClear(p.t) - rt.MapClear(p.d) - return p + p.o = opts.GetDefaultOptions() + rt.MapClear(p.t) + rt.MapClear(p.d) + return p } func newBasicBlock() *BasicBlock { - if v := basicBlockPool.Get(); v != nil { - return v.(*BasicBlock) - } else { - return new(BasicBlock) - } + if v := basicBlockPool.Get(); v != nil { + return v.(*BasicBlock) + } else { + return new(BasicBlock) + } } func freeBasicBlock(p *BasicBlock) { - basicBlockPool.Put(p) + basicBlockPool.Put(p) } func newGraphBuilder() *GraphBuilder { - if v := graphBuilderPool.Get(); v == nil { - return allocGraphBuilder() - } else { - return resetGraphBuilder(v.(*GraphBuilder)) - } + if v := graphBuilderPool.Get(); v == nil { + return allocGraphBuilder() + } else { + return resetGraphBuilder(v.(*GraphBuilder)) + } } func freeGraphBuilder(p *GraphBuilder) { - graphBuilderPool.Put(p) + graphBuilderPool.Put(p) } func allocGraphBuilder() *GraphBuilder { - return &GraphBuilder { - Pin : make(map[int]bool), - Graph : make(map[int]*BasicBlock), - } + return &GraphBuilder{ + Pin: make(map[int]bool), + Graph: make(map[int]*BasicBlock), + } } func resetGraphBuilder(p *GraphBuilder) *GraphBuilder { - rt.MapClear(p.Pin) - rt.MapClear(p.Graph) - return p + rt.MapClear(p.Pin) + rt.MapClear(p.Graph) + return p } func newRuntimeState() *RuntimeState { - if v := runtimeStatePool.Get(); v != nil { - return v.(*RuntimeState) - } else { - return new(RuntimeState) - } + if v := runtimeStatePool.Get(); v != nil { + return v.(*RuntimeState) + } else { + return new(RuntimeState) + } } func freeRuntimeState(p *RuntimeState) { - runtimeStatePool.Put(p) + runtimeStatePool.Put(p) } func newOptimizerState() *_OptimizerState { - if v := optimizerStatePool.Get(); v == nil { - return allocOptimizerState() - } else { - return resetOptimizerState(v.(*_OptimizerState)) - } + if v := optimizerStatePool.Get(); v == nil { + return allocOptimizerState() + } else { + return resetOptimizerState(v.(*_OptimizerState)) + } } func freeOptimizerState(p *_OptimizerState) { - optimizerStatePool.Put(p) + optimizerStatePool.Put(p) } func allocOptimizerState() *_OptimizerState { - return &_OptimizerState { - buf : make([]*BasicBlock, 0, 16), - refs : make(map[int]int), - mask : make(map[*BasicBlock]bool), - } + return &_OptimizerState{ + buf: make([]*BasicBlock, 0, 16), + refs: make(map[int]int), + mask: make(map[*BasicBlock]bool), + } } func resetOptimizerState(p *_OptimizerState) *_OptimizerState { - p.buf = p.buf[:0] - rt.MapClear(p.refs) - rt.MapClear(p.mask) - return p + p.buf = p.buf[:0] + rt.MapClear(p.refs) + rt.MapClear(p.mask) + return p } diff --git a/internal/binary/decoder/skipping.go b/internal/binary/decoder/skipping.go index d356b47..2d62927 100644 --- a/internal/binary/decoder/skipping.go +++ b/internal/binary/decoder/skipping.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,15 +17,15 @@ package decoder import ( - `github.com/cloudwego/frugal/internal/atm/hir` + "github.com/cloudwego/frugal/internal/atm/hir" ) const ( - ETAG = -1 - EEOF = -2 - ESTACK = -3 + ETAG = -1 + EEOF = -2 + ESTACK = -3 ) var ( - C_skip = hir.RegisterCCall(archSkippingFn(), emu_ccall_skip) + C_skip = hir.RegisterCCall(archSkippingFn(), emu_ccall_skip) ) diff --git a/internal/binary/decoder/skipping_amd64.go b/internal/binary/decoder/skipping_amd64.go index c70b20e..e31c1a1 100644 --- a/internal/binary/decoder/skipping_amd64.go +++ b/internal/binary/decoder/skipping_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,9 @@ package decoder import ( - `unsafe` + "unsafe" ) func archSkippingFn() unsafe.Pointer { - return *(*unsafe.Pointer)(unsafe.Pointer(&_subr__do_skip)) + return *(*unsafe.Pointer)(unsafe.Pointer(&_subr__do_skip)) } diff --git a/internal/binary/decoder/skipping_arm64.go b/internal/binary/decoder/skipping_arm64.go index 31e556a..c12d521 100644 --- a/internal/binary/decoder/skipping_arm64.go +++ b/internal/binary/decoder/skipping_arm64.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 ByteDance Inc. + * Copyright 2024 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ package decoder import ( - `unsafe` + "unsafe" ) // NOTE: this file is temporary and is used only for emu on arm diff --git a/internal/binary/decoder/skipping_emu.go b/internal/binary/decoder/skipping_emu.go index 0ca1d8e..f2f44fa 100644 --- a/internal/binary/decoder/skipping_emu.go +++ b/internal/binary/decoder/skipping_emu.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,266 +17,280 @@ package decoder import ( - `math/bits` - `unsafe` + "math/bits" + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/binary/defs` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/binary/defs" ) type ( - _skipbuf_t [defs.StackSize]SkipItem + _skipbuf_t [defs.StackSize]SkipItem ) -var _SkipSizeFixed = [256]int { - defs.T_bool : 1, - defs.T_i8 : 1, - defs.T_double : 8, - defs.T_i16 : 2, - defs.T_i32 : 4, - defs.T_i64 : 8, +var _SkipSizeFixed = [256]int{ + defs.T_bool: 1, + defs.T_i8: 1, + defs.T_double: 8, + defs.T_i16: 2, + defs.T_i32: 4, + defs.T_i64: 8, } const ( - _T_list_elem defs.Tag = 0xfe - _T_map_pair defs.Tag = 0xff + _T_list_elem defs.Tag = 0xfe + _T_map_pair defs.Tag = 0xff ) func u32be(s unsafe.Pointer) int { - return int(bits.ReverseBytes32(*(*uint32)(s))) + return int(bits.ReverseBytes32(*(*uint32)(s))) } func stpop(s *_skipbuf_t, p *int) bool { - if s[*p].N == 0 { - *p-- - return true - } else { - s[*p].N-- - return false - } + if s[*p].N == 0 { + *p-- + return true + } else { + s[*p].N-- + return false + } } func stadd(s *_skipbuf_t, p *int, t defs.Tag) bool { - if *p++; *p >= defs.StackSize { - return false - } else { - s[*p].T, s[*p].N = t, 0 - return true - } + if *p++; *p >= defs.StackSize { + return false + } else { + s[*p].T, s[*p].N = t, 0 + return true + } } func mvbuf(s *unsafe.Pointer, n *int, r *int, nb int) { - *n = *n - nb - *r = *r + nb - *s = unsafe.Pointer(uintptr(*s) + uintptr(nb)) + *n = *n - nb + *r = *r + nb + *s = unsafe.Pointer(uintptr(*s) + uintptr(nb)) } func do_skip(st *_skipbuf_t, s unsafe.Pointer, n int, t defs.Tag) (rv int) { - sp := 0 - st[0].T = t - - /* run until drain */ - for sp >= 0 { - switch st[sp].T { - default: { - return ETAG - } - - /* simple fixed types */ - case defs.T_bool : fallthrough - case defs.T_i8 : fallthrough - case defs.T_double : fallthrough - case defs.T_i16 : fallthrough - case defs.T_i32 : fallthrough - case defs.T_i64 : { - if nb := _SkipSizeFixed[st[sp].T]; n < nb { - return EEOF - } else { - stpop(st, &sp) - mvbuf(&s, &n, &rv, nb) - } - } - - /* strings & binaries */ - case defs.T_string: { - if n < 4 { - return EEOF - } else if nb := u32be(s) + 4; n < nb { - return EEOF - } else { - stpop(st, &sp) - mvbuf(&s, &n, &rv, nb) - } - } - - /* structs */ - case defs.T_struct: { - var nb int - var vt defs.Tag - - /* must have at least 1 byte */ - if n < 1 { - return EEOF - } - - /* check for end of tag */ - if vt = *(*defs.Tag)(s); vt == 0 { - stpop(st, &sp) - mvbuf(&s, &n, &rv, 1) - continue - } - - /* check for tag value */ - if !vt.IsWireTag() { - return ETAG - } - - /* fast-path for primitive fields */ - if nb = _SkipSizeFixed[vt]; nb != 0 { - if n < nb + 3 { - return EEOF - } else { - mvbuf(&s, &n, &rv, nb + 3) - continue - } - } - - /* must have more than 3 bytes (fields cannot have a size of zero), also skip the field ID cause we don't care */ - if n <= 3 { - return EEOF - } else if !stadd(st, &sp, vt) { - return ESTACK - } else { - mvbuf(&s, &n, &rv, 3) - } - } - - /* maps */ - case defs.T_map: { - var np int - var kt defs.Tag - var vt defs.Tag - - /* must have at least 6 bytes */ - if n < 6 { - return EEOF - } - - /* get the element type and count */ - kt = (*[2]defs.Tag)(s)[0] - vt = (*[2]defs.Tag)(s)[1] - np = u32be(unsafe.Pointer(uintptr(s) + 2)) - - /* check for tag value */ - if !kt.IsWireTag() || !vt.IsWireTag() { - return ETAG - } - - /* empty map */ - if np == 0 { - stpop(st, &sp) - mvbuf(&s, &n, &rv, 6) - continue - } - - /* fast path for fixed key and value */ - if nk, nv := _SkipSizeFixed[kt], _SkipSizeFixed[vt]; nk != 0 && nv != 0 { - if nb := np * (nk + nv) + 6; n < nb { - return EEOF - } else { - stpop(st, &sp) - mvbuf(&s, &n, &rv, nb) - continue - } - } - - /* set to parse the map pairs */ - st[sp].K = kt - st[sp].V = vt - st[sp].T = _T_map_pair - st[sp].N = uint32(np) * 2 - 1 - mvbuf(&s, &n, &rv, 6) - } - - /* map pairs */ - case _T_map_pair: { - if vt := st[sp].V; stpop(st, &sp) || st[sp].N & 1 != 0 { - if !stadd(st, &sp, vt) { - return ESTACK - } - } else { - if !stadd(st, &sp, st[sp].K) { - return ESTACK - } - } - } - - /* sets and lists */ - case defs.T_set : fallthrough - case defs.T_list : { - var nv int - var et defs.Tag - - /* must have at least 5 bytes */ - if n < 5 { - return EEOF - } - - /* get the element type and count */ - et = *(*defs.Tag)(s) - nv = u32be(unsafe.Pointer(uintptr(s) + 1)) - - /* check for tag value */ - if !et.IsWireTag() { - return ETAG - } - - /* empty sequence */ - if nv == 0 { - stpop(st, &sp) - mvbuf(&s, &n, &rv, 5) - continue - } - - /* fast path for fixed types */ - if nt := _SkipSizeFixed[et]; nt != 0 { - if nb := nv * nt + 5; n < nb { - return EEOF - } else { - stpop(st, &sp) - mvbuf(&s, &n, &rv, nb) - continue - } - } - - /* set to parse the elements */ - st[sp].T = _T_list_elem - st[sp].V = et - st[sp].N = uint32(nv) - 1 - mvbuf(&s, &n, &rv, 5) - } - - /* list elem */ - case _T_list_elem: { - et := st[sp].V - stpop(st, &sp) - - /* push the element onto stack */ - if !stadd(st, &sp, et) { - return ESTACK - } - } - } - } - - /* all done */ - return + sp := 0 + st[0].T = t + + /* run until drain */ + for sp >= 0 { + switch st[sp].T { + default: + { + return ETAG + } + + /* simple fixed types */ + case defs.T_bool: + fallthrough + case defs.T_i8: + fallthrough + case defs.T_double: + fallthrough + case defs.T_i16: + fallthrough + case defs.T_i32: + fallthrough + case defs.T_i64: + { + if nb := _SkipSizeFixed[st[sp].T]; n < nb { + return EEOF + } else { + stpop(st, &sp) + mvbuf(&s, &n, &rv, nb) + } + } + + /* strings & binaries */ + case defs.T_string: + { + if n < 4 { + return EEOF + } else if nb := u32be(s) + 4; n < nb { + return EEOF + } else { + stpop(st, &sp) + mvbuf(&s, &n, &rv, nb) + } + } + + /* structs */ + case defs.T_struct: + { + var nb int + var vt defs.Tag + + /* must have at least 1 byte */ + if n < 1 { + return EEOF + } + + /* check for end of tag */ + if vt = *(*defs.Tag)(s); vt == 0 { + stpop(st, &sp) + mvbuf(&s, &n, &rv, 1) + continue + } + + /* check for tag value */ + if !vt.IsWireTag() { + return ETAG + } + + /* fast-path for primitive fields */ + if nb = _SkipSizeFixed[vt]; nb != 0 { + if n < nb+3 { + return EEOF + } else { + mvbuf(&s, &n, &rv, nb+3) + continue + } + } + + /* must have more than 3 bytes (fields cannot have a size of zero), also skip the field ID cause we don't care */ + if n <= 3 { + return EEOF + } else if !stadd(st, &sp, vt) { + return ESTACK + } else { + mvbuf(&s, &n, &rv, 3) + } + } + + /* maps */ + case defs.T_map: + { + var np int + var kt defs.Tag + var vt defs.Tag + + /* must have at least 6 bytes */ + if n < 6 { + return EEOF + } + + /* get the element type and count */ + kt = (*[2]defs.Tag)(s)[0] + vt = (*[2]defs.Tag)(s)[1] + np = u32be(unsafe.Pointer(uintptr(s) + 2)) + + /* check for tag value */ + if !kt.IsWireTag() || !vt.IsWireTag() { + return ETAG + } + + /* empty map */ + if np == 0 { + stpop(st, &sp) + mvbuf(&s, &n, &rv, 6) + continue + } + + /* fast path for fixed key and value */ + if nk, nv := _SkipSizeFixed[kt], _SkipSizeFixed[vt]; nk != 0 && nv != 0 { + if nb := np*(nk+nv) + 6; n < nb { + return EEOF + } else { + stpop(st, &sp) + mvbuf(&s, &n, &rv, nb) + continue + } + } + + /* set to parse the map pairs */ + st[sp].K = kt + st[sp].V = vt + st[sp].T = _T_map_pair + st[sp].N = uint32(np)*2 - 1 + mvbuf(&s, &n, &rv, 6) + } + + /* map pairs */ + case _T_map_pair: + { + if vt := st[sp].V; stpop(st, &sp) || st[sp].N&1 != 0 { + if !stadd(st, &sp, vt) { + return ESTACK + } + } else { + if !stadd(st, &sp, st[sp].K) { + return ESTACK + } + } + } + + /* sets and lists */ + case defs.T_set: + fallthrough + case defs.T_list: + { + var nv int + var et defs.Tag + + /* must have at least 5 bytes */ + if n < 5 { + return EEOF + } + + /* get the element type and count */ + et = *(*defs.Tag)(s) + nv = u32be(unsafe.Pointer(uintptr(s) + 1)) + + /* check for tag value */ + if !et.IsWireTag() { + return ETAG + } + + /* empty sequence */ + if nv == 0 { + stpop(st, &sp) + mvbuf(&s, &n, &rv, 5) + continue + } + + /* fast path for fixed types */ + if nt := _SkipSizeFixed[et]; nt != 0 { + if nb := nv*nt + 5; n < nb { + return EEOF + } else { + stpop(st, &sp) + mvbuf(&s, &n, &rv, nb) + continue + } + } + + /* set to parse the elements */ + st[sp].T = _T_list_elem + st[sp].V = et + st[sp].N = uint32(nv) - 1 + mvbuf(&s, &n, &rv, 5) + } + + /* list elem */ + case _T_list_elem: + { + et := st[sp].V + stpop(st, &sp) + + /* push the element onto stack */ + if !stadd(st, &sp, et) { + return ESTACK + } + } + } + } + + /* all done */ + return } func emu_ccall_skip(ctx hir.CallContext) { - if !ctx.Verify("**ii", "i") { - panic("invalid skip call") - } else { - ctx.Ru(0, uint64(do_skip((*_skipbuf_t)(ctx.Ap(0)), ctx.Ap(1), int(ctx.Au(2)), defs.Tag(ctx.Au(3))))) - } + if !ctx.Verify("**ii", "i") { + panic("invalid skip call") + } else { + ctx.Ru(0, uint64(do_skip((*_skipbuf_t)(ctx.Ap(0)), ctx.Ap(1), int(ctx.Au(2)), defs.Tag(ctx.Au(3))))) + } } diff --git a/internal/binary/decoder/skipping_emu_test.go b/internal/binary/decoder/skipping_emu_test.go index 1443d18..76169c7 100644 --- a/internal/binary/decoder/skipping_emu_test.go +++ b/internal/binary/decoder/skipping_emu_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,180 +17,180 @@ package decoder import ( - `encoding/hex` - `testing` - `unsafe` + "encoding/hex" + "testing" + "unsafe" - `github.com/cloudwego/frugal/internal/binary/defs` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/binary/defs" + "github.com/cloudwego/frugal/internal/rt" ) func run_skipping_emu(t *testing.T, v []byte, exp int, tag defs.Tag) { - var sb _skipbuf_t - mm := *(*rt.GoSlice)(unsafe.Pointer(&v)) - rv := do_skip(&sb, mm.Ptr, mm.Len, tag) - if rv != exp { - if rv >= 0 && exp < 0 { - t.Errorf("got %d while expecting error %d", rv, exp) - } else if rv < 0 && exp >= 0 { - t.Errorf("got unexpected error %d while expecting %d", rv, exp) - } else if rv < 0 && exp < 0 { - t.Errorf("got a wrong error %d while expecting error %d", rv, exp) - } else { - t.Errorf("got unexpected return value %d while expecting %d", rv, exp) - } - t.FailNow() - } + var sb _skipbuf_t + mm := *(*rt.GoSlice)(unsafe.Pointer(&v)) + rv := do_skip(&sb, mm.Ptr, mm.Len, tag) + if rv != exp { + if rv >= 0 && exp < 0 { + t.Errorf("got %d while expecting error %d", rv, exp) + } else if rv < 0 && exp >= 0 { + t.Errorf("got unexpected error %d while expecting %d", rv, exp) + } else if rv < 0 && exp < 0 { + t.Errorf("got a wrong error %d while expecting error %d", rv, exp) + } else { + t.Errorf("got unexpected return value %d while expecting %d", rv, exp) + } + t.FailNow() + } } func TestSkippingEmu_SkipPrimitives(t *testing.T) { - run_skipping_emu(t, []byte{0} , 1, defs.T_bool) - run_skipping_emu(t, []byte{0} , 1, defs.T_i8) - run_skipping_emu(t, []byte{0, 1, 2, 3, 4, 5, 6, 7} , 8, defs.T_double) - run_skipping_emu(t, []byte{0, 1} , 2, defs.T_i16) - run_skipping_emu(t, []byte{0, 1, 2, 3} , 4, defs.T_i32) - run_skipping_emu(t, []byte{0, 1, 2, 3, 4, 5, 6, 7} , 8, defs.T_i64) + run_skipping_emu(t, []byte{0}, 1, defs.T_bool) + run_skipping_emu(t, []byte{0}, 1, defs.T_i8) + run_skipping_emu(t, []byte{0, 1, 2, 3, 4, 5, 6, 7}, 8, defs.T_double) + run_skipping_emu(t, []byte{0, 1}, 2, defs.T_i16) + run_skipping_emu(t, []byte{0, 1, 2, 3}, 4, defs.T_i32) + run_skipping_emu(t, []byte{0, 1, 2, 3, 4, 5, 6, 7}, 8, defs.T_i64) } func TestSkippingEmu_SkipPrimitivesButGotEOF(t *testing.T) { - run_skipping_emu(t, []byte{} , EEOF, defs.T_bool) - run_skipping_emu(t, []byte{} , EEOF, defs.T_i8) - run_skipping_emu(t, []byte{} , EEOF, defs.T_double) - run_skipping_emu(t, []byte{0} , EEOF, defs.T_double) - run_skipping_emu(t, []byte{0, 1} , EEOF, defs.T_double) - run_skipping_emu(t, []byte{0, 1, 2} , EEOF, defs.T_double) - run_skipping_emu(t, []byte{0, 1, 2, 3} , EEOF, defs.T_double) - run_skipping_emu(t, []byte{0, 1, 2, 3, 4} , EEOF, defs.T_double) - run_skipping_emu(t, []byte{0, 1, 2, 3, 4, 5} , EEOF, defs.T_double) - run_skipping_emu(t, []byte{0, 1, 2, 3, 4, 5, 6} , EEOF, defs.T_double) - run_skipping_emu(t, []byte{} , EEOF, defs.T_i16) - run_skipping_emu(t, []byte{0} , EEOF, defs.T_i16) - run_skipping_emu(t, []byte{} , EEOF, defs.T_i32) - run_skipping_emu(t, []byte{0} , EEOF, defs.T_i32) - run_skipping_emu(t, []byte{0, 1} , EEOF, defs.T_i32) - run_skipping_emu(t, []byte{0, 1, 2} , EEOF, defs.T_i32) - run_skipping_emu(t, []byte{} , EEOF, defs.T_i64) - run_skipping_emu(t, []byte{0} , EEOF, defs.T_i64) - run_skipping_emu(t, []byte{0, 1} , EEOF, defs.T_i64) - run_skipping_emu(t, []byte{0, 1, 2} , EEOF, defs.T_i64) - run_skipping_emu(t, []byte{0, 1, 2, 3} , EEOF, defs.T_i64) - run_skipping_emu(t, []byte{0, 1, 2, 3, 4} , EEOF, defs.T_i64) - run_skipping_emu(t, []byte{0, 1, 2, 3, 4, 5} , EEOF, defs.T_i64) - run_skipping_emu(t, []byte{0, 1, 2, 3, 4, 5, 6} , EEOF, defs.T_i64) + run_skipping_emu(t, []byte{}, EEOF, defs.T_bool) + run_skipping_emu(t, []byte{}, EEOF, defs.T_i8) + run_skipping_emu(t, []byte{}, EEOF, defs.T_double) + run_skipping_emu(t, []byte{0}, EEOF, defs.T_double) + run_skipping_emu(t, []byte{0, 1}, EEOF, defs.T_double) + run_skipping_emu(t, []byte{0, 1, 2}, EEOF, defs.T_double) + run_skipping_emu(t, []byte{0, 1, 2, 3}, EEOF, defs.T_double) + run_skipping_emu(t, []byte{0, 1, 2, 3, 4}, EEOF, defs.T_double) + run_skipping_emu(t, []byte{0, 1, 2, 3, 4, 5}, EEOF, defs.T_double) + run_skipping_emu(t, []byte{0, 1, 2, 3, 4, 5, 6}, EEOF, defs.T_double) + run_skipping_emu(t, []byte{}, EEOF, defs.T_i16) + run_skipping_emu(t, []byte{0}, EEOF, defs.T_i16) + run_skipping_emu(t, []byte{}, EEOF, defs.T_i32) + run_skipping_emu(t, []byte{0}, EEOF, defs.T_i32) + run_skipping_emu(t, []byte{0, 1}, EEOF, defs.T_i32) + run_skipping_emu(t, []byte{0, 1, 2}, EEOF, defs.T_i32) + run_skipping_emu(t, []byte{}, EEOF, defs.T_i64) + run_skipping_emu(t, []byte{0}, EEOF, defs.T_i64) + run_skipping_emu(t, []byte{0, 1}, EEOF, defs.T_i64) + run_skipping_emu(t, []byte{0, 1, 2}, EEOF, defs.T_i64) + run_skipping_emu(t, []byte{0, 1, 2, 3}, EEOF, defs.T_i64) + run_skipping_emu(t, []byte{0, 1, 2, 3, 4}, EEOF, defs.T_i64) + run_skipping_emu(t, []byte{0, 1, 2, 3, 4, 5}, EEOF, defs.T_i64) + run_skipping_emu(t, []byte{0, 1, 2, 3, 4, 5, 6}, EEOF, defs.T_i64) } func TestSkippingEmu_SkipStringsAndBinaries(t *testing.T) { - run_skipping_emu(t, []byte{0, 0, 0, 0} , 4, defs.T_string) - run_skipping_emu(t, []byte("\x00\x00\x00\x0chello, world") , 16, defs.T_string) + run_skipping_emu(t, []byte{0, 0, 0, 0}, 4, defs.T_string) + run_skipping_emu(t, []byte("\x00\x00\x00\x0chello, world"), 16, defs.T_string) } func TestSkippingEmu_SkipStringsAndBinariesButGotEOF(t *testing.T) { - run_skipping_emu(t, []byte{} , EEOF, defs.T_string) - run_skipping_emu(t, []byte{0} , EEOF, defs.T_string) - run_skipping_emu(t, []byte{0, 0} , EEOF, defs.T_string) - run_skipping_emu(t, []byte{0, 0, 0} , EEOF, defs.T_string) - run_skipping_emu(t, []byte{0, 0, 0, 12} , EEOF, defs.T_string) - run_skipping_emu(t, []byte("\x00\x00\x00\x0ch") , EEOF, defs.T_string) - run_skipping_emu(t, []byte("\x00\x00\x00\x0che") , EEOF, defs.T_string) - run_skipping_emu(t, []byte("\x00\x00\x00\x0chel") , EEOF, defs.T_string) - run_skipping_emu(t, []byte("\x00\x00\x00\x0chell") , EEOF, defs.T_string) - run_skipping_emu(t, []byte("\x00\x00\x00\x0chello") , EEOF, defs.T_string) - run_skipping_emu(t, []byte("\x00\x00\x00\x0chello,") , EEOF, defs.T_string) - run_skipping_emu(t, []byte("\x00\x00\x00\x0chello, ") , EEOF, defs.T_string) - run_skipping_emu(t, []byte("\x00\x00\x00\x0chello, w") , EEOF, defs.T_string) - run_skipping_emu(t, []byte("\x00\x00\x00\x0chello, wo") , EEOF, defs.T_string) - run_skipping_emu(t, []byte("\x00\x00\x00\x0chello, wor") , EEOF, defs.T_string) - run_skipping_emu(t, []byte("\x00\x00\x00\x0chello, worl") , EEOF, defs.T_string) + run_skipping_emu(t, []byte{}, EEOF, defs.T_string) + run_skipping_emu(t, []byte{0}, EEOF, defs.T_string) + run_skipping_emu(t, []byte{0, 0}, EEOF, defs.T_string) + run_skipping_emu(t, []byte{0, 0, 0}, EEOF, defs.T_string) + run_skipping_emu(t, []byte{0, 0, 0, 12}, EEOF, defs.T_string) + run_skipping_emu(t, []byte("\x00\x00\x00\x0ch"), EEOF, defs.T_string) + run_skipping_emu(t, []byte("\x00\x00\x00\x0che"), EEOF, defs.T_string) + run_skipping_emu(t, []byte("\x00\x00\x00\x0chel"), EEOF, defs.T_string) + run_skipping_emu(t, []byte("\x00\x00\x00\x0chell"), EEOF, defs.T_string) + run_skipping_emu(t, []byte("\x00\x00\x00\x0chello"), EEOF, defs.T_string) + run_skipping_emu(t, []byte("\x00\x00\x00\x0chello,"), EEOF, defs.T_string) + run_skipping_emu(t, []byte("\x00\x00\x00\x0chello, "), EEOF, defs.T_string) + run_skipping_emu(t, []byte("\x00\x00\x00\x0chello, w"), EEOF, defs.T_string) + run_skipping_emu(t, []byte("\x00\x00\x00\x0chello, wo"), EEOF, defs.T_string) + run_skipping_emu(t, []byte("\x00\x00\x00\x0chello, wor"), EEOF, defs.T_string) + run_skipping_emu(t, []byte("\x00\x00\x00\x0chello, worl"), EEOF, defs.T_string) } func TestSkippingEmu_SkipStruct(t *testing.T) { - run_skipping_emu(t, []byte{ 0} , 1, defs.T_struct) - run_skipping_emu(t, []byte{ 2, 0, 0, 1, 0} , 5, defs.T_struct) - run_skipping_emu(t, []byte{ 3, 0, 0, 1, 0} , 5, defs.T_struct) - run_skipping_emu(t, []byte{ 3, 0, 0, 1, 6, 0, 1, 1, 2, 0} , 10, defs.T_struct) - run_skipping_emu(t, []byte{ 4, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0} , 12, defs.T_struct) - run_skipping_emu(t, []byte{ 6, 0, 0, 1, 2, 0} , 6, defs.T_struct) - run_skipping_emu(t, []byte{ 8, 0, 0, 1, 2, 3, 4, 0} , 8, defs.T_struct) - run_skipping_emu(t, []byte{10, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0} , 12, defs.T_struct) - run_skipping_emu(t, []byte("\x0b\x00\x00\x00\x00\x00\x0chello, world\x00") , 20, defs.T_struct) - run_skipping_emu(t, []byte{12, 0, 0, 0, 0} , 5, defs.T_struct) - run_skipping_emu(t, []byte{12, 0, 0, 3, 0, 0, 1, 0, 0} , 9, defs.T_struct) - run_skipping_emu(t, []byte{13, 0, 0, 3, 6, 0, 0, 0, 1, 1, 2, 3, 0} , 13, defs.T_struct) - run_skipping_emu(t, []byte{14, 0, 0, 8, 0, 0, 0, 1, 1, 2, 3, 4, 0} , 13, defs.T_struct) - run_skipping_emu(t, []byte{15, 0, 0, 6, 0, 0, 0, 2, 1, 2, 3, 4, 0} , 13, defs.T_struct) + run_skipping_emu(t, []byte{0}, 1, defs.T_struct) + run_skipping_emu(t, []byte{2, 0, 0, 1, 0}, 5, defs.T_struct) + run_skipping_emu(t, []byte{3, 0, 0, 1, 0}, 5, defs.T_struct) + run_skipping_emu(t, []byte{3, 0, 0, 1, 6, 0, 1, 1, 2, 0}, 10, defs.T_struct) + run_skipping_emu(t, []byte{4, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0}, 12, defs.T_struct) + run_skipping_emu(t, []byte{6, 0, 0, 1, 2, 0}, 6, defs.T_struct) + run_skipping_emu(t, []byte{8, 0, 0, 1, 2, 3, 4, 0}, 8, defs.T_struct) + run_skipping_emu(t, []byte{10, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0}, 12, defs.T_struct) + run_skipping_emu(t, []byte("\x0b\x00\x00\x00\x00\x00\x0chello, world\x00"), 20, defs.T_struct) + run_skipping_emu(t, []byte{12, 0, 0, 0, 0}, 5, defs.T_struct) + run_skipping_emu(t, []byte{12, 0, 0, 3, 0, 0, 1, 0, 0}, 9, defs.T_struct) + run_skipping_emu(t, []byte{13, 0, 0, 3, 6, 0, 0, 0, 1, 1, 2, 3, 0}, 13, defs.T_struct) + run_skipping_emu(t, []byte{14, 0, 0, 8, 0, 0, 0, 1, 1, 2, 3, 4, 0}, 13, defs.T_struct) + run_skipping_emu(t, []byte{15, 0, 0, 6, 0, 0, 0, 2, 1, 2, 3, 4, 0}, 13, defs.T_struct) } func TestSkippingEmu_SkipStructButGotErrors(t *testing.T) { - run_skipping_emu(t, []byte{} , EEOF, defs.T_struct) - run_skipping_emu(t, []byte{ 2} , EEOF, defs.T_struct) - run_skipping_emu(t, []byte{ 2, 0, 0} , EEOF, defs.T_struct) - run_skipping_emu(t, []byte{ 9} , ETAG, defs.T_struct) - run_skipping_emu(t, []byte{11} , EEOF, defs.T_struct) - run_skipping_emu(t, []byte{11, 0, 0} , EEOF, defs.T_struct) + run_skipping_emu(t, []byte{}, EEOF, defs.T_struct) + run_skipping_emu(t, []byte{2}, EEOF, defs.T_struct) + run_skipping_emu(t, []byte{2, 0, 0}, EEOF, defs.T_struct) + run_skipping_emu(t, []byte{9}, ETAG, defs.T_struct) + run_skipping_emu(t, []byte{11}, EEOF, defs.T_struct) + run_skipping_emu(t, []byte{11, 0, 0}, EEOF, defs.T_struct) } func TestSkippingEmu_SkipMapOfPrimitives(t *testing.T) { - run_skipping_emu(t, []byte{3, 6, 0, 0, 0, 0} , 6, defs.T_map) - run_skipping_emu(t, []byte{3, 6, 0, 0, 0, 3, 1, 2, 3, 4, 5, 6, 7, 8, 9} , 15, defs.T_map) - run_skipping_emu(t, []byte{6, 8, 0, 0, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4} , 18, defs.T_map) + run_skipping_emu(t, []byte{3, 6, 0, 0, 0, 0}, 6, defs.T_map) + run_skipping_emu(t, []byte{3, 6, 0, 0, 0, 3, 1, 2, 3, 4, 5, 6, 7, 8, 9}, 15, defs.T_map) + run_skipping_emu(t, []byte{6, 8, 0, 0, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4}, 18, defs.T_map) } func TestSkippingEmu_SkipMapWithNonPrimitiveKeys(t *testing.T) { - run_skipping_emu(t, []byte{11, 8, 0, 0, 0, 0} , 6, defs.T_map) - run_skipping_emu(t, []byte("\x0b\x08\x00\x00\x00\x01\x00\x00\x00\x04test\x01\x02\x03\x04") , 18, defs.T_map) + run_skipping_emu(t, []byte{11, 8, 0, 0, 0, 0}, 6, defs.T_map) + run_skipping_emu(t, []byte("\x0b\x08\x00\x00\x00\x01\x00\x00\x00\x04test\x01\x02\x03\x04"), 18, defs.T_map) } func TestSkippingEmu_SkipMapButGotErrors(t *testing.T) { - run_skipping_emu(t, []byte{} , EEOF, defs.T_map) - run_skipping_emu(t, []byte{3} , EEOF, defs.T_map) - run_skipping_emu(t, []byte{3, 6} , EEOF, defs.T_map) - run_skipping_emu(t, []byte{3, 6, 0} , EEOF, defs.T_map) - run_skipping_emu(t, []byte{3, 6, 0, 0} , EEOF, defs.T_map) - run_skipping_emu(t, []byte{3, 6, 0, 0, 0} , EEOF, defs.T_map) - run_skipping_emu(t, []byte{3, 6, 0, 0, 0, 1} , EEOF, defs.T_map) - run_skipping_emu(t, []byte{3, 6, 0, 0, 0, 2, 1} , EEOF, defs.T_map) - run_skipping_emu(t, []byte{9, 6, 0, 0, 0, 0} , ETAG, defs.T_map) - run_skipping_emu(t, []byte{3, 9, 0, 0, 0, 3, 1, 2, 3, 4, 5, 6, 7, 8, 9} , ETAG, defs.T_map) - run_skipping_emu(t, []byte{9, 9, 0, 0, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4} , ETAG, defs.T_map) - run_skipping_emu(t, []byte("\x0b\x0b\x00\x00\x00\x01") , EEOF, defs.T_map) + run_skipping_emu(t, []byte{}, EEOF, defs.T_map) + run_skipping_emu(t, []byte{3}, EEOF, defs.T_map) + run_skipping_emu(t, []byte{3, 6}, EEOF, defs.T_map) + run_skipping_emu(t, []byte{3, 6, 0}, EEOF, defs.T_map) + run_skipping_emu(t, []byte{3, 6, 0, 0}, EEOF, defs.T_map) + run_skipping_emu(t, []byte{3, 6, 0, 0, 0}, EEOF, defs.T_map) + run_skipping_emu(t, []byte{3, 6, 0, 0, 0, 1}, EEOF, defs.T_map) + run_skipping_emu(t, []byte{3, 6, 0, 0, 0, 2, 1}, EEOF, defs.T_map) + run_skipping_emu(t, []byte{9, 6, 0, 0, 0, 0}, ETAG, defs.T_map) + run_skipping_emu(t, []byte{3, 9, 0, 0, 0, 3, 1, 2, 3, 4, 5, 6, 7, 8, 9}, ETAG, defs.T_map) + run_skipping_emu(t, []byte{9, 9, 0, 0, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4}, ETAG, defs.T_map) + run_skipping_emu(t, []byte("\x0b\x0b\x00\x00\x00\x01"), EEOF, defs.T_map) } func TestSkippingEmu_SkipSetOrListOfPrimitives(t *testing.T) { - run_skipping_emu(t, []byte{ 2, 0, 0, 0, 16, 1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1, 8, 7, 6, 5}, 21, defs.T_list) - run_skipping_emu(t, []byte{ 3, 0, 0, 0, 16, 1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1, 8, 7, 6, 5}, 21, defs.T_list) - run_skipping_emu(t, []byte{ 4, 0, 0, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1, 8, 7, 6, 5}, 21, defs.T_list) - run_skipping_emu(t, []byte{ 6, 0, 0, 0, 8, 1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1, 8, 7, 6, 5}, 21, defs.T_list) - run_skipping_emu(t, []byte{ 8, 0, 0, 0, 4, 1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1, 8, 7, 6, 5}, 21, defs.T_list) - run_skipping_emu(t, []byte{10, 0, 0, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1, 8, 7, 6, 5}, 21, defs.T_list) + run_skipping_emu(t, []byte{2, 0, 0, 0, 16, 1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1, 8, 7, 6, 5}, 21, defs.T_list) + run_skipping_emu(t, []byte{3, 0, 0, 0, 16, 1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1, 8, 7, 6, 5}, 21, defs.T_list) + run_skipping_emu(t, []byte{4, 0, 0, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1, 8, 7, 6, 5}, 21, defs.T_list) + run_skipping_emu(t, []byte{6, 0, 0, 0, 8, 1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1, 8, 7, 6, 5}, 21, defs.T_list) + run_skipping_emu(t, []byte{8, 0, 0, 0, 4, 1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1, 8, 7, 6, 5}, 21, defs.T_list) + run_skipping_emu(t, []byte{10, 0, 0, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1, 8, 7, 6, 5}, 21, defs.T_list) } func TestSkippingEmu_SkipSetOrListOfBinariesOrStrings(t *testing.T) { - run_skipping_emu(t, []byte("\x0b\x00\x00\x00\x00") , 5, defs.T_list) - run_skipping_emu(t, []byte("\x0b\x00\x00\x00\x01\x00\x00\x00\x00") , 9, defs.T_list) - run_skipping_emu(t, []byte("\x0b\x00\x00\x00\x01\x00\x00\x00\x0chello, world") , 21, defs.T_list) + run_skipping_emu(t, []byte("\x0b\x00\x00\x00\x00"), 5, defs.T_list) + run_skipping_emu(t, []byte("\x0b\x00\x00\x00\x01\x00\x00\x00\x00"), 9, defs.T_list) + run_skipping_emu(t, []byte("\x0b\x00\x00\x00\x01\x00\x00\x00\x0chello, world"), 21, defs.T_list) } func TestSkippingEmu_SkipSetOrListButGotErrors(t *testing.T) { - run_skipping_emu(t, []byte{} , EEOF, defs.T_list) - run_skipping_emu(t, []byte{2} , EEOF, defs.T_list) - run_skipping_emu(t, []byte{2, 0} , EEOF, defs.T_list) - run_skipping_emu(t, []byte{2, 0, 0} , EEOF, defs.T_list) - run_skipping_emu(t, []byte{2, 0, 0, 0} , EEOF, defs.T_list) - run_skipping_emu(t, []byte{2, 0, 0, 0, 1} , EEOF, defs.T_list) - run_skipping_emu(t, []byte{2, 0, 0, 0, 2, 1} , EEOF, defs.T_list) - run_skipping_emu(t, []byte{9, 0, 0, 0, 1, 2} , ETAG, defs.T_list) - run_skipping_emu(t, []byte("\x0b\x00\x00\x00\x01") , EEOF, defs.T_list) + run_skipping_emu(t, []byte{}, EEOF, defs.T_list) + run_skipping_emu(t, []byte{2}, EEOF, defs.T_list) + run_skipping_emu(t, []byte{2, 0}, EEOF, defs.T_list) + run_skipping_emu(t, []byte{2, 0, 0}, EEOF, defs.T_list) + run_skipping_emu(t, []byte{2, 0, 0, 0}, EEOF, defs.T_list) + run_skipping_emu(t, []byte{2, 0, 0, 0, 1}, EEOF, defs.T_list) + run_skipping_emu(t, []byte{2, 0, 0, 0, 2, 1}, EEOF, defs.T_list) + run_skipping_emu(t, []byte{9, 0, 0, 0, 1, 2}, ETAG, defs.T_list) + run_skipping_emu(t, []byte("\x0b\x00\x00\x00\x01"), EEOF, defs.T_list) } func TestSkipListMap(t *testing.T) { - listMap, err := hex.DecodeString("0f00010d000000020b0b00000001000000016100000001620b0b000000010000000161000000016200") - if err != nil { - t.Fatal(err) - } - var fsm _skipbuf_t - var in = (*rt.GoSlice)(unsafe.Pointer(&listMap)) - rv := do_skip(&fsm, in.Ptr, in.Len, defs.T_struct) - if rv != len(listMap) { - t.Fatalf("skip failed: %d", rv) - } -} \ No newline at end of file + listMap, err := hex.DecodeString("0f00010d000000020b0b00000001000000016100000001620b0b000000010000000161000000016200") + if err != nil { + t.Fatal(err) + } + var fsm _skipbuf_t + var in = (*rt.GoSlice)(unsafe.Pointer(&listMap)) + rv := do_skip(&fsm, in.Ptr, in.Len, defs.T_struct) + if rv != len(listMap) { + t.Fatalf("skip failed: %d", rv) + } +} diff --git a/internal/binary/decoder/state.go b/internal/binary/decoder/state.go index 79cde37..06a7507 100644 --- a/internal/binary/decoder/state.go +++ b/internal/binary/decoder/state.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,47 +17,47 @@ package decoder import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/binary/defs` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/binary/defs" + "github.com/cloudwego/frugal/internal/rt" ) const ( - NbOffset = int64(unsafe.Offsetof(StateItem{}.Nb)) - MpOffset = int64(unsafe.Offsetof(StateItem{}.Mp)) - WpOffset = int64(unsafe.Offsetof(StateItem{}.Wp)) - FmOffset = int64(unsafe.Offsetof(StateItem{}.Fm)) + NbOffset = int64(unsafe.Offsetof(StateItem{}.Nb)) + MpOffset = int64(unsafe.Offsetof(StateItem{}.Mp)) + WpOffset = int64(unsafe.Offsetof(StateItem{}.Wp)) + FmOffset = int64(unsafe.Offsetof(StateItem{}.Fm)) ) const ( - SkOffset = int64(unsafe.Offsetof(RuntimeState{}.Sk)) - PrOffset = int64(unsafe.Offsetof(RuntimeState{}.Pr)) - IvOffset = int64(unsafe.Offsetof(RuntimeState{}.Iv)) + SkOffset = int64(unsafe.Offsetof(RuntimeState{}.Sk)) + PrOffset = int64(unsafe.Offsetof(RuntimeState{}.Pr)) + IvOffset = int64(unsafe.Offsetof(RuntimeState{}.Iv)) ) const ( - StateMax = (defs.StackSize - 1) * StateSize - StateSize = int64(unsafe.Sizeof(StateItem{})) + StateMax = (defs.StackSize - 1) * StateSize + StateSize = int64(unsafe.Sizeof(StateItem{})) ) type SkipItem struct { - T defs.Tag - K defs.Tag - V defs.Tag - N uint32 + T defs.Tag + K defs.Tag + V defs.Tag + N uint32 } type StateItem struct { - Nb uint64 - Mp *rt.GoMap - Wp unsafe.Pointer - Fm *FieldBitmap + Nb uint64 + Mp *rt.GoMap + Wp unsafe.Pointer + Fm *FieldBitmap } type RuntimeState struct { - St [defs.StackSize]StateItem // Must be the first field. - Sk [defs.StackSize]SkipItem // Skip buffer, used for non-recursive skipping - Pr unsafe.Pointer // Pointer spill space, used for non-fast string or pointer map access. - Iv uint64 // Integer spill space, used for non-fast string map access. + St [defs.StackSize]StateItem // Must be the first field. + Sk [defs.StackSize]SkipItem // Skip buffer, used for non-recursive skipping + Pr unsafe.Pointer // Pointer spill space, used for non-fast string or pointer map access. + Iv uint64 // Integer spill space, used for non-fast string map access. } diff --git a/internal/binary/decoder/strconv.go b/internal/binary/decoder/strconv.go index ea8638c..d86d45e 100644 --- a/internal/binary/decoder/strconv.go +++ b/internal/binary/decoder/strconv.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,9 @@ package decoder import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` + "github.com/cloudwego/frugal/internal/atm/hir" ) //go:noescape @@ -28,5 +28,5 @@ import ( func slicebytetostring(buf unsafe.Pointer, ptr unsafe.Pointer, n int) string var ( - F_slicebytetostring = hir.RegisterGCall(slicebytetostring, emu_gcall_slicebytetostring) + F_slicebytetostring = hir.RegisterGCall(slicebytetostring, emu_gcall_slicebytetostring) ) diff --git a/internal/binary/decoder/strconv_emu.go b/internal/binary/decoder/strconv_emu.go index 2ff396b..3a2e27a 100644 --- a/internal/binary/decoder/strconv_emu.go +++ b/internal/binary/decoder/strconv_emu.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,16 +17,16 @@ package decoder import ( - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" ) func emu_gcall_slicebytetostring(ctx hir.CallContext) { - if !ctx.Verify("**i", "*i") { - panic("invalid slicebytetostring call") - } else { - v := slicebytetostring(ctx.Ap(0), ctx.Ap(1), int(ctx.Au(2))) - ctx.Rp(0, rt.StringPtr(v)) - ctx.Ru(1, uint64(len(v))) - } -} \ No newline at end of file + if !ctx.Verify("**i", "*i") { + panic("invalid slicebytetostring call") + } else { + v := slicebytetostring(ctx.Ap(0), ctx.Ap(1), int(ctx.Au(2))) + ctx.Rp(0, rt.StringPtr(v)) + ctx.Ru(1, uint64(len(v))) + } +} diff --git a/internal/binary/decoder/translator.go b/internal/binary/decoder/translator.go index f72d82b..d911ff4 100644 --- a/internal/binary/decoder/translator.go +++ b/internal/binary/decoder/translator.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,12 +17,12 @@ package decoder import ( - `fmt` - `reflect` + "fmt" + "reflect" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/binary/defs` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/binary/defs" + "github.com/cloudwego/frugal/internal/rt" ) /** Function Prototype @@ -31,12 +31,12 @@ import ( */ const ( - ARG_buf = 0 - ARG_nb = 1 - ARG_i = 2 - ARG_p = 3 - ARG_rs = 4 - ARG_st = 5 + ARG_buf = 0 + ARG_nb = 1 + ARG_i = 2 + ARG_p = 3 + ARG_rs = 4 + ARG_st = 5 ) /** Register Allocations @@ -53,796 +53,816 @@ const ( */ const ( - WP = hir.P1 - IP = hir.P2 - RS = hir.P3 - ET = hir.P4 // may also be used as a temporary pointer register - EP = hir.P5 // may also be used as a temporary pointer register + WP = hir.P1 + IP = hir.P2 + RS = hir.P3 + ET = hir.P4 // may also be used as a temporary pointer register + EP = hir.P5 // may also be used as a temporary pointer register ) const ( - IC = hir.R2 - ST = hir.R3 - TG = hir.R4 + IC = hir.R2 + ST = hir.R3 + TG = hir.R4 ) const ( - TP = hir.P0 - TR = hir.R0 - UR = hir.R1 + TP = hir.P0 + TR = hir.R0 + UR = hir.R1 ) const ( - LB_eof = "_eof" - LB_halt = "_halt" - LB_type = "_type" - LB_skip = "_skip" - LB_error = "_error" - LB_missing = "_missing" - LB_overflow = "_overflow" + LB_eof = "_eof" + LB_halt = "_halt" + LB_type = "_type" + LB_skip = "_skip" + LB_error = "_error" + LB_missing = "_missing" + LB_overflow = "_overflow" ) var ( - _T_byte *rt.GoType - _E_overflow error - _V_zerovalue uint64 + _T_byte *rt.GoType + _E_overflow error + _V_zerovalue uint64 ) func init() { - _T_byte = rt.UnpackType(reflect.TypeOf(byte(0))) - _E_overflow = fmt.Errorf("frugal: decoder stack overflow") + _T_byte = rt.UnpackType(reflect.TypeOf(byte(0))) + _E_overflow = fmt.Errorf("frugal: decoder stack overflow") } func Translate(s Program) hir.Program { - p := hir.CreateBuilder() - prologue (p) - program (p, s) - epilogue (p) - errors (p) - return p.Build() + p := hir.CreateBuilder() + prologue(p) + program(p, s) + epilogue(p) + errors(p) + return p.Build() } func errors(p *hir.Builder) { - p.Label (LB_eof) - p.LDAQ (ARG_nb, UR) - p.SUB (TR, UR, TR) - p.GCALL (F_error_eof). - A0 (TR). - R0 (ET). - R1 (EP) - p.JMP (LB_error) - p.Label (LB_type) - p.GCALL (F_error_type). - A0 (UR). - A1 (TR). - R0 (ET). - R1 (EP) - p.JMP (LB_error) - p.Label (LB_skip) - p.GCALL (F_error_skip). - A0 (TR). - R0 (ET). - R1 (EP) - p.JMP (LB_error) - p.Label (LB_missing) - p.GCALL (F_error_missing). - A0 (ET). - A1 (UR). - A2 (TR). - R0 (ET). - R1 (EP) - p.JMP (LB_error) - p.Label (LB_overflow) - p.IP (&_E_overflow, TP) - p.LP (TP, 0, ET) - p.LP (TP, 8, EP) - p.JMP (LB_error) + p.Label(LB_eof) + p.LDAQ(ARG_nb, UR) + p.SUB(TR, UR, TR) + p.GCALL(F_error_eof). + A0(TR). + R0(ET). + R1(EP) + p.JMP(LB_error) + p.Label(LB_type) + p.GCALL(F_error_type). + A0(UR). + A1(TR). + R0(ET). + R1(EP) + p.JMP(LB_error) + p.Label(LB_skip) + p.GCALL(F_error_skip). + A0(TR). + R0(ET). + R1(EP) + p.JMP(LB_error) + p.Label(LB_missing) + p.GCALL(F_error_missing). + A0(ET). + A1(UR). + A2(TR). + R0(ET). + R1(EP) + p.JMP(LB_error) + p.Label(LB_overflow) + p.IP(&_E_overflow, TP) + p.LP(TP, 0, ET) + p.LP(TP, 8, EP) + p.JMP(LB_error) } func program(p *hir.Builder, s Program) { - for i, v := range s { - p.Mark(i) - translators[v.Op](p, v) - } + for i, v := range s { + p.Mark(i) + translators[v.Op](p, v) + } } func prologue(p *hir.Builder) { - p.LDAP (ARG_buf, IP) - p.LDAQ (ARG_i, IC) - p.LDAP (ARG_p, WP) - p.LDAP (ARG_rs, RS) - p.LDAQ (ARG_st, ST) - p.MOV (hir.Rz, TR) - p.MOV (hir.Rz, UR) + p.LDAP(ARG_buf, IP) + p.LDAQ(ARG_i, IC) + p.LDAP(ARG_p, WP) + p.LDAP(ARG_rs, RS) + p.LDAQ(ARG_st, ST) + p.MOV(hir.Rz, TR) + p.MOV(hir.Rz, UR) } func epilogue(p *hir.Builder) { - p.Label (LB_halt) - p.MOVP (hir.Pn, ET) - p.MOVP (hir.Pn, EP) - p.Label (LB_error) - p.RET (). - R0 (IC). - R1 (ET). - R2 (EP) -} - -var translators = [256]func(*hir.Builder, Instr) { - OP_int : translate_OP_int, - OP_str : translate_OP_str, - OP_str_nocopy : translate_OP_str_nocopy, - OP_bin : translate_OP_bin, - OP_bin_nocopy : translate_OP_bin_nocopy, - OP_enum : translate_OP_enum, - OP_size : translate_OP_size, - OP_type : translate_OP_type, - OP_seek : translate_OP_seek, - OP_deref : translate_OP_deref, - OP_ctr_load : translate_OP_ctr_load, - OP_ctr_decr : translate_OP_ctr_decr, - OP_ctr_is_zero : translate_OP_ctr_is_zero, - OP_map_alloc : translate_OP_map_alloc, - OP_map_close : translate_OP_map_close, - OP_map_set_i8 : translate_OP_map_set_i8, - OP_map_set_i16 : translate_OP_map_set_i16, - OP_map_set_i32 : translate_OP_map_set_i32, - OP_map_set_i64 : translate_OP_map_set_i64, - OP_map_set_str : translate_OP_map_set_str, - OP_map_set_enum : translate_OP_map_set_enum, - OP_map_set_pointer : translate_OP_map_set_pointer, - OP_list_alloc : translate_OP_list_alloc, - OP_struct_skip : translate_OP_struct_skip, - OP_struct_ignore : translate_OP_struct_ignore, - OP_struct_bitmap : translate_OP_struct_bitmap, - OP_struct_switch : translate_OP_struct_switch, - OP_struct_require : translate_OP_struct_require, - OP_struct_is_stop : translate_OP_struct_is_stop, - OP_struct_mark_tag : translate_OP_struct_mark_tag, - OP_struct_read_type : translate_OP_struct_read_type, - OP_struct_check_type : translate_OP_struct_check_type, - OP_make_state : translate_OP_make_state, - OP_drop_state : translate_OP_drop_state, - OP_construct : translate_OP_construct, - OP_initialize : translate_OP_initialize, - OP_defer : translate_OP_defer, - OP_goto : translate_OP_goto, - OP_halt : translate_OP_halt, + p.Label(LB_halt) + p.MOVP(hir.Pn, ET) + p.MOVP(hir.Pn, EP) + p.Label(LB_error) + p.RET(). + R0(IC). + R1(ET). + R2(EP) +} + +var translators = [256]func(*hir.Builder, Instr){ + OP_int: translate_OP_int, + OP_str: translate_OP_str, + OP_str_nocopy: translate_OP_str_nocopy, + OP_bin: translate_OP_bin, + OP_bin_nocopy: translate_OP_bin_nocopy, + OP_enum: translate_OP_enum, + OP_size: translate_OP_size, + OP_type: translate_OP_type, + OP_seek: translate_OP_seek, + OP_deref: translate_OP_deref, + OP_ctr_load: translate_OP_ctr_load, + OP_ctr_decr: translate_OP_ctr_decr, + OP_ctr_is_zero: translate_OP_ctr_is_zero, + OP_map_alloc: translate_OP_map_alloc, + OP_map_close: translate_OP_map_close, + OP_map_set_i8: translate_OP_map_set_i8, + OP_map_set_i16: translate_OP_map_set_i16, + OP_map_set_i32: translate_OP_map_set_i32, + OP_map_set_i64: translate_OP_map_set_i64, + OP_map_set_str: translate_OP_map_set_str, + OP_map_set_enum: translate_OP_map_set_enum, + OP_map_set_pointer: translate_OP_map_set_pointer, + OP_list_alloc: translate_OP_list_alloc, + OP_struct_skip: translate_OP_struct_skip, + OP_struct_ignore: translate_OP_struct_ignore, + OP_struct_bitmap: translate_OP_struct_bitmap, + OP_struct_switch: translate_OP_struct_switch, + OP_struct_require: translate_OP_struct_require, + OP_struct_is_stop: translate_OP_struct_is_stop, + OP_struct_mark_tag: translate_OP_struct_mark_tag, + OP_struct_read_type: translate_OP_struct_read_type, + OP_struct_check_type: translate_OP_struct_check_type, + OP_make_state: translate_OP_make_state, + OP_drop_state: translate_OP_drop_state, + OP_construct: translate_OP_construct, + OP_initialize: translate_OP_initialize, + OP_defer: translate_OP_defer, + OP_goto: translate_OP_goto, + OP_halt: translate_OP_halt, } func translate_OP_int(p *hir.Builder, v Instr) { - switch v.Iv { - case 1 : p.ADDP(IP, IC, EP); p.LB(EP, 0, TR); p.SB(TR, WP, 0); p.ADDI(IC, 1, IC) - case 2 : p.ADDP(IP, IC, EP); p.LW(EP, 0, TR); p.SWAPW(TR, TR); p.SW(TR, WP, 0); p.ADDI(IC, 2, IC) - case 4 : p.ADDP(IP, IC, EP); p.LL(EP, 0, TR); p.SWAPL(TR, TR); p.SL(TR, WP, 0); p.ADDI(IC, 4, IC) - case 8 : p.ADDP(IP, IC, EP); p.LQ(EP, 0, TR); p.SWAPQ(TR, TR); p.SQ(TR, WP, 0); p.ADDI(IC, 8, IC) - default : panic("can only convert 1, 2, 4 or 8 bytes at a time") - } + switch v.Iv { + case 1: + p.ADDP(IP, IC, EP) + p.LB(EP, 0, TR) + p.SB(TR, WP, 0) + p.ADDI(IC, 1, IC) + case 2: + p.ADDP(IP, IC, EP) + p.LW(EP, 0, TR) + p.SWAPW(TR, TR) + p.SW(TR, WP, 0) + p.ADDI(IC, 2, IC) + case 4: + p.ADDP(IP, IC, EP) + p.LL(EP, 0, TR) + p.SWAPL(TR, TR) + p.SL(TR, WP, 0) + p.ADDI(IC, 4, IC) + case 8: + p.ADDP(IP, IC, EP) + p.LQ(EP, 0, TR) + p.SWAPQ(TR, TR) + p.SQ(TR, WP, 0) + p.ADDI(IC, 8, IC) + default: + panic("can only convert 1, 2, 4 or 8 bytes at a time") + } } func translate_OP_str(p *hir.Builder, _ Instr) { - p.SP (hir.Pn, WP, 0) - p.ADDP (IP, IC, EP) - p.ADDI (IC, 4, IC) - p.LL (EP, 0, TR) - p.SWAPL (TR, TR) - p.LDAQ (ARG_nb, UR) - p.BLTU (UR, TR, LB_eof) - p.BEQ (TR, hir.Rz, "_empty_{n}") - p.ADDPI (EP, 4, EP) - p.ADD (IC, TR, IC) - p.GCALL (F_slicebytetostring). - A0 (hir.Pn). - A1 (EP). - A2 (TR). - R0 (TP). - R1 (TR) - p.SP (TP, WP, 0) - p.Label ("_empty_{n}") - p.SQ (TR, WP, 8) + p.SP(hir.Pn, WP, 0) + p.ADDP(IP, IC, EP) + p.ADDI(IC, 4, IC) + p.LL(EP, 0, TR) + p.SWAPL(TR, TR) + p.LDAQ(ARG_nb, UR) + p.BLTU(UR, TR, LB_eof) + p.BEQ(TR, hir.Rz, "_empty_{n}") + p.ADDPI(EP, 4, EP) + p.ADD(IC, TR, IC) + p.GCALL(F_slicebytetostring). + A0(hir.Pn). + A1(EP). + A2(TR). + R0(TP). + R1(TR) + p.SP(TP, WP, 0) + p.Label("_empty_{n}") + p.SQ(TR, WP, 8) } func translate_OP_str_nocopy(p *hir.Builder, _ Instr) { - p.SP (hir.Pn, WP, 0) - translate_OP_binstr_nocopy(p) + p.SP(hir.Pn, WP, 0) + translate_OP_binstr_nocopy(p) } func translate_OP_bin(p *hir.Builder, _ Instr) { - p.IP (&_V_zerovalue, TP) - p.SP (TP, WP, 0) - p.ADDP (IP, IC, EP) - p.ADDI (IC, 4, IC) - p.LL (EP, 0, TR) - p.SWAPL (TR, TR) - p.LDAQ (ARG_nb, UR) - p.BLTU (UR, TR, LB_eof) - p.BEQ (TR, hir.Rz, "_empty_{n}") - p.ADDPI (EP, 4, EP) - p.ADD (IC, TR, IC) - p.IP (_T_byte, TP) - p.GCALL (F_mallocgc). - A0 (TR). - A1 (TP). - A2 (hir.Rz). - R0 (TP) - p.BCOPY (EP, TR, TP) - p.SP (TP, WP, 0) - p.Label ("_empty_{n}") - p.SQ (TR, WP, 8) - p.SQ (TR, WP, 16) + p.IP(&_V_zerovalue, TP) + p.SP(TP, WP, 0) + p.ADDP(IP, IC, EP) + p.ADDI(IC, 4, IC) + p.LL(EP, 0, TR) + p.SWAPL(TR, TR) + p.LDAQ(ARG_nb, UR) + p.BLTU(UR, TR, LB_eof) + p.BEQ(TR, hir.Rz, "_empty_{n}") + p.ADDPI(EP, 4, EP) + p.ADD(IC, TR, IC) + p.IP(_T_byte, TP) + p.GCALL(F_mallocgc). + A0(TR). + A1(TP). + A2(hir.Rz). + R0(TP) + p.BCOPY(EP, TR, TP) + p.SP(TP, WP, 0) + p.Label("_empty_{n}") + p.SQ(TR, WP, 8) + p.SQ(TR, WP, 16) } func translate_OP_bin_nocopy(p *hir.Builder, _ Instr) { - p.IP (&_V_zerovalue, TP) - p.SP (TP, WP, 0) - translate_OP_binstr_nocopy(p) - p.SQ (TR, WP, 16) + p.IP(&_V_zerovalue, TP) + p.SP(TP, WP, 0) + translate_OP_binstr_nocopy(p) + p.SQ(TR, WP, 16) } func translate_OP_binstr_nocopy(p *hir.Builder) { - p.ADDP (IP, IC, EP) - p.ADDI (IC, 4, IC) - p.LL (EP, 0, TR) - p.SWAPL (TR, TR) - p.LDAQ (ARG_nb, UR) - p.BLTU (UR, TR, LB_eof) - p.BEQ (TR, hir.Rz, "_empty_{n}") - p.ADDPI (EP, 4, EP) - p.ADD (IC, TR, IC) - p.SP (EP, WP, 0) - p.Label ("_empty_{n}") - p.SQ (TR, WP, 8) + p.ADDP(IP, IC, EP) + p.ADDI(IC, 4, IC) + p.LL(EP, 0, TR) + p.SWAPL(TR, TR) + p.LDAQ(ARG_nb, UR) + p.BLTU(UR, TR, LB_eof) + p.BEQ(TR, hir.Rz, "_empty_{n}") + p.ADDPI(EP, 4, EP) + p.ADD(IC, TR, IC) + p.SP(EP, WP, 0) + p.Label("_empty_{n}") + p.SQ(TR, WP, 8) } func translate_OP_enum(p *hir.Builder, _ Instr) { - p.ADDP (IP, IC, EP) - p.LL (EP, 0, TR) - p.SWAPL (TR, TR) - p.SXLQ (TR, TR) - p.SQ (TR, WP, 0) - p.ADDI (IC, 4, IC) + p.ADDP(IP, IC, EP) + p.LL(EP, 0, TR) + p.SWAPL(TR, TR) + p.SXLQ(TR, TR) + p.SQ(TR, WP, 0) + p.ADDI(IC, 4, IC) } func translate_OP_size(p *hir.Builder, v Instr) { - p.IQ (v.Iv, TR) - p.LDAQ (ARG_nb, UR) - p.BLTU (UR, TR, LB_eof) + p.IQ(v.Iv, TR) + p.LDAQ(ARG_nb, UR) + p.BLTU(UR, TR, LB_eof) } func translate_OP_type(p *hir.Builder, v Instr) { - p.ADDP (IP, IC, TP) - p.LB (TP, 0, TR) - p.IB (int8(v.Tx), UR) - p.BNE (TR, UR, LB_type) - p.ADDI (IC, 1, IC) + p.ADDP(IP, IC, TP) + p.LB(TP, 0, TR) + p.IB(int8(v.Tx), UR) + p.BNE(TR, UR, LB_type) + p.ADDI(IC, 1, IC) } func translate_OP_seek(p *hir.Builder, v Instr) { - p.ADDPI (WP, v.Iv, WP) + p.ADDPI(WP, v.Iv, WP) } func translate_OP_deref(p *hir.Builder, v Instr) { - p.LQ (WP, 0, TR) - p.BNE (TR, hir.Rz, "_skip_{n}") - p.IB (1, UR) - p.IP (v.Vt, TP) - p.IQ (int64(v.Vt.Size), TR) - p.GCALL (F_mallocgc). - A0 (TR). - A1 (TP). - A2 (UR). - R0 (TP) - p.SP (TP, WP, 0) - p.Label ("_skip_{n}") - p.LP (WP, 0, WP) + p.LQ(WP, 0, TR) + p.BNE(TR, hir.Rz, "_skip_{n}") + p.IB(1, UR) + p.IP(v.Vt, TP) + p.IQ(int64(v.Vt.Size), TR) + p.GCALL(F_mallocgc). + A0(TR). + A1(TP). + A2(UR). + R0(TP) + p.SP(TP, WP, 0) + p.Label("_skip_{n}") + p.LP(WP, 0, WP) } func translate_OP_ctr_load(p *hir.Builder, _ Instr) { - p.ADDP (IP, IC, EP) - p.ADDI (IC, 4, IC) - p.LL (EP, 0, TR) - p.SWAPL (TR, TR) - p.ADDP (RS, ST, TP) - p.SQ (TR, TP, NbOffset) + p.ADDP(IP, IC, EP) + p.ADDI(IC, 4, IC) + p.LL(EP, 0, TR) + p.SWAPL(TR, TR) + p.ADDP(RS, ST, TP) + p.SQ(TR, TP, NbOffset) } func translate_OP_ctr_decr(p *hir.Builder, _ Instr) { - p.ADDP (RS, ST, TP) - p.LQ (TP, NbOffset, TR) - p.SUBI (TR, 1, TR) - p.SQ (TR, TP, NbOffset) + p.ADDP(RS, ST, TP) + p.LQ(TP, NbOffset, TR) + p.SUBI(TR, 1, TR) + p.SQ(TR, TP, NbOffset) } func translate_OP_ctr_is_zero(p *hir.Builder, v Instr) { - p.ADDP (RS, ST, TP) - p.LQ (TP, NbOffset, TR) - p.BEQ (TR, hir.Rz, p.At(v.To)) + p.ADDP(RS, ST, TP) + p.LQ(TP, NbOffset, TR) + p.BEQ(TR, hir.Rz, p.At(v.To)) } func translate_OP_map_alloc(p *hir.Builder, v Instr) { - p.ADDP (RS, ST, TP) - p.LQ (TP, NbOffset, TR) - p.IP (v.Vt, ET) - p.GCALL (F_makemap). - A0 (ET). - A1 (TR). - A2 (hir.Pn). - R0 (TP) - p.SP (TP, WP, 0) - p.ADDP (RS, ST, EP) - p.SP (TP, EP, MpOffset) + p.ADDP(RS, ST, TP) + p.LQ(TP, NbOffset, TR) + p.IP(v.Vt, ET) + p.GCALL(F_makemap). + A0(ET). + A1(TR). + A2(hir.Pn). + R0(TP) + p.SP(TP, WP, 0) + p.ADDP(RS, ST, EP) + p.SP(TP, EP, MpOffset) } func translate_OP_map_close(p *hir.Builder, _ Instr) { - p.ADDP (RS, ST, TP) - p.SP (hir.Pn, TP, MpOffset) + p.ADDP(RS, ST, TP) + p.SP(hir.Pn, TP, MpOffset) } func translate_OP_map_set_i8(p *hir.Builder, v Instr) { - p.ADDP (IP, IC, EP) - p.ADDP (RS, ST, TP) - p.LP (TP, MpOffset, TP) - p.IP (v.Vt, ET) - p.GCALL (F_mapassign). - A0 (ET). - A1 (TP). - A2 (EP). - R0 (WP) - p.ADDI (IC, 1, IC) + p.ADDP(IP, IC, EP) + p.ADDP(RS, ST, TP) + p.LP(TP, MpOffset, TP) + p.IP(v.Vt, ET) + p.GCALL(F_mapassign). + A0(ET). + A1(TP). + A2(EP). + R0(WP) + p.ADDI(IC, 1, IC) } func translate_OP_map_set_i16(p *hir.Builder, v Instr) { - p.ADDP (IP, IC, ET) - p.ADDI (IC, 2, IC) - p.ADDP (RS, ST, TP) - p.LP (TP, MpOffset, EP) - p.LW (ET, 0, TR) - p.SWAPW (TR, TR) - p.SW (TR, RS, IvOffset) - p.ADDPI (RS, IvOffset, TP) - p.IP (v.Vt, ET) - p.GCALL (F_mapassign). - A0 (ET). - A1 (EP). - A2 (TP). - R0 (WP) + p.ADDP(IP, IC, ET) + p.ADDI(IC, 2, IC) + p.ADDP(RS, ST, TP) + p.LP(TP, MpOffset, EP) + p.LW(ET, 0, TR) + p.SWAPW(TR, TR) + p.SW(TR, RS, IvOffset) + p.ADDPI(RS, IvOffset, TP) + p.IP(v.Vt, ET) + p.GCALL(F_mapassign). + A0(ET). + A1(EP). + A2(TP). + R0(WP) } func translate_OP_map_set_i32(p *hir.Builder, v Instr) { - if rt.MapType(v.Vt).IsFastMap() { - translate_OP_map_set_i32_fast(p, v) - } else { - translate_OP_map_set_i32_safe(p, v) - } + if rt.MapType(v.Vt).IsFastMap() { + translate_OP_map_set_i32_fast(p, v) + } else { + translate_OP_map_set_i32_safe(p, v) + } } func translate_OP_map_set_i32_fast(p *hir.Builder, v Instr) { - p.ADDP (IP, IC, EP) - p.ADDI (IC, 4, IC) - p.ADDP (RS, ST, TP) - p.LP (TP, MpOffset, TP) - p.LL (EP, 0, TR) - p.SWAPL (TR, TR) - p.IP (v.Vt, ET) - p.GCALL (F_mapassign_fast32). - A0 (ET). - A1 (TP). - A2 (TR). - R0 (WP) + p.ADDP(IP, IC, EP) + p.ADDI(IC, 4, IC) + p.ADDP(RS, ST, TP) + p.LP(TP, MpOffset, TP) + p.LL(EP, 0, TR) + p.SWAPL(TR, TR) + p.IP(v.Vt, ET) + p.GCALL(F_mapassign_fast32). + A0(ET). + A1(TP). + A2(TR). + R0(WP) } func translate_OP_map_set_i32_safe(p *hir.Builder, v Instr) { - p.ADDP (IP, IC, ET) - p.ADDI (IC, 4, IC) - p.ADDP (RS, ST, TP) - p.LP (TP, MpOffset, EP) - p.LL (ET, 0, TR) - p.SWAPL (TR, TR) - p.SL (TR, RS, IvOffset) - p.ADDPI (RS, IvOffset, TP) - p.IP (v.Vt, ET) - p.GCALL (F_mapassign). - A0 (ET). - A1 (EP). - A2 (TP). - R0 (WP) + p.ADDP(IP, IC, ET) + p.ADDI(IC, 4, IC) + p.ADDP(RS, ST, TP) + p.LP(TP, MpOffset, EP) + p.LL(ET, 0, TR) + p.SWAPL(TR, TR) + p.SL(TR, RS, IvOffset) + p.ADDPI(RS, IvOffset, TP) + p.IP(v.Vt, ET) + p.GCALL(F_mapassign). + A0(ET). + A1(EP). + A2(TP). + R0(WP) } func translate_OP_map_set_i64(p *hir.Builder, v Instr) { - if rt.MapType(v.Vt).IsFastMap() { - translate_OP_map_set_i64_fast(p, v) - } else { - translate_OP_map_set_i64_safe(p, v) - } + if rt.MapType(v.Vt).IsFastMap() { + translate_OP_map_set_i64_fast(p, v) + } else { + translate_OP_map_set_i64_safe(p, v) + } } func translate_OP_map_set_i64_fast(p *hir.Builder, v Instr) { - p.ADDP (IP, IC, EP) - p.ADDI (IC, 8, IC) - p.ADDP (RS, ST, TP) - p.LP (TP, MpOffset, TP) - p.LQ (EP, 0, TR) - p.SWAPQ (TR, TR) - p.IP (v.Vt, ET) - p.GCALL (F_mapassign_fast64). - A0 (ET). - A1 (TP). - A2 (TR). - R0 (WP) + p.ADDP(IP, IC, EP) + p.ADDI(IC, 8, IC) + p.ADDP(RS, ST, TP) + p.LP(TP, MpOffset, TP) + p.LQ(EP, 0, TR) + p.SWAPQ(TR, TR) + p.IP(v.Vt, ET) + p.GCALL(F_mapassign_fast64). + A0(ET). + A1(TP). + A2(TR). + R0(WP) } func translate_OP_map_set_i64_safe(p *hir.Builder, v Instr) { - p.ADDP (IP, IC, ET) - p.ADDI (IC, 2, IC) - p.ADDP (RS, ST, TP) - p.LP (TP, MpOffset, EP) - p.LQ (ET, 0, TR) - p.SWAPQ (TR, TR) - p.SQ (TR, RS, IvOffset) - p.ADDPI (RS, IvOffset, TP) - p.IP (v.Vt, ET) - p.GCALL (F_mapassign). - A0 (ET). - A1 (EP). - A2 (TP). - R0 (WP) + p.ADDP(IP, IC, ET) + p.ADDI(IC, 2, IC) + p.ADDP(RS, ST, TP) + p.LP(TP, MpOffset, EP) + p.LQ(ET, 0, TR) + p.SWAPQ(TR, TR) + p.SQ(TR, RS, IvOffset) + p.ADDPI(RS, IvOffset, TP) + p.IP(v.Vt, ET) + p.GCALL(F_mapassign). + A0(ET). + A1(EP). + A2(TP). + R0(WP) } func translate_OP_map_set_str(p *hir.Builder, v Instr) { - if rt.MapType(v.Vt).IsFastMap() { - translate_OP_map_set_str_fast(p, v) - } else { - translate_OP_map_set_str_safe(p, v) - } + if rt.MapType(v.Vt).IsFastMap() { + translate_OP_map_set_str_fast(p, v) + } else { + translate_OP_map_set_str_safe(p, v) + } } func translate_OP_map_set_str_fast(p *hir.Builder, v Instr) { - p.ADDP (IP, IC, EP) - p.ADDI (IC, 4, IC) - p.LL (EP, 0, TR) - p.SWAPL (TR, TR) - p.LDAQ (ARG_nb, UR) - p.BLTU (UR, TR, LB_eof) - p.MOVP (hir.Pn, EP) - p.BEQ (TR, hir.Rz, "_empty_{n}") - p.ADDP (IP, IC, ET) - p.ADD (IC, TR, IC) - p.GCALL (F_slicebytetostring). - A0 (hir.Pn). - A1 (ET). - A2 (TR). - R0 (EP). - R1 (TR) - p.Label ("_empty_{n}") - p.ADDP (RS, ST, TP) - p.LP (TP, MpOffset, TP) - p.IP (v.Vt, ET) - p.GCALL (F_mapassign_faststr). - A0 (ET). - A1 (TP). - A2 (EP). - A3 (TR). - R0 (WP) + p.ADDP(IP, IC, EP) + p.ADDI(IC, 4, IC) + p.LL(EP, 0, TR) + p.SWAPL(TR, TR) + p.LDAQ(ARG_nb, UR) + p.BLTU(UR, TR, LB_eof) + p.MOVP(hir.Pn, EP) + p.BEQ(TR, hir.Rz, "_empty_{n}") + p.ADDP(IP, IC, ET) + p.ADD(IC, TR, IC) + p.GCALL(F_slicebytetostring). + A0(hir.Pn). + A1(ET). + A2(TR). + R0(EP). + R1(TR) + p.Label("_empty_{n}") + p.ADDP(RS, ST, TP) + p.LP(TP, MpOffset, TP) + p.IP(v.Vt, ET) + p.GCALL(F_mapassign_faststr). + A0(ET). + A1(TP). + A2(EP). + A3(TR). + R0(WP) } func translate_OP_map_set_str_safe(p *hir.Builder, v Instr) { - p.ADDP (IP, IC, ET) - p.ADDI (IC, 4, IC) - p.LL (ET, 0, TR) - p.SWAPL (TR, TR) - p.LDAQ (ARG_nb, UR) - p.BLTU (UR, TR, LB_eof) - p.SQ (TR, RS, IvOffset) - p.SP (hir.Pn, RS, PrOffset) - p.BEQ (TR, hir.Rz, "_empty_{n}") - p.ADDPI (ET, 4, ET) - p.ADD (IC, TR, IC) - p.GCALL (F_slicebytetostring). - A0 (hir.Pn). - A1 (ET). - A2 (TR). - R0 (TP). - R1 (TR) - p.SP (TP, RS, PrOffset) - p.Label ("_empty_{n}") - p.ADDP (RS, ST, EP) - p.LP (EP, MpOffset, EP) - p.IP (v.Vt, ET) - p.ADDPI (RS, PrOffset, TP) - p.GCALL (F_mapassign). - A0 (ET). - A1 (EP). - A2 (TP). - R0 (WP) - p.SP (hir.Pn, RS, PrOffset) + p.ADDP(IP, IC, ET) + p.ADDI(IC, 4, IC) + p.LL(ET, 0, TR) + p.SWAPL(TR, TR) + p.LDAQ(ARG_nb, UR) + p.BLTU(UR, TR, LB_eof) + p.SQ(TR, RS, IvOffset) + p.SP(hir.Pn, RS, PrOffset) + p.BEQ(TR, hir.Rz, "_empty_{n}") + p.ADDPI(ET, 4, ET) + p.ADD(IC, TR, IC) + p.GCALL(F_slicebytetostring). + A0(hir.Pn). + A1(ET). + A2(TR). + R0(TP). + R1(TR) + p.SP(TP, RS, PrOffset) + p.Label("_empty_{n}") + p.ADDP(RS, ST, EP) + p.LP(EP, MpOffset, EP) + p.IP(v.Vt, ET) + p.ADDPI(RS, PrOffset, TP) + p.GCALL(F_mapassign). + A0(ET). + A1(EP). + A2(TP). + R0(WP) + p.SP(hir.Pn, RS, PrOffset) } func translate_OP_map_set_enum(p *hir.Builder, v Instr) { - if rt.MapType(v.Vt).IsFastMap() { - translate_OP_map_set_enum_fast(p, v) - } else { - translate_OP_map_set_enum_safe(p, v) - } + if rt.MapType(v.Vt).IsFastMap() { + translate_OP_map_set_enum_fast(p, v) + } else { + translate_OP_map_set_enum_safe(p, v) + } } func translate_OP_map_set_enum_fast(p *hir.Builder, v Instr) { - p.ADDP (IP, IC, EP) - p.ADDI (IC, 4, IC) - p.ADDP (RS, ST, TP) - p.LP (TP, MpOffset, TP) - p.LL (EP, 0, TR) - p.SWAPL (TR, TR) - p.SXLQ (TR, TR) - p.IP (v.Vt, ET) - p.GCALL (F_mapassign_fast64). - A0 (ET). - A1 (TP). - A2 (TR). - R0 (WP) + p.ADDP(IP, IC, EP) + p.ADDI(IC, 4, IC) + p.ADDP(RS, ST, TP) + p.LP(TP, MpOffset, TP) + p.LL(EP, 0, TR) + p.SWAPL(TR, TR) + p.SXLQ(TR, TR) + p.IP(v.Vt, ET) + p.GCALL(F_mapassign_fast64). + A0(ET). + A1(TP). + A2(TR). + R0(WP) } func translate_OP_map_set_enum_safe(p *hir.Builder, v Instr) { - p.ADDP (IP, IC, ET) - p.ADDI (IC, 4, IC) - p.ADDP (RS, ST, TP) - p.LP (TP, MpOffset, EP) - p.LL (ET, 0, TR) - p.SWAPL (TR, TR) - p.SXLQ (TR, TR) - p.SQ (TR, RS, IvOffset) - p.ADDPI (RS, IvOffset, TP) - p.IP (v.Vt, ET) - p.GCALL (F_mapassign). - A0 (ET). - A1 (EP). - A2 (TP). - R0 (WP) + p.ADDP(IP, IC, ET) + p.ADDI(IC, 4, IC) + p.ADDP(RS, ST, TP) + p.LP(TP, MpOffset, EP) + p.LL(ET, 0, TR) + p.SWAPL(TR, TR) + p.SXLQ(TR, TR) + p.SQ(TR, RS, IvOffset) + p.ADDPI(RS, IvOffset, TP) + p.IP(v.Vt, ET) + p.GCALL(F_mapassign). + A0(ET). + A1(EP). + A2(TP). + R0(WP) } func translate_OP_map_set_pointer(p *hir.Builder, v Instr) { - if rt.MapType(v.Vt).IsFastMap() { - translate_OP_map_set_pointer_fast(p, v) - } else { - translate_OP_map_set_pointer_safe(p, v) - } + if rt.MapType(v.Vt).IsFastMap() { + translate_OP_map_set_pointer_fast(p, v) + } else { + translate_OP_map_set_pointer_safe(p, v) + } } func translate_OP_map_set_pointer_fast(p *hir.Builder, v Instr) { - p.ADDP (RS, ST, TP) - p.LP (TP, MpOffset, TP) - p.IP (v.Vt, ET) - p.GCALL (F_mapassign_fast64ptr). - A0 (ET). - A1 (TP). - A2 (WP). - R0 (WP) + p.ADDP(RS, ST, TP) + p.LP(TP, MpOffset, TP) + p.IP(v.Vt, ET) + p.GCALL(F_mapassign_fast64ptr). + A0(ET). + A1(TP). + A2(WP). + R0(WP) } func translate_OP_map_set_pointer_safe(p *hir.Builder, v Instr) { - p.ADDP (RS, ST, TP) - p.LP (TP, MpOffset, EP) - p.SP (WP, RS, PrOffset) - p.IP (v.Vt, ET) - p.ADDPI (RS, PrOffset, TP) - p.GCALL (F_mapassign). - A0 (ET). - A1 (EP). - A2 (TP). - R0 (WP) - p.SP (hir.Pn, RS, PrOffset) + p.ADDP(RS, ST, TP) + p.LP(TP, MpOffset, EP) + p.SP(WP, RS, PrOffset) + p.IP(v.Vt, ET) + p.ADDPI(RS, PrOffset, TP) + p.GCALL(F_mapassign). + A0(ET). + A1(EP). + A2(TP). + R0(WP) + p.SP(hir.Pn, RS, PrOffset) } func translate_OP_list_alloc(p *hir.Builder, v Instr) { - p.ADDP (RS, ST, TP) - p.LQ (TP, NbOffset, TR) - p.SQ (TR, WP, 8) - p.LQ (WP, 16, UR) - p.BNE (TR, hir.Rz, "_alloc_{n}") - p.BNE (UR, hir.Rz, "_done_{n}") - p.IP (&_V_zerovalue, TP) - p.SP (TP, WP, 0) - p.SQ (hir.Rz, WP, 16) - p.JMP ("_done_{n}") - p.Label ("_alloc_{n}") - p.BGEU (UR, TR, "_done_{n}") - p.SQ (TR, WP, 16) - p.IB (1, UR) - p.IP (v.Vt, TP) - p.MULI (TR, int64(v.Vt.Size), TR) - p.GCALL (F_mallocgc). - A0 (TR). - A1 (TP). - A2 (UR). - R0 (TP) - p.SP (TP, WP, 0) - p.Label ("_done_{n}") - p.LP (WP, 0, WP) + p.ADDP(RS, ST, TP) + p.LQ(TP, NbOffset, TR) + p.SQ(TR, WP, 8) + p.LQ(WP, 16, UR) + p.BNE(TR, hir.Rz, "_alloc_{n}") + p.BNE(UR, hir.Rz, "_done_{n}") + p.IP(&_V_zerovalue, TP) + p.SP(TP, WP, 0) + p.SQ(hir.Rz, WP, 16) + p.JMP("_done_{n}") + p.Label("_alloc_{n}") + p.BGEU(UR, TR, "_done_{n}") + p.SQ(TR, WP, 16) + p.IB(1, UR) + p.IP(v.Vt, TP) + p.MULI(TR, int64(v.Vt.Size), TR) + p.GCALL(F_mallocgc). + A0(TR). + A1(TP). + A2(UR). + R0(TP) + p.SP(TP, WP, 0) + p.Label("_done_{n}") + p.LP(WP, 0, WP) } func translate_OP_struct_skip(p *hir.Builder, _ Instr) { - p.ADDPI (RS, SkOffset, TP) - p.LDAQ (ARG_nb, TR) - p.SUB (TR, IC, TR) - p.ADDP (IP, IC, EP) - p.CCALL (C_skip). - A0 (TP). - A1 (EP). - A2 (TR). - A3 (TG). - R0 (TR) - p.BLT (TR, hir.Rz, LB_skip) - p.ADD (IC, TR, IC) + p.ADDPI(RS, SkOffset, TP) + p.LDAQ(ARG_nb, TR) + p.SUB(TR, IC, TR) + p.ADDP(IP, IC, EP) + p.CCALL(C_skip). + A0(TP). + A1(EP). + A2(TR). + A3(TG). + R0(TR) + p.BLT(TR, hir.Rz, LB_skip) + p.ADD(IC, TR, IC) } func translate_OP_struct_ignore(p *hir.Builder, _ Instr) { - p.ADDPI (RS, SkOffset, TP) - p.LDAQ (ARG_nb, TR) - p.SUB (TR, IC, TR) - p.ADDP (IP, IC, EP) - p.IB (int8(defs.T_struct), TG) - p.CCALL (C_skip). - A0 (TP). - A1 (EP). - A2 (TR). - A3 (TG). - R0 (TR) - p.BLT (TR, hir.Rz, LB_skip) - p.ADD (IC, TR, IC) + p.ADDPI(RS, SkOffset, TP) + p.LDAQ(ARG_nb, TR) + p.SUB(TR, IC, TR) + p.ADDP(IP, IC, EP) + p.IB(int8(defs.T_struct), TG) + p.CCALL(C_skip). + A0(TP). + A1(EP). + A2(TR). + A3(TG). + R0(TR) + p.BLT(TR, hir.Rz, LB_skip) + p.ADD(IC, TR, IC) } func translate_OP_struct_bitmap(p *hir.Builder, v Instr) { - buf := newFieldBitmap() - buf.Clear() + buf := newFieldBitmap() + buf.Clear() - /* add all the bits */ - for _, i := range v.IntSeq() { - buf.Append(i) - } + /* add all the bits */ + for _, i := range v.IntSeq() { + buf.Append(i) + } - /* allocate a new bitmap */ - p.GCALL (F_newFieldBitmap).R0(TP) - p.ADDP (RS, ST, EP) - p.SP (TP, EP, FmOffset) + /* allocate a new bitmap */ + p.GCALL(F_newFieldBitmap).R0(TP) + p.ADDP(RS, ST, EP) + p.SP(TP, EP, FmOffset) - /* clear bits of required fields if any */ - for i := int64(0); i < MaxBitmap; i++ { - if buf[i] != 0 { - p.SQ(hir.Rz, TP, i * 8) - } - } + /* clear bits of required fields if any */ + for i := int64(0); i < MaxBitmap; i++ { + if buf[i] != 0 { + p.SQ(hir.Rz, TP, i*8) + } + } - /* release the buffer */ - buf.Clear() - buf.Free() + /* release the buffer */ + buf.Clear() + buf.Free() } func translate_OP_struct_switch(p *hir.Builder, v Instr) { - stab := v.IntSeq() - ptab := make([]string, v.Iv) + stab := v.IntSeq() + ptab := make([]string, v.Iv) - /* convert the switch table */ - for i, to := range stab { - if to >= 0 { - ptab[i] = p.At(to) - } - } + /* convert the switch table */ + for i, to := range stab { + if to >= 0 { + ptab[i] = p.At(to) + } + } - /* load and dispatch the field */ - p.ADDP (IP, IC, EP) - p.ADDI (IC, 2, IC) - p.LW (EP, 0, TR) - p.SWAPW (TR, TR) - p.BSW (TR, ptab) + /* load and dispatch the field */ + p.ADDP(IP, IC, EP) + p.ADDI(IC, 2, IC) + p.LW(EP, 0, TR) + p.SWAPW(TR, TR) + p.BSW(TR, ptab) } func translate_OP_struct_require(p *hir.Builder, v Instr) { - buf := newFieldBitmap() - buf.Clear() - - /* add all the bits */ - for _, i := range v.IntSeq() { - buf.Append(i) - } - - /* load the bitmap */ - p.ADDP (RS, ST, EP) - p.LP (EP, FmOffset, TP) - - /* test mask for each word if any */ - for i := int64(0); i < MaxBitmap; i++ { - if buf[i] != 0 { - p.LQ (TP, i * 8, TR) - p.ANDI (TR, buf[i], TR) - p.XORI (TR, buf[i], TR) - p.IQ (i, UR) - p.IP (v.Vt, ET) - p.BNE (TR, hir.Rz, LB_missing) - } - } - - /* free the bitmap */ - p.SP (hir.Pn, EP, FmOffset) - p.GCALL (F_FieldBitmap_Free).A0(TP) - - /* release the buffer */ - buf.Clear() - buf.Free() + buf := newFieldBitmap() + buf.Clear() + + /* add all the bits */ + for _, i := range v.IntSeq() { + buf.Append(i) + } + + /* load the bitmap */ + p.ADDP(RS, ST, EP) + p.LP(EP, FmOffset, TP) + + /* test mask for each word if any */ + for i := int64(0); i < MaxBitmap; i++ { + if buf[i] != 0 { + p.LQ(TP, i*8, TR) + p.ANDI(TR, buf[i], TR) + p.XORI(TR, buf[i], TR) + p.IQ(i, UR) + p.IP(v.Vt, ET) + p.BNE(TR, hir.Rz, LB_missing) + } + } + + /* free the bitmap */ + p.SP(hir.Pn, EP, FmOffset) + p.GCALL(F_FieldBitmap_Free).A0(TP) + + /* release the buffer */ + buf.Clear() + buf.Free() } func translate_OP_struct_is_stop(p *hir.Builder, v Instr) { - p.BEQ (TG, hir.Rz, p.At(v.To)) + p.BEQ(TG, hir.Rz, p.At(v.To)) } func translate_OP_struct_mark_tag(p *hir.Builder, v Instr) { - p.ADDP (RS, ST, TP) - p.LP (TP, FmOffset, TP) - p.LQ (TP, v.Iv / 64 * 8, TR) - p.BSI (TR, v.Iv % 64, TR) - p.SQ (TR, TP, v.Iv / 64 * 8) + p.ADDP(RS, ST, TP) + p.LP(TP, FmOffset, TP) + p.LQ(TP, v.Iv/64*8, TR) + p.BSI(TR, v.Iv%64, TR) + p.SQ(TR, TP, v.Iv/64*8) } func translate_OP_struct_read_type(p *hir.Builder, _ Instr) { - p.ADDP (IP, IC, EP) - p.ADDI (IC, 1, IC) - p.LB (EP, 0, TG) + p.ADDP(IP, IC, EP) + p.ADDI(IC, 1, IC) + p.LB(EP, 0, TG) } func translate_OP_struct_check_type(p *hir.Builder, v Instr) { - p.IB (int8(v.Tx), TR) - p.BNE (TG, TR, p.At(v.To)) + p.IB(int8(v.Tx), TR) + p.BNE(TG, TR, p.At(v.To)) } func translate_OP_make_state(p *hir.Builder, _ Instr) { - p.IQ (StateMax, TR) - p.BGEU (ST, TR, LB_overflow) - p.ADDP (RS, ST, TP) - p.SP (WP, TP, WpOffset) - p.ADDI (ST, StateSize, ST) + p.IQ(StateMax, TR) + p.BGEU(ST, TR, LB_overflow) + p.ADDP(RS, ST, TP) + p.SP(WP, TP, WpOffset) + p.ADDI(ST, StateSize, ST) } func translate_OP_drop_state(p *hir.Builder, _ Instr) { - p.SUBI (ST, StateSize, ST) - p.ADDP (RS, ST, TP) - p.LP (TP, WpOffset, WP) - p.SP (hir.Pn, TP, WpOffset) + p.SUBI(ST, StateSize, ST) + p.ADDP(RS, ST, TP) + p.LP(TP, WpOffset, WP) + p.SP(hir.Pn, TP, WpOffset) } func translate_OP_construct(p *hir.Builder, v Instr) { - p.IB (1, UR) - p.IP (v.Vt, TP) - p.IQ (int64(v.Vt.Size), TR) - p.GCALL (F_mallocgc). - A0 (TR). - A1 (TP). - A2 (UR). - R0 (WP) + p.IB(1, UR) + p.IP(v.Vt, TP) + p.IQ(int64(v.Vt.Size), TR) + p.GCALL(F_mallocgc). + A0(TR). + A1(TP). + A2(UR). + R0(WP) } func translate_OP_initialize(p *hir.Builder, v Instr) { - p.GCALL (addInitFn(v.Fn)). - A0 (WP) + p.GCALL(addInitFn(v.Fn)). + A0(WP) } func translate_OP_defer(p *hir.Builder, v Instr) { - p.IP (v.Vt, TP) - p.LDAQ (ARG_nb, TR) - p.GCALL (F_decode). - A0 (TP). - A1 (IP). - A2 (TR). - A3 (IC). - A4 (WP). - A5 (RS). - A6 (ST). - R0 (IC). - R1 (ET). - R2 (EP) - p.BNEP (ET, hir.Pn, LB_error) + p.IP(v.Vt, TP) + p.LDAQ(ARG_nb, TR) + p.GCALL(F_decode). + A0(TP). + A1(IP). + A2(TR). + A3(IC). + A4(WP). + A5(RS). + A6(ST). + R0(IC). + R1(ET). + R2(EP) + p.BNEP(ET, hir.Pn, LB_error) } func translate_OP_goto(p *hir.Builder, v Instr) { - p.JMP (p.At(v.To)) + p.JMP(p.At(v.To)) } func translate_OP_halt(p *hir.Builder, _ Instr) { - p.JMP (LB_halt) + p.JMP(LB_halt) } diff --git a/internal/binary/decoder/translator_test.go b/internal/binary/decoder/translator_test.go index ee8a369..7df9049 100644 --- a/internal/binary/decoder/translator_test.go +++ b/internal/binary/decoder/translator_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,38 +17,38 @@ package decoder import ( - `reflect` - `testing` + "reflect" + "testing" - `github.com/stretchr/testify/require` + "github.com/stretchr/testify/require" ) type TranslatorTestStruct struct { - A bool `frugal:"0,default,bool"` - B int8 `frugal:"1,required,i8"` - C float64 `frugal:"2,default,double"` - D int16 `frugal:"3,default,i16"` - E int32 `frugal:"4,default,i32"` - F int64 `frugal:"5,default,i64"` - G string `frugal:"6,default,string"` - H []byte `frugal:"7,default,binary"` - I []int32 `frugal:"8,required,list"` - J map[string]string `frugal:"9,default,map"` - K map[string]*TranslatorTestStruct `frugal:"65,required,map"` + A bool `frugal:"0,default,bool"` + B int8 `frugal:"1,required,i8"` + C float64 `frugal:"2,default,double"` + D int16 `frugal:"3,default,i16"` + E int32 `frugal:"4,default,i32"` + F int64 `frugal:"5,default,i64"` + G string `frugal:"6,default,string"` + H []byte `frugal:"7,default,binary"` + I []int32 `frugal:"8,required,list"` + J map[string]string `frugal:"9,default,map"` + K map[string]*TranslatorTestStruct `frugal:"65,required,map"` } func TestTranslator_Translate(t *testing.T) { - var v TranslatorTestStruct - p, err := CreateCompiler().Compile(reflect.TypeOf(v)) - require.NoError(t, err) - tr := Translate(p) - println(tr.Disassemble()) + var v TranslatorTestStruct + p, err := CreateCompiler().Compile(reflect.TypeOf(v)) + require.NoError(t, err) + tr := Translate(p) + println(tr.Disassemble()) } func TestTranslator_NoCopyString(t *testing.T) { - var v NoCopyStringTestStruct - p, err := CreateCompiler().Compile(reflect.TypeOf(v)) - require.NoError(t, err) - tr := Translate(p) - println(tr.Disassemble()) + var v NoCopyStringTestStruct + p, err := CreateCompiler().Compile(reflect.TypeOf(v)) + require.NoError(t, err) + tr := Translate(p) + println(tr.Disassemble()) } diff --git a/internal/binary/defs/defaults.go b/internal/binary/defs/defaults.go index bd7858f..ee05a51 100644 --- a/internal/binary/defs/defaults.go +++ b/internal/binary/defs/defaults.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,37 +17,37 @@ package defs import ( - `fmt` - `reflect` - `unsafe` + "fmt" + "reflect" + "unsafe" ) type DefaultInitializer interface { - InitDefault() + InitDefault() } func GetDefaultInitializer(vt reflect.Type) (unsafe.Pointer, error) { - var ok bool - var mt reflect.Method + var ok bool + var mt reflect.Method - /* element type and pointer type */ - et := vt - pt := reflect.PtrTo(vt) + /* element type and pointer type */ + et := vt + pt := reflect.PtrTo(vt) - /* dereference the type */ - for et.Kind() == reflect.Ptr { - pt = et - et = et.Elem() - } + /* dereference the type */ + for et.Kind() == reflect.Ptr { + pt = et + et = et.Elem() + } - /* find the default initializer method */ - if mt, ok = et.MethodByName("InitDefault"); ok { - return nil, fmt.Errorf("implementation of `InitDefault()` must have a pointer receiver: %s", mt.Type) - } else if mt, ok = pt.MethodByName("InitDefault"); !ok { - return nil, nil - } else if mt.Type.NumIn() != 1 || mt.Type.NumOut() != 0 { - return nil, fmt.Errorf("invalid implementation of `InitDefault()`: %s", mt.Type) - } else { - return *(*[2]*unsafe.Pointer)(unsafe.Pointer(&mt.Func))[1], nil - } + /* find the default initializer method */ + if mt, ok = et.MethodByName("InitDefault"); ok { + return nil, fmt.Errorf("implementation of `InitDefault()` must have a pointer receiver: %s", mt.Type) + } else if mt, ok = pt.MethodByName("InitDefault"); !ok { + return nil, nil + } else if mt.Type.NumIn() != 1 || mt.Type.NumOut() != 0 { + return nil, fmt.Errorf("invalid implementation of `InitDefault()`: %s", mt.Type) + } else { + return *(*[2]*unsafe.Pointer)(unsafe.Pointer(&mt.Func))[1], nil + } } diff --git a/internal/binary/defs/defaults_test.go b/internal/binary/defs/defaults_test.go index 1f613a4..3f8bb13 100644 --- a/internal/binary/defs/defaults_test.go +++ b/internal/binary/defs/defaults_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,34 +17,34 @@ package defs import ( - `reflect` - `testing` - `unsafe` + "reflect" + "testing" + "unsafe" - `github.com/davecgh/go-spew/spew` - `github.com/stretchr/testify/require` + "github.com/davecgh/go-spew/spew" + "github.com/stretchr/testify/require" ) type CtorTestStruct struct { - X int - Y int - Z int + X int + Y int + Z int } func (self *CtorTestStruct) InitDefault() { - *self = CtorTestStruct { - X: 1, - Y: 2, - Z: 3, - } + *self = CtorTestStruct{ + X: 1, + Y: 2, + Z: 3, + } } func TestDefaults_Resolve(t *testing.T) { - fp := (*CtorTestStruct).InitDefault - fa := **(**unsafe.Pointer)(unsafe.Pointer(&fp)) - spew.Dump(fa) - fn, err := GetDefaultInitializer(reflect.TypeOf(CtorTestStruct{})) - require.NoError(t, err) - spew.Dump(fn) - require.Equal(t, fa, fn) + fp := (*CtorTestStruct).InitDefault + fa := **(**unsafe.Pointer)(unsafe.Pointer(&fp)) + spew.Dump(fa) + fn, err := GetDefaultInitializer(reflect.TypeOf(CtorTestStruct{})) + require.NoError(t, err) + spew.Dump(fn) + require.Equal(t, fa, fn) } diff --git a/internal/binary/defs/resolver.go b/internal/binary/defs/resolver.go index 0ba433e..34b50b8 100644 --- a/internal/binary/defs/resolver.go +++ b/internal/binary/defs/resolver.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,219 +17,229 @@ package defs import ( - `fmt` - `math/bits` - `reflect` - `sort` - `strconv` - `strings` - `sync` + "fmt" + "math/bits" + "reflect" + "sort" + "strconv" + "strings" + "sync" ) type ( - Options uint8 + Options uint8 Requiredness uint8 ) const ( - NoCopy Options = 1 << iota + NoCopy Options = 1 << iota ) const ( - Default Requiredness = iota - Required - Optional + Default Requiredness = iota + Required + Optional ) func (self Options) String() string { - nb := bits.OnesCount8(uint8(self)) - ret := make([]string, 0, nb) - - /* check for "nocopy" option */ - if self & NoCopy != 0 { - ret = append(ret, "nocopy") - } - - /* join them together */ - return fmt.Sprintf( - "{%s}", - strings.Join(ret, ","), - ) + nb := bits.OnesCount8(uint8(self)) + ret := make([]string, 0, nb) + + /* check for "nocopy" option */ + if self&NoCopy != 0 { + ret = append(ret, "nocopy") + } + + /* join them together */ + return fmt.Sprintf( + "{%s}", + strings.Join(ret, ","), + ) } func (self Requiredness) String() string { - switch self { - case Default : return "default" - case Required : return "required" - case Optional : return "optional" - default : panic("unreachable") - } + switch self { + case Default: + return "default" + case Required: + return "required" + case Optional: + return "optional" + default: + panic("unreachable") + } } type Field struct { - F int - ID uint16 - Type *Type - Opts Options - Spec Requiredness - Default reflect.Value + F int + ID uint16 + Type *Type + Opts Options + Spec Requiredness + Default reflect.Value } var ( - fieldsLock = new(sync.RWMutex) - fieldsCache = make(map[reflect.Type][]Field) + fieldsLock = new(sync.RWMutex) + fieldsCache = make(map[reflect.Type][]Field) ) func ResolveFields(vt reflect.Type) ([]Field, error) { - var ok bool - var ex error - var fv []Field - - /* attempt to find in cache */ - fieldsLock.RLock() - fv, ok = fieldsCache[vt] - fieldsLock.RUnlock() - - /* check if it exists */ - if ok { - return fv, nil - } - - /* retry with write lock */ - fieldsLock.Lock() - defer fieldsLock.Unlock() - - /* try again */ - if fv, ok = fieldsCache[vt]; ok { - return fv, nil - } - - /* still not found, do the actual resolving */ - if fv, ex = doResolveFields(vt); ex != nil { - return nil, ex - } - - /* update cache */ - fieldsCache[vt] = fv - return fv, nil + var ok bool + var ex error + var fv []Field + + /* attempt to find in cache */ + fieldsLock.RLock() + fv, ok = fieldsCache[vt] + fieldsLock.RUnlock() + + /* check if it exists */ + if ok { + return fv, nil + } + + /* retry with write lock */ + fieldsLock.Lock() + defer fieldsLock.Unlock() + + /* try again */ + if fv, ok = fieldsCache[vt]; ok { + return fv, nil + } + + /* still not found, do the actual resolving */ + if fv, ex = doResolveFields(vt); ex != nil { + return nil, ex + } + + /* update cache */ + fieldsCache[vt] = fv + return fv, nil } func doResolveFields(vt reflect.Type) ([]Field, error) { - var err error - var ret []Field - var mem reflect.Value - - /* field ID map and default values */ - val := reflect.New(vt) - ids := make(map[uint64]struct{}, vt.NumField()) - - /* check for default values */ - if def, ok := val.Interface().(DefaultInitializer); ok { - mem = val.Elem() - def.InitDefault() - } - - /* traverse all the fields */ - for i := 0; i < vt.NumField(); i++ { - var ok bool - var pt *Type - var id uint64 - var tv string - var fv Options - var ft []string - var rx Requiredness - var rv reflect.Value - var sf reflect.StructField - - /* extract the field, ignore anonymous or private fields */ - if sf = vt.Field(i); sf.Anonymous || sf.PkgPath != "" { - continue - } - - /* ignore fields that does not declare the "frugal" tag */ - if tv, ok = sf.Tag.Lookup("frugal"); !ok { - continue - } - - /* must have at least 2 fields: ID and Requiredness */ - if ft = strings.Split(tv, ","); len(ft) < 2 { - return nil, fmt.Errorf("invalid tag for field %s.%s", vt, sf.Name) - } - - /* parse the field index */ - if id, err = strconv.ParseUint(strings.TrimSpace(ft[0]), 10, 16); err != nil { - return nil, fmt.Errorf("invalid field number for field %s.%s: %w", vt, sf.Name, err) - } - - /* convert the requiredness of this field */ - switch strings.TrimSpace(ft[1]) { - case "default" : rx = Default - case "required" : rx = Required - case "optional" : rx = Optional - default : return nil, fmt.Errorf("invalid requiredness for field %s.%s", vt, sf.Name) - } - - /* check for duplicates */ - if _, ok = ids[id]; !ok { - ids[id] = struct{}{} - } else { - return nil, fmt.Errorf("duplicated field ID %d for field %s.%s", id, vt, sf.Name) - } - - /* types and other options are optional */ - if len(ft) == 2 { - tv, ft = "", nil - } else { - tv, ft = strings.TrimSpace(ft[2]), ft[3:] - } - - /* parse the type descriptor */ - if pt, err = ParseType(sf.Type, tv); err != nil { - return nil, fmt.Errorf("cannot parse type descriptor: %w", err) - } - - /* only optional fields or structs can be pointers */ - if rx != Optional && pt.T == T_pointer && pt.V.T != T_struct { - return nil, fmt.Errorf("only optional fields or structs can be pointers, not %s: %s.%s", sf.Type, vt, sf.Name) - } - - /* scan for the options */ - for _, opt := range ft { - switch opt { - default: { - return nil, fmt.Errorf("invalid option: %s", opt) - } - - /* "nocopy" option enables zero-copy string decoding */ - case "nocopy": { - if pt.Tag() != T_string { - return nil, fmt.Errorf(`"nocopy" is only applicable to "string" and "binary" types, not %s`, pt) - } else if fv & NoCopy != 0 { - return nil, fmt.Errorf(`duplicated option "nocopy" for field %s.%s`, vt, sf.Name) - } else { - fv |= NoCopy - } - } - } - } - - /* get the default value if any */ - if mem.IsValid() { - rv = mem.FieldByIndex(sf.Index) - } - - /* add to result */ - ret = append(ret, Field { - F : int(sf.Offset), - ID : uint16(id), - Type : pt, - Opts : fv, - Spec : rx, - Default : rv, - }) - } - - /* sort the field by ID */ - sort.Slice(ret, func(i, j int) bool { return ret[i].ID < ret[j].ID }) - return ret, nil + var err error + var ret []Field + var mem reflect.Value + + /* field ID map and default values */ + val := reflect.New(vt) + ids := make(map[uint64]struct{}, vt.NumField()) + + /* check for default values */ + if def, ok := val.Interface().(DefaultInitializer); ok { + mem = val.Elem() + def.InitDefault() + } + + /* traverse all the fields */ + for i := 0; i < vt.NumField(); i++ { + var ok bool + var pt *Type + var id uint64 + var tv string + var fv Options + var ft []string + var rx Requiredness + var rv reflect.Value + var sf reflect.StructField + + /* extract the field, ignore anonymous or private fields */ + if sf = vt.Field(i); sf.Anonymous || sf.PkgPath != "" { + continue + } + + /* ignore fields that does not declare the "frugal" tag */ + if tv, ok = sf.Tag.Lookup("frugal"); !ok { + continue + } + + /* must have at least 2 fields: ID and Requiredness */ + if ft = strings.Split(tv, ","); len(ft) < 2 { + return nil, fmt.Errorf("invalid tag for field %s.%s", vt, sf.Name) + } + + /* parse the field index */ + if id, err = strconv.ParseUint(strings.TrimSpace(ft[0]), 10, 16); err != nil { + return nil, fmt.Errorf("invalid field number for field %s.%s: %w", vt, sf.Name, err) + } + + /* convert the requiredness of this field */ + switch strings.TrimSpace(ft[1]) { + case "default": + rx = Default + case "required": + rx = Required + case "optional": + rx = Optional + default: + return nil, fmt.Errorf("invalid requiredness for field %s.%s", vt, sf.Name) + } + + /* check for duplicates */ + if _, ok = ids[id]; !ok { + ids[id] = struct{}{} + } else { + return nil, fmt.Errorf("duplicated field ID %d for field %s.%s", id, vt, sf.Name) + } + + /* types and other options are optional */ + if len(ft) == 2 { + tv, ft = "", nil + } else { + tv, ft = strings.TrimSpace(ft[2]), ft[3:] + } + + /* parse the type descriptor */ + if pt, err = ParseType(sf.Type, tv); err != nil { + return nil, fmt.Errorf("cannot parse type descriptor: %w", err) + } + + /* only optional fields or structs can be pointers */ + if rx != Optional && pt.T == T_pointer && pt.V.T != T_struct { + return nil, fmt.Errorf("only optional fields or structs can be pointers, not %s: %s.%s", sf.Type, vt, sf.Name) + } + + /* scan for the options */ + for _, opt := range ft { + switch opt { + default: + { + return nil, fmt.Errorf("invalid option: %s", opt) + } + + /* "nocopy" option enables zero-copy string decoding */ + case "nocopy": + { + if pt.Tag() != T_string { + return nil, fmt.Errorf(`"nocopy" is only applicable to "string" and "binary" types, not %s`, pt) + } else if fv&NoCopy != 0 { + return nil, fmt.Errorf(`duplicated option "nocopy" for field %s.%s`, vt, sf.Name) + } else { + fv |= NoCopy + } + } + } + } + + /* get the default value if any */ + if mem.IsValid() { + rv = mem.FieldByIndex(sf.Index) + } + + /* add to result */ + ret = append(ret, Field{ + F: int(sf.Offset), + ID: uint16(id), + Type: pt, + Opts: fv, + Spec: rx, + Default: rv, + }) + } + + /* sort the field by ID */ + sort.Slice(ret, func(i, j int) bool { return ret[i].ID < ret[j].ID }) + return ret, nil } diff --git a/internal/binary/defs/resolver_test.go b/internal/binary/defs/resolver_test.go index 68b142a..27a0c6c 100644 --- a/internal/binary/defs/resolver_test.go +++ b/internal/binary/defs/resolver_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,26 +17,26 @@ package defs import ( - `reflect` - `testing` + "reflect" + "testing" - `github.com/davecgh/go-spew/spew` - `github.com/stretchr/testify/require` + "github.com/davecgh/go-spew/spew" + "github.com/stretchr/testify/require" ) type NoCopyStringFields struct { - NormalString string `frugal:"1,default,string"` - NoCopyString string `frugal:"2,default,string,nocopy"` - TypelessString string `frugal:"3,default"` - TypelessString2 string `frugal:"4,default,"` - NoCopyTypelessString string `frugal:"5,default,,nocopy"` + NormalString string `frugal:"1,default,string"` + NoCopyString string `frugal:"2,default,string,nocopy"` + TypelessString string `frugal:"3,default"` + TypelessString2 string `frugal:"4,default,"` + NoCopyTypelessString string `frugal:"5,default,,nocopy"` } func TestResolver_StringOptions(t *testing.T) { - var vv NoCopyStringFields - ret, err := ResolveFields(reflect.TypeOf(vv)) - require.NoError(t, err) - spew.Config.SortKeys = true - spew.Config.DisablePointerMethods = true - spew.Dump(ret) + var vv NoCopyStringFields + ret, err := ResolveFields(reflect.TypeOf(vv)) + require.NoError(t, err) + spew.Config.SortKeys = true + spew.Config.DisablePointerMethods = true + spew.Dump(ret) } diff --git a/internal/binary/defs/sizes.go b/internal/binary/defs/sizes.go index 7ca1c4b..9918b16 100644 --- a/internal/binary/defs/sizes.go +++ b/internal/binary/defs/sizes.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,53 +17,66 @@ package defs import ( - `reflect` + "reflect" ) const ( - IntSize = 4 << (^uint(0) >> 63) - StackSize = 1024 + IntSize = 4 << (^uint(0) >> 63) + StackSize = 1024 ) func GetSize(vt reflect.Type) int { - switch vt.Kind() { - case reflect.Bool : return 1 - case reflect.Int : return IntSize - case reflect.Int8 : return 1 - case reflect.Int16 : return 2 - case reflect.Int32 : return 4 - case reflect.Int64 : return measureInt64(vt) - case reflect.Float64 : return 8 - case reflect.Map : return -1 - case reflect.Ptr : return -1 - case reflect.Slice : return -1 - case reflect.String : return -1 - case reflect.Struct : return measureStruct(vt) - default : panic("unsupported type by Thrift") - } + switch vt.Kind() { + case reflect.Bool: + return 1 + case reflect.Int: + return IntSize + case reflect.Int8: + return 1 + case reflect.Int16: + return 2 + case reflect.Int32: + return 4 + case reflect.Int64: + return measureInt64(vt) + case reflect.Float64: + return 8 + case reflect.Map: + return -1 + case reflect.Ptr: + return -1 + case reflect.Slice: + return -1 + case reflect.String: + return -1 + case reflect.Struct: + return measureStruct(vt) + default: + panic("unsupported type by Thrift") + } } func measureInt64(vt reflect.Type) int { - if vt == i64type { - return 8 - } else { - return 4 - } + if vt == i64type { + return 8 + } else { + return 4 + } } func measureStruct(vt reflect.Type) int { - var fs int - var rs int + var fs int + var rs int - /* measure each field, plus the 3-byte field header */ - for i := 0; i < vt.NumField(); i++ { - if fs = GetSize(vt.Field(i).Type); fs > 0 { - rs += fs + 3 - } else { - return -1 - } - } + /* measure each field, plus the 3-byte field header */ + for i := 0; i < vt.NumField(); i++ { + if fs = GetSize(vt.Field(i).Type); fs > 0 { + rs += fs + 3 + } else { + return -1 + } + } - /* all fields have fixed size, plus the STOP field */ - return rs + 1 + /* all fields have fixed size, plus the STOP field */ + return rs + 1 } diff --git a/internal/binary/defs/types.go b/internal/binary/defs/types.go index 8150477..0a63963 100644 --- a/internal/binary/defs/types.go +++ b/internal/binary/defs/types.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,439 +17,499 @@ package defs import ( - `fmt` - `reflect` - `strings` - `sync` - `unicode` + "fmt" + "reflect" + "strings" + "sync" + "unicode" - `github.com/cloudwego/frugal/internal/utils` + "github.com/cloudwego/frugal/internal/utils" ) type Tag uint8 const ( - T_bool Tag = 2 - T_i8 Tag = 3 - T_double Tag = 4 - T_i16 Tag = 6 - T_i32 Tag = 8 - T_i64 Tag = 10 - T_string Tag = 11 - T_struct Tag = 12 - T_map Tag = 13 - T_set Tag = 14 - T_list Tag = 15 - T_enum Tag = 0x80 - T_binary Tag = 0x81 - T_pointer Tag = 0x82 + T_bool Tag = 2 + T_i8 Tag = 3 + T_double Tag = 4 + T_i16 Tag = 6 + T_i32 Tag = 8 + T_i64 Tag = 10 + T_string Tag = 11 + T_struct Tag = 12 + T_map Tag = 13 + T_set Tag = 14 + T_list Tag = 15 + T_enum Tag = 0x80 + T_binary Tag = 0x81 + T_pointer Tag = 0x82 ) -var wireTags = [256]bool { - T_bool : true, - T_i8 : true, - T_double : true, - T_i16 : true, - T_i32 : true, - T_i64 : true, - T_string : true, - T_struct : true, - T_map : true, - T_set : true, - T_list : true, +var wireTags = [256]bool{ + T_bool: true, + T_i8: true, + T_double: true, + T_i16: true, + T_i32: true, + T_i64: true, + T_string: true, + T_struct: true, + T_map: true, + T_set: true, + T_list: true, } -var keywordTab = [256]string { - T_bool : "bool", - T_i8 : "i8 byte", - T_double : "double", - T_i16 : "i16", - T_i32 : "i32", - T_i64 : "i64", - T_string : "string", - T_binary : "binary", - T_struct : "struct", - T_map : "map", +var keywordTab = [256]string{ + T_bool: "bool", + T_i8: "i8 byte", + T_double: "double", + T_i16: "i16", + T_i32: "i32", + T_i64: "i64", + T_string: "string", + T_binary: "binary", + T_struct: "struct", + T_map: "map", } var ( - i64type = reflect.TypeOf(int64(0)) + i64type = reflect.TypeOf(int64(0)) ) func T_int() Tag { - switch IntSize { - case 4 : return T_i32 - case 8 : return T_i64 - default : panic("invalid int size") - } + switch IntSize { + case 4: + return T_i32 + case 8: + return T_i64 + default: + panic("invalid int size") + } } func (self Tag) IsWireTag() bool { - return wireTags[self] + return wireTags[self] } type Type struct { - T Tag - K *Type - V *Type - S reflect.Type + T Tag + K *Type + V *Type + S reflect.Type } var ( - typePool sync.Pool + typePool sync.Pool ) func newType() *Type { - if v := typePool.Get(); v == nil { - return new(Type) - } else { - return resetType(v.(*Type)) - } + if v := typePool.Get(); v == nil { + return new(Type) + } else { + return resetType(v.(*Type)) + } } func resetType(p *Type) *Type { - *p = Type{} - return p + *p = Type{} + return p } func (self *Type) Tag() Tag { - switch self.T { - case T_enum : return T_i32 - case T_binary : return T_string - case T_pointer : return self.V.Tag() - default : return self.T - } + switch self.T { + case T_enum: + return T_i32 + case T_binary: + return T_string + case T_pointer: + return self.V.Tag() + default: + return self.T + } } func (self *Type) Free() { - typePool.Put(self) + typePool.Put(self) } func (self *Type) String() string { - switch self.T { - case T_bool : return "bool" - case T_i8 : return "i8" - case T_double : return "double" - case T_i16 : return "i16" - case T_i32 : return "i32" - case T_i64 : return "i64" - case T_string : return "string" - case T_struct : return self.S.Name() - case T_map : return fmt.Sprintf("map<%s:%s>", self.K.String(), self.V.String()) - case T_set : return fmt.Sprintf("set<%s>", self.V.String()) - case T_list : return fmt.Sprintf("list<%s>", self.V.String()) - case T_enum : return "enum" - case T_binary : return "binary" - case T_pointer : return "*" + self.V.String() - default : return fmt.Sprintf("Type(Tag(%d))", self.T) - } + switch self.T { + case T_bool: + return "bool" + case T_i8: + return "i8" + case T_double: + return "double" + case T_i16: + return "i16" + case T_i32: + return "i32" + case T_i64: + return "i64" + case T_string: + return "string" + case T_struct: + return self.S.Name() + case T_map: + return fmt.Sprintf("map<%s:%s>", self.K.String(), self.V.String()) + case T_set: + return fmt.Sprintf("set<%s>", self.V.String()) + case T_list: + return fmt.Sprintf("list<%s>", self.V.String()) + case T_enum: + return "enum" + case T_binary: + return "binary" + case T_pointer: + return "*" + self.V.String() + default: + return fmt.Sprintf("Type(Tag(%d))", self.T) + } } func (self *Type) IsKeyType() bool { - switch self.T { - case T_bool : return true - case T_i8 : return true - case T_double : return true - case T_i16 : return true - case T_i32 : return true - case T_i64 : return true - case T_string : return true - case T_enum : return true - case T_pointer : return self.V.T == T_struct - default : return false - } + switch self.T { + case T_bool: + return true + case T_i8: + return true + case T_double: + return true + case T_i16: + return true + case T_i32: + return true + case T_i64: + return true + case T_string: + return true + case T_enum: + return true + case T_pointer: + return self.V.T == T_struct + default: + return false + } } func (self *Type) IsValueType() bool { - return self.T != T_pointer || self.V.T == T_struct + return self.T != T_pointer || self.V.T == T_struct } func (self *Type) IsSimpleType() bool { - switch self.T { - case T_bool : return true - case T_i8 : return true - case T_double : return true - case T_i16 : return true - case T_i32 : return true - case T_i64 : return true - case T_string : return true - case T_enum : return true - default : return false - } + switch self.T { + case T_bool: + return true + case T_i8: + return true + case T_double: + return true + case T_i16: + return true + case T_i32: + return true + case T_i64: + return true + case T_string: + return true + case T_enum: + return true + default: + return false + } } func ParseType(vt reflect.Type, def string) (*Type, error) { - var i int - return doParseType(vt, def, &i, true) + var i int + return doParseType(vt, def, &i, true) } func isident(c byte) bool { - return isident0(c) || c >= '0' && c <= '9' + return isident0(c) || c >= '0' && c <= '9' } func isident0(c byte) bool { - return c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' + return c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' } func readToken(src string, i *int, eofok bool) (string, error) { - p := *i - n := len(src) - - /* skip the spaces */ - for p < n && unicode.IsSpace(rune(src[p])) { - p++ - } - - /* check for EOF */ - if p == n { - if eofok { - return "", nil - } else { - return "", utils.ESyntax(p, src, "unexpected EOF") - } - } - - /* skip the character */ - q := p - p++ - - /* check for identifiers */ - if isident0(src[q]) { - for p < n && isident(src[p]) { - p++ - } - } - - /* slice the source */ - *i = p - return src[q:p], nil + p := *i + n := len(src) + + /* skip the spaces */ + for p < n && unicode.IsSpace(rune(src[p])) { + p++ + } + + /* check for EOF */ + if p == n { + if eofok { + return "", nil + } else { + return "", utils.ESyntax(p, src, "unexpected EOF") + } + } + + /* skip the character */ + q := p + p++ + + /* check for identifiers */ + if isident0(src[q]) { + for p < n && isident(src[p]) { + p++ + } + } + + /* slice the source */ + *i = p + return src[q:p], nil } func mkMistyped(pos int, src string, tv string, tag Tag, vt reflect.Type) utils.SyntaxError { - if tag != T_struct { - return utils.ESyntax(pos, src, fmt.Sprintf("type mismatch, %s expected, got %s", keywordTab[tag], tv)) - } else { - return utils.ESyntax(pos, src, fmt.Sprintf("struct name mismatch, %s expected, got %s", vt.Name(), tv)) - } + if tag != T_struct { + return utils.ESyntax(pos, src, fmt.Sprintf("type mismatch, %s expected, got %s", keywordTab[tag], tv)) + } else { + return utils.ESyntax(pos, src, fmt.Sprintf("struct name mismatch, %s expected, got %s", vt.Name(), tv)) + } } func doParseType(vt reflect.Type, def string, i *int, allowPtrs bool) (*Type, error) { - var tag Tag - var err error - var ret *Type - - /* dereference the pointer if possible */ - if ret = newType(); vt.Kind() == reflect.Ptr { - ret.S = vt - ret.T = T_pointer - - /* prohibit nested pointers */ - if !allowPtrs { - return nil, utils.EType(ret.V.S, "nested pointer is not allowed") - } - - /* parse the pointer element recursively */ - if ret.V, err = doParseType(vt.Elem(), def, i, false); err != nil { - return nil, err - } else { - return ret, nil - } - } - - /* check for value kind */ - switch vt.Kind() { - case reflect.Bool : tag = T_bool - case reflect.Int : tag = T_int() - case reflect.Int8 : tag = T_i8 - case reflect.Int16 : tag = T_i16 - case reflect.Int32 : tag = T_i32 - case reflect.Int64 : tag = T_i64 - case reflect.Uint : return nil, utils.EUseOther(vt, "int") - case reflect.Uint8 : return nil, utils.EUseOther(vt, "int8") - case reflect.Uint16 : return nil, utils.EUseOther(vt, "int16") - case reflect.Uint32 : return nil, utils.EUseOther(vt, "int32") - case reflect.Uint64 : return nil, utils.EUseOther(vt, "int64") - case reflect.Float32 : return nil, utils.EUseOther(vt, "float64") - case reflect.Float64 : tag = T_double - case reflect.Array : return nil, utils.EUseOther(vt, "[]" + vt.Elem().String()) - case reflect.Map : tag = T_map - case reflect.Slice : break - case reflect.String : tag = T_string - case reflect.Struct : tag = T_struct - default : return nil, utils.EType(vt, "unsupported type") - } - - /* it's a slice, check for byte slice */ - if tag == 0 { - if et := vt.Elem(); utils.IsByteType(et) { - tag = T_binary - } else if def == "" { - return nil, utils.ESetList(*i, def, et) - } else { - return doParseSlice(vt, et, def, i, ret) - } - } - - /* match the type if any */ - if def != "" { - if tv, et := readToken(def, i, false); et != nil { - return nil, et - } else if !strings.Contains(keywordTab[tag], tv) { - if !isident0(tv[0]) { - return nil, mkMistyped(*i - len(tv), def, tv, tag, vt) - } else if ok, ex := doMatchStruct(vt, def, i, &tv); ex != nil { - return nil, ex - } else if !ok { - return nil, mkMistyped(*i - len(tv), def, tv, tag, vt) - } else if tag == T_i64 && vt != i64type { - tag = T_enum - } - } - } - - /* simple types */ - if tag != T_map { - ret.S = vt - ret.T = tag - return ret, nil - } - - /* map begin */ - if def != "" { - if tk, et := readToken(def, i, false); et != nil { - return nil, et - } else if tk != "<" { - return nil, utils.ESyntax(*i - len(tk), def, "'<' expected") - } - } - - /* parse the key type */ - if ret.K, err = doParseType(vt.Key(), def, i, true); err != nil { - return nil, err - } - - /* validate map key */ - if !ret.K.IsKeyType() { - return nil, utils.EType(ret.K.S, "not a valid map key type") - } - - /* key-value delimiter */ - if def != "" { - if tk, et := readToken(def, i, false); et != nil { - return nil, et - } else if tk != ":" { - return nil, utils.ESyntax(*i - len(tk), def, "':' expected") - } - } - - /* parse the value type */ - if ret.V, err = doParseType(vt.Elem(), def, i, true); err != nil { - return nil, err - } - - /* map end */ - if def != "" { - if tk, et := readToken(def, i, false); et != nil { - return nil, et - } else if tk != ">" { - return nil, utils.ESyntax(*i - len(tk), def, "'>' expected") - } - } - - /* check for list elements */ - if !ret.V.IsValueType() { - return nil, utils.EType(ret.V.S, "non-struct pointers are not valid map value types") - } - - /* set the type tag */ - ret.S = vt - ret.T = T_map - return ret, nil + var tag Tag + var err error + var ret *Type + + /* dereference the pointer if possible */ + if ret = newType(); vt.Kind() == reflect.Ptr { + ret.S = vt + ret.T = T_pointer + + /* prohibit nested pointers */ + if !allowPtrs { + return nil, utils.EType(ret.V.S, "nested pointer is not allowed") + } + + /* parse the pointer element recursively */ + if ret.V, err = doParseType(vt.Elem(), def, i, false); err != nil { + return nil, err + } else { + return ret, nil + } + } + + /* check for value kind */ + switch vt.Kind() { + case reflect.Bool: + tag = T_bool + case reflect.Int: + tag = T_int() + case reflect.Int8: + tag = T_i8 + case reflect.Int16: + tag = T_i16 + case reflect.Int32: + tag = T_i32 + case reflect.Int64: + tag = T_i64 + case reflect.Uint: + return nil, utils.EUseOther(vt, "int") + case reflect.Uint8: + return nil, utils.EUseOther(vt, "int8") + case reflect.Uint16: + return nil, utils.EUseOther(vt, "int16") + case reflect.Uint32: + return nil, utils.EUseOther(vt, "int32") + case reflect.Uint64: + return nil, utils.EUseOther(vt, "int64") + case reflect.Float32: + return nil, utils.EUseOther(vt, "float64") + case reflect.Float64: + tag = T_double + case reflect.Array: + return nil, utils.EUseOther(vt, "[]"+vt.Elem().String()) + case reflect.Map: + tag = T_map + case reflect.Slice: + break + case reflect.String: + tag = T_string + case reflect.Struct: + tag = T_struct + default: + return nil, utils.EType(vt, "unsupported type") + } + + /* it's a slice, check for byte slice */ + if tag == 0 { + if et := vt.Elem(); utils.IsByteType(et) { + tag = T_binary + } else if def == "" { + return nil, utils.ESetList(*i, def, et) + } else { + return doParseSlice(vt, et, def, i, ret) + } + } + + /* match the type if any */ + if def != "" { + if tv, et := readToken(def, i, false); et != nil { + return nil, et + } else if !strings.Contains(keywordTab[tag], tv) { + if !isident0(tv[0]) { + return nil, mkMistyped(*i-len(tv), def, tv, tag, vt) + } else if ok, ex := doMatchStruct(vt, def, i, &tv); ex != nil { + return nil, ex + } else if !ok { + return nil, mkMistyped(*i-len(tv), def, tv, tag, vt) + } else if tag == T_i64 && vt != i64type { + tag = T_enum + } + } + } + + /* simple types */ + if tag != T_map { + ret.S = vt + ret.T = tag + return ret, nil + } + + /* map begin */ + if def != "" { + if tk, et := readToken(def, i, false); et != nil { + return nil, et + } else if tk != "<" { + return nil, utils.ESyntax(*i-len(tk), def, "'<' expected") + } + } + + /* parse the key type */ + if ret.K, err = doParseType(vt.Key(), def, i, true); err != nil { + return nil, err + } + + /* validate map key */ + if !ret.K.IsKeyType() { + return nil, utils.EType(ret.K.S, "not a valid map key type") + } + + /* key-value delimiter */ + if def != "" { + if tk, et := readToken(def, i, false); et != nil { + return nil, et + } else if tk != ":" { + return nil, utils.ESyntax(*i-len(tk), def, "':' expected") + } + } + + /* parse the value type */ + if ret.V, err = doParseType(vt.Elem(), def, i, true); err != nil { + return nil, err + } + + /* map end */ + if def != "" { + if tk, et := readToken(def, i, false); et != nil { + return nil, et + } else if tk != ">" { + return nil, utils.ESyntax(*i-len(tk), def, "'>' expected") + } + } + + /* check for list elements */ + if !ret.V.IsValueType() { + return nil, utils.EType(ret.V.S, "non-struct pointers are not valid map value types") + } + + /* set the type tag */ + ret.S = vt + ret.T = T_map + return ret, nil } func doParseSlice(vt reflect.Type, et reflect.Type, def string, i *int, rt *Type) (*Type, error) { - var err error - var tok string - - /* read the next token */ - if tok, err = readToken(def, i, false); err != nil { - return nil, err - } - - /* identify "set" or "list" */ - if tok == "set" { - rt.T = T_set - } else if tok == "list" { - rt.T = T_list - } else { - return nil, utils.ESyntax(*i - len(tok), def, `"set" or "list" expected`) - } - - /* list or set begin */ - if tok, err = readToken(def, i, false); err != nil { - return nil, err - } else if tok != "<" { - return nil, utils.ESyntax(*i - len(tok), def, "'<' expected") - } - - /* set or list element */ - if rt.V, err = doParseType(et, def, i, true); err != nil { - return nil, err - } - - /* list or set end */ - if tok, err = readToken(def, i, false); err != nil { - return nil, err - } else if tok != ">" { - return nil, utils.ESyntax(*i - len(tok), def, "'>' expected") - } - - /* check for list elements */ - if !rt.V.IsValueType() { - return nil, utils.EType(rt.V.S, "non-struct pointers are not valid list/set elements") - } - - /* set the type */ - rt.S = vt - return rt, nil + var err error + var tok string + + /* read the next token */ + if tok, err = readToken(def, i, false); err != nil { + return nil, err + } + + /* identify "set" or "list" */ + if tok == "set" { + rt.T = T_set + } else if tok == "list" { + rt.T = T_list + } else { + return nil, utils.ESyntax(*i-len(tok), def, `"set" or "list" expected`) + } + + /* list or set begin */ + if tok, err = readToken(def, i, false); err != nil { + return nil, err + } else if tok != "<" { + return nil, utils.ESyntax(*i-len(tok), def, "'<' expected") + } + + /* set or list element */ + if rt.V, err = doParseType(et, def, i, true); err != nil { + return nil, err + } + + /* list or set end */ + if tok, err = readToken(def, i, false); err != nil { + return nil, err + } else if tok != ">" { + return nil, utils.ESyntax(*i-len(tok), def, "'>' expected") + } + + /* check for list elements */ + if !rt.V.IsValueType() { + return nil, utils.EType(rt.V.S, "non-struct pointers are not valid list/set elements") + } + + /* set the type */ + rt.S = vt + return rt, nil } func doMatchStruct(vt reflect.Type, def string, i *int, tv *string) (bool, error) { - var err error - var tok string - - /* mark the starting position */ - sp := *i - tn := vt.Name() - - /* read the next token */ - if tok, err = readToken(def, &sp, true); err != nil { - return false, err - } - - /* anonymous struct */ - if tn == "" && vt.Kind() == reflect.Struct { - return true, nil - } - - /* just a simple type with no qualifiers */ - if tok == "" || tok == ":" || tok == ">" { - return tn == *tv, nil - } - - /* otherwise, it must be a "." */ - if tok != "." { - return false, utils.ESyntax(sp, def, "'.' or '>' expected") - } - - /* must be an identifier */ - if *tv, err = readToken(def, &sp, false); err != nil { - return false, err - } else if !isident0((*tv)[0]) { - return false, utils.ESyntax(sp, def, "struct name expected") - } - - /* update parsing position */ - *i = sp - return tn == *tv, nil + var err error + var tok string + + /* mark the starting position */ + sp := *i + tn := vt.Name() + + /* read the next token */ + if tok, err = readToken(def, &sp, true); err != nil { + return false, err + } + + /* anonymous struct */ + if tn == "" && vt.Kind() == reflect.Struct { + return true, nil + } + + /* just a simple type with no qualifiers */ + if tok == "" || tok == ":" || tok == ">" { + return tn == *tv, nil + } + + /* otherwise, it must be a "." */ + if tok != "." { + return false, utils.ESyntax(sp, def, "'.' or '>' expected") + } + + /* must be an identifier */ + if *tv, err = readToken(def, &sp, false); err != nil { + return false, err + } else if !isident0((*tv)[0]) { + return false, utils.ESyntax(sp, def, "struct name expected") + } + + /* update parsing position */ + *i = sp + return tn == *tv, nil } diff --git a/internal/binary/defs/types_test.go b/internal/binary/defs/types_test.go index 19b98fe..7ac67d2 100644 --- a/internal/binary/defs/types_test.go +++ b/internal/binary/defs/types_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,23 +17,23 @@ package defs import ( - `fmt` - `reflect` - `testing` + "fmt" + "reflect" + "testing" - `github.com/stretchr/testify/require` + "github.com/stretchr/testify/require" ) func TestTypes_Parsing(t *testing.T) { - var v map[string][]reflect.SliceHeader - tt, err := ParseType(reflect.TypeOf(v), "map>") - require.NoError(t, err) - fmt.Println(tt) + var v map[string][]reflect.SliceHeader + tt, err := ParseType(reflect.TypeOf(v), "map>") + require.NoError(t, err) + fmt.Println(tt) } func TestTypes_MapKeyType(t *testing.T) { - var v map[*reflect.SliceHeader]int - tt, err := ParseType(reflect.TypeOf(v), "map") - require.NoError(t, err) - fmt.Println(tt) + var v map[*reflect.SliceHeader]int + tt, err := ParseType(reflect.TypeOf(v), "map") + require.NoError(t, err) + fmt.Println(tt) } diff --git a/internal/binary/encoder/asm.s b/internal/binary/encoder/asm.s index a8c6cc1..9c99547 100644 --- a/internal/binary/encoder/asm.s +++ b/internal/binary/encoder/asm.s @@ -1,4 +1,4 @@ -// Copyright 2022 ByteDance Inc. +// Copyright 2022 CloudWeGo Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/internal/binary/encoder/bucket.go b/internal/binary/encoder/bucket.go index d4a0654..991e2db 100644 --- a/internal/binary/encoder/bucket.go +++ b/internal/binary/encoder/bucket.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,28 +17,28 @@ package encoder import ( - `sync` - `unsafe` + "sync" + "unsafe" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) type _Bucket struct { - h int - v uint64 - p unsafe.Pointer + h int + v uint64 + p unsafe.Pointer } func (self _Bucket) String() string { - return rt.StringFrom(self.p, int(self.v)) + return rt.StringFrom(self.p, int(self.v)) } var ( - bucketPool sync.Pool + bucketPool sync.Pool ) const ( - _BucketSize = unsafe.Sizeof(_Bucket{}) + _BucketSize = unsafe.Sizeof(_Bucket{}) ) //go:noescape @@ -47,72 +47,72 @@ const ( func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) func newBucket(n int) []_Bucket { - var r []_Bucket - var v interface{} - - /* attempt to get one from pool */ - if v = bucketPool.Get(); v == nil { - return make([]_Bucket, n) - } - - /* check for capacity, and update it's capacity */ - if r = v.([]_Bucket); n <= cap(r) { - return bucketClear(r[:n:cap(r)]) - } - - /* not enough space, reallocate a new one */ - bucketPool.Put(v) - return make([]_Bucket, n) + var r []_Bucket + var v interface{} + + /* attempt to get one from pool */ + if v = bucketPool.Get(); v == nil { + return make([]_Bucket, n) + } + + /* check for capacity, and update it's capacity */ + if r = v.([]_Bucket); n <= cap(r) { + return bucketClear(r[:n:cap(r)]) + } + + /* not enough space, reallocate a new one */ + bucketPool.Put(v) + return make([]_Bucket, n) } func freeBucket(p []_Bucket) { - bucketPool.Put(p) + bucketPool.Put(p) } func bucketClear(bm []_Bucket) []_Bucket { - v := (*rt.GoSlice)(unsafe.Pointer(&bm)) - memclrNoHeapPointers(v.Ptr, uintptr(v.Len) * _BucketSize) - return bm + v := (*rt.GoSlice)(unsafe.Pointer(&bm)) + memclrNoHeapPointers(v.Ptr, uintptr(v.Len)*_BucketSize) + return bm } func bucketAppend64(bm []_Bucket, v uint64) bool { - h := hash64(v) - i := h % len(bm) - - /* search for an empty slot (linear probing, this must - * success, since item count never exceeds the bucket size) */ - for bm[i].h != 0 { - if bm[i].v == v { - return true - } else { - i = (i + 1) % len(bm) - } - } - - /* assign the slot */ - bm[i].h = h - bm[i].v = v - return false + h := hash64(v) + i := h % len(bm) + + /* search for an empty slot (linear probing, this must + * success, since item count never exceeds the bucket size) */ + for bm[i].h != 0 { + if bm[i].v == v { + return true + } else { + i = (i + 1) % len(bm) + } + } + + /* assign the slot */ + bm[i].h = h + bm[i].v = v + return false } func bucketAppendStr(bm []_Bucket, p unsafe.Pointer) bool { - h := hashstr(p) - i := h % len(bm) - v := (*rt.GoString)(p) - - /* search for an empty slot (linear probing, this must - * success, since item count never exceeds the bucket size) */ - for bm[i].h != 0 { - if bm[i].h == h && bm[i].v == uint64(v.Len) && (bm[i].p == v.Ptr || bm[i].String() == *(*string)(p)) { - return true - } else { - i = (i + 1) % len(bm) - } - } - - /* assign the slot */ - bm[i].h = h - bm[i].p = v.Ptr - bm[i].v = uint64(v.Len) - return false + h := hashstr(p) + i := h % len(bm) + v := (*rt.GoString)(p) + + /* search for an empty slot (linear probing, this must + * success, since item count never exceeds the bucket size) */ + for bm[i].h != 0 { + if bm[i].h == h && bm[i].v == uint64(v.Len) && (bm[i].p == v.Ptr || bm[i].String() == *(*string)(p)) { + return true + } else { + i = (i + 1) % len(bm) + } + } + + /* assign the slot */ + bm[i].h = h + bm[i].p = v.Ptr + bm[i].v = uint64(v.Len) + return false } diff --git a/internal/binary/encoder/compiler.go b/internal/binary/encoder/compiler.go index 9329f66..7fed9c9 100644 --- a/internal/binary/encoder/compiler.go +++ b/internal/binary/encoder/compiler.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,200 +17,230 @@ package encoder import ( - `fmt` - `reflect` - `strings` - `unsafe` - - `github.com/cloudwego/frugal/internal/binary/defs` - `github.com/cloudwego/frugal/internal/opts` - `github.com/cloudwego/frugal/internal/rt` + "fmt" + "reflect" + "strings" + "unsafe" + + "github.com/cloudwego/frugal/internal/binary/defs" + "github.com/cloudwego/frugal/internal/opts" + "github.com/cloudwego/frugal/internal/rt" ) type Instr struct { - Op OpCode - Uv int32 - Iv int64 - To int - Pr unsafe.Pointer + Op OpCode + Uv int32 + Iv int64 + To int + Pr unsafe.Pointer } type ( - Program []Instr + Program []Instr ) func (self Instr) Vt() *rt.GoType { - return (*rt.GoType)(self.Pr) + return (*rt.GoType)(self.Pr) } func (self Instr) Str() string { - return rt.StringFrom(self.Pr, int(self.Uv)) + return rt.StringFrom(self.Pr, int(self.Uv)) } func (self Instr) Byte(i int64) int8 { - return *(*int8)(unsafe.Pointer(uintptr(self.Pr) + uintptr(i))) + return *(*int8)(unsafe.Pointer(uintptr(self.Pr) + uintptr(i))) } func (self Instr) Word(i int64) int16 { - return *(*int16)(unsafe.Pointer(uintptr(self.Pr) + uintptr(i))) + return *(*int16)(unsafe.Pointer(uintptr(self.Pr) + uintptr(i))) } func (self Instr) Long(i int64) int32 { - return *(*int32)(unsafe.Pointer(uintptr(self.Pr) + uintptr(i))) + return *(*int32)(unsafe.Pointer(uintptr(self.Pr) + uintptr(i))) } func (self Instr) Quad(i int64) int64 { - return *(*int64)(unsafe.Pointer(uintptr(self.Pr) + uintptr(i))) + return *(*int64)(unsafe.Pointer(uintptr(self.Pr) + uintptr(i))) } func (self Instr) Disassemble() string { - switch self.Op { - case OP_size_check : fallthrough - case OP_size_const : fallthrough - case OP_size_map : fallthrough - case OP_seek : fallthrough - case OP_sint : fallthrough - case OP_length : return fmt.Sprintf("%-18s%d", self.Op, self.Iv) - case OP_size_dyn : fallthrough - case OP_memcpy_be : return fmt.Sprintf("%-18s%d, %d", self.Op, self.Uv, self.Iv) - case OP_size_defer : fallthrough - case OP_defer : fallthrough - case OP_map_begin : fallthrough - case OP_unique : return fmt.Sprintf("%-18s%s", self.Op, self.Vt()) - case OP_byte : return fmt.Sprintf("%-18s0x%02x", self.Op, self.Iv) - case OP_word : return fmt.Sprintf("%-18s0x%04x", self.Op, self.Iv) - case OP_long : return fmt.Sprintf("%-18s0x%08x", self.Op, self.Iv) - case OP_quad : return fmt.Sprintf("%-18s0x%016x", self.Op, self.Iv) - case OP_map_if_next : fallthrough - case OP_map_if_empty : fallthrough - case OP_list_if_next : fallthrough - case OP_list_if_empty : fallthrough - case OP_goto : fallthrough - case OP_if_nil : fallthrough - case OP_if_hasbuf : return fmt.Sprintf("%-18sL_%d", self.Op, self.To) - case OP_if_eq_imm : return fmt.Sprintf("%-18s%d:%d, L_%d", self.Op, self.Iv, self.Uv, self.To) - case OP_if_eq_str : return fmt.Sprintf("%-18s%q, L_%d", self.Op, self.Str(), self.To) - default : return self.Op.String() - } + switch self.Op { + case OP_size_check: + fallthrough + case OP_size_const: + fallthrough + case OP_size_map: + fallthrough + case OP_seek: + fallthrough + case OP_sint: + fallthrough + case OP_length: + return fmt.Sprintf("%-18s%d", self.Op, self.Iv) + case OP_size_dyn: + fallthrough + case OP_memcpy_be: + return fmt.Sprintf("%-18s%d, %d", self.Op, self.Uv, self.Iv) + case OP_size_defer: + fallthrough + case OP_defer: + fallthrough + case OP_map_begin: + fallthrough + case OP_unique: + return fmt.Sprintf("%-18s%s", self.Op, self.Vt()) + case OP_byte: + return fmt.Sprintf("%-18s0x%02x", self.Op, self.Iv) + case OP_word: + return fmt.Sprintf("%-18s0x%04x", self.Op, self.Iv) + case OP_long: + return fmt.Sprintf("%-18s0x%08x", self.Op, self.Iv) + case OP_quad: + return fmt.Sprintf("%-18s0x%016x", self.Op, self.Iv) + case OP_map_if_next: + fallthrough + case OP_map_if_empty: + fallthrough + case OP_list_if_next: + fallthrough + case OP_list_if_empty: + fallthrough + case OP_goto: + fallthrough + case OP_if_nil: + fallthrough + case OP_if_hasbuf: + return fmt.Sprintf("%-18sL_%d", self.Op, self.To) + case OP_if_eq_imm: + return fmt.Sprintf("%-18s%d:%d, L_%d", self.Op, self.Iv, self.Uv, self.To) + case OP_if_eq_str: + return fmt.Sprintf("%-18s%q, L_%d", self.Op, self.Str(), self.To) + default: + return self.Op.String() + } } func (self Program) pc() int { return len(self) } func (self Program) pin(i int) { self[i].To = self.pc() } func (self Program) tag(n int) { - if n >= defs.StackSize { - panic("type nesting too deep") - } + if n >= defs.StackSize { + panic("type nesting too deep") + } } -func (self *Program) ins(iv Instr) { *self = append(*self, iv) } -func (self *Program) add(op OpCode) { self.ins(Instr { Op: op }) } -func (self *Program) jmp(op OpCode, to int) { self.ins(Instr { Op: op, To: to }) } -func (self *Program) i64(op OpCode, iv int64) { self.ins(Instr { Op: op, Iv: iv }) } -func (self *Program) str(op OpCode, sv string) { self.ins(Instr { Op: op, Iv: int64(len(sv)), Pr: rt.StringPtr(sv) }) } -func (self *Program) rtt(op OpCode, vt reflect.Type) { self.ins(Instr { Op: op, Pr: unsafe.Pointer(rt.UnpackType(vt)) }) } -func (self *Program) dyn(op OpCode, uv int32, iv int64) { self.ins(Instr { Op: op, Uv: uv, Iv: iv }) } +func (self *Program) ins(iv Instr) { *self = append(*self, iv) } +func (self *Program) add(op OpCode) { self.ins(Instr{Op: op}) } +func (self *Program) jmp(op OpCode, to int) { self.ins(Instr{Op: op, To: to}) } +func (self *Program) i64(op OpCode, iv int64) { self.ins(Instr{Op: op, Iv: iv}) } +func (self *Program) str(op OpCode, sv string) { + self.ins(Instr{Op: op, Iv: int64(len(sv)), Pr: rt.StringPtr(sv)}) +} +func (self *Program) rtt(op OpCode, vt reflect.Type) { + self.ins(Instr{Op: op, Pr: unsafe.Pointer(rt.UnpackType(vt))}) +} +func (self *Program) dyn(op OpCode, uv int32, iv int64) { self.ins(Instr{Op: op, Uv: uv, Iv: iv}) } func (self Program) Free() { - freeProgram(self) + freeProgram(self) } func (self Program) Disassemble() string { - nb := len(self) - tab := make([]bool, nb + 1) - ret := make([]string, 0, nb + 1) - - /* prescan to get all the labels */ - for _, ins := range self { - if _OpBranches[ins.Op] { - tab[ins.To] = true - } - } - - /* disassemble each instruction */ - for i, ins := range self { - if !tab[i] { - ret = append(ret, " " + ins.Disassemble()) - } else { - ret = append(ret, fmt.Sprintf("L_%d:\n %s", i, ins.Disassemble())) - } - } - - /* add an "end" indicator, and join all the strings */ - if !tab[nb] { - return strings.Join(append(ret, " end"), "\n") - } else { - return strings.Join(append(ret, fmt.Sprintf("L_%d:", nb), " end"), "\n") - } + nb := len(self) + tab := make([]bool, nb+1) + ret := make([]string, 0, nb+1) + + /* prescan to get all the labels */ + for _, ins := range self { + if _OpBranches[ins.Op] { + tab[ins.To] = true + } + } + + /* disassemble each instruction */ + for i, ins := range self { + if !tab[i] { + ret = append(ret, " "+ins.Disassemble()) + } else { + ret = append(ret, fmt.Sprintf("L_%d:\n %s", i, ins.Disassemble())) + } + } + + /* add an "end" indicator, and join all the strings */ + if !tab[nb] { + return strings.Join(append(ret, " end"), "\n") + } else { + return strings.Join(append(ret, fmt.Sprintf("L_%d:", nb), " end"), "\n") + } } type Compiler struct { - o opts.Options - t map[reflect.Type]bool + o opts.Options + t map[reflect.Type]bool } func CreateCompiler() *Compiler { - return newCompiler() + return newCompiler() } func (self *Compiler) rescue(ep *error) { - if val := recover(); val != nil { - if err, ok := val.(error); ok { - *ep = err - } else { - panic(val) - } - } + if val := recover(); val != nil { + if err, ok := val.(error); ok { + *ep = err + } else { + panic(val) + } + } } func (self *Compiler) Free() { - freeCompiler(self) + freeCompiler(self) } func (self *Compiler) Apply(o opts.Options) *Compiler { - self.o = o - return self + self.o = o + return self } func (self *Compiler) resetState() *Compiler { - o := self.o // prevent resetting opts - resetCompiler(self) - self.o = o - return self + o := self.o // prevent resetting opts + resetCompiler(self) + self.o = o + return self } func (self *Compiler) Compile(vt reflect.Type) (_ Program, err error) { - ret := newProgram() - vtp := (*defs.Type)(nil) + ret := newProgram() + vtp := (*defs.Type)(nil) - /* parse the type */ - if vtp, err = defs.ParseType(vt, ""); err != nil { - return nil, err - } + /* parse the type */ + if vtp, err = defs.ParseType(vt, ""); err != nil { + return nil, err + } - /* catch the exceptions, and free the type */ - defer self.rescue(&err) - defer vtp.Free() + /* catch the exceptions, and free the type */ + defer self.rescue(&err) + defer vtp.Free() - /* object measuring */ - i := ret.pc() - ret.add(OP_if_hasbuf) - self.resetState().measure(&ret, 0, vtp, ret.pc()) + /* object measuring */ + i := ret.pc() + ret.add(OP_if_hasbuf) + self.resetState().measure(&ret, 0, vtp, ret.pc()) - /* object encoding */ - j := ret.pc() - ret.add(OP_goto) - ret.pin(i) - self.resetState().compile(&ret, 0, vtp, ret.pc()) + /* object encoding */ + j := ret.pc() + ret.add(OP_goto) + ret.pin(i) + self.resetState().compile(&ret, 0, vtp, ret.pc()) - /* halt the program */ - ret.pin(j) - ret.add(OP_halt) - return Optimize(ret), nil + /* halt the program */ + ret.pin(j) + ret.add(OP_halt) + return Optimize(ret), nil } func (self *Compiler) CompileAndFree(vt reflect.Type) (ret Program, err error) { - ret, err = self.Compile(vt) - self.Free() - return + ret, err = self.Compile(vt) + self.Free() + return } diff --git a/internal/binary/encoder/compiler_encode.go b/internal/binary/encoder/compiler_encode.go index 555c896..a5f0527 100644 --- a/internal/binary/encoder/compiler_encode.go +++ b/internal/binary/encoder/compiler_encode.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,328 +17,385 @@ package encoder import ( - `math` + "math" - `github.com/cloudwego/frugal/internal/atm/abi` - `github.com/cloudwego/frugal/internal/binary/defs` + "github.com/cloudwego/frugal/internal/atm/abi" + "github.com/cloudwego/frugal/internal/binary/defs" ) func (self *Compiler) compile(p *Program, sp int, vt *defs.Type, startpc int) { - rt := vt.S - tt := vt.T - - /* only recurse on structs */ - if tt != defs.T_struct { - self.compileOne(p, sp, vt, startpc) - return - } - - /* check for loops */ - if self.t[rt] || !self.o.CanInline(sp, (p.pc() - startpc) * 2) { - p.rtt(OP_defer, rt) - return - } - - /* compile the type recursively */ - self.t[rt] = true - self.compileOne(p, sp, vt, startpc) - delete(self.t, rt) + rt := vt.S + tt := vt.T + + /* only recurse on structs */ + if tt != defs.T_struct { + self.compileOne(p, sp, vt, startpc) + return + } + + /* check for loops */ + if self.t[rt] || !self.o.CanInline(sp, (p.pc()-startpc)*2) { + p.rtt(OP_defer, rt) + return + } + + /* compile the type recursively */ + self.t[rt] = true + self.compileOne(p, sp, vt, startpc) + delete(self.t, rt) } func (self *Compiler) compileOne(p *Program, sp int, vt *defs.Type, startpc int) { - switch vt.T { - case defs.T_bool : p.i64(OP_size_check, 1); p.i64(OP_sint, 1) - case defs.T_i8 : p.i64(OP_size_check, 1); p.i64(OP_sint, 1) - case defs.T_i16 : p.i64(OP_size_check, 2); p.i64(OP_sint, 2) - case defs.T_i32 : p.i64(OP_size_check, 4); p.i64(OP_sint, 4) - case defs.T_i64 : p.i64(OP_size_check, 8); p.i64(OP_sint, 8) - case defs.T_enum : p.i64(OP_size_check, 4); p.i64(OP_sint, 4) - case defs.T_double : p.i64(OP_size_check, 8); p.i64(OP_sint, 8) - case defs.T_string : p.i64(OP_size_check, 4); p.i64(OP_length, abi.PtrSize); p.dyn(OP_memcpy_be, abi.PtrSize, 1) - case defs.T_binary : p.i64(OP_size_check, 4); p.i64(OP_length, abi.PtrSize); p.dyn(OP_memcpy_be, abi.PtrSize, 1) - case defs.T_map : self.compileMap(p, sp, vt, startpc) - case defs.T_set : self.compileSeq(p, sp, vt, startpc, true) - case defs.T_list : self.compileSeq(p, sp, vt, startpc, false) - case defs.T_struct : self.compileStruct(p, sp, vt, startpc) - case defs.T_pointer : self.compilePtr(p, sp, vt, startpc) - default : panic("unreachable") - } + switch vt.T { + case defs.T_bool: + p.i64(OP_size_check, 1) + p.i64(OP_sint, 1) + case defs.T_i8: + p.i64(OP_size_check, 1) + p.i64(OP_sint, 1) + case defs.T_i16: + p.i64(OP_size_check, 2) + p.i64(OP_sint, 2) + case defs.T_i32: + p.i64(OP_size_check, 4) + p.i64(OP_sint, 4) + case defs.T_i64: + p.i64(OP_size_check, 8) + p.i64(OP_sint, 8) + case defs.T_enum: + p.i64(OP_size_check, 4) + p.i64(OP_sint, 4) + case defs.T_double: + p.i64(OP_size_check, 8) + p.i64(OP_sint, 8) + case defs.T_string: + p.i64(OP_size_check, 4) + p.i64(OP_length, abi.PtrSize) + p.dyn(OP_memcpy_be, abi.PtrSize, 1) + case defs.T_binary: + p.i64(OP_size_check, 4) + p.i64(OP_length, abi.PtrSize) + p.dyn(OP_memcpy_be, abi.PtrSize, 1) + case defs.T_map: + self.compileMap(p, sp, vt, startpc) + case defs.T_set: + self.compileSeq(p, sp, vt, startpc, true) + case defs.T_list: + self.compileSeq(p, sp, vt, startpc, false) + case defs.T_struct: + self.compileStruct(p, sp, vt, startpc) + case defs.T_pointer: + self.compilePtr(p, sp, vt, startpc) + default: + panic("unreachable") + } } func (self *Compiler) compilePtr(p *Program, sp int, vt *defs.Type, startpc int) { - i := p.pc() - p.tag(sp) - p.add(OP_if_nil) - p.add(OP_make_state) - p.add(OP_deref) - self.compile(p, sp + 1, vt.V, startpc) - p.add(OP_drop_state) - p.pin(i) + i := p.pc() + p.tag(sp) + p.add(OP_if_nil) + p.add(OP_make_state) + p.add(OP_deref) + self.compile(p, sp+1, vt.V, startpc) + p.add(OP_drop_state) + p.pin(i) } func (self *Compiler) compileMap(p *Program, sp int, vt *defs.Type, startpc int) { - kt := vt.K - et := vt.V - - /* 6-byte map header */ - p.tag(sp) - p.i64(OP_size_check, 6) - p.i64(OP_byte, int64(kt.Tag())) - p.i64(OP_byte, int64(et.Tag())) - - /* check for nil maps */ - i := p.pc() - p.add(OP_if_nil) - - /* encode the map */ - p.add(OP_map_len) - j := p.pc() - p.add(OP_map_if_empty) - p.add(OP_make_state) - p.rtt(OP_map_begin, vt.S) - k := p.pc() - p.add(OP_map_key) - self.compileItem(p, sp + 1, kt, startpc) - p.add(OP_map_value) - self.compileItem(p, sp + 1, et, startpc) - p.add(OP_map_next) - p.jmp(OP_map_if_next, k) - p.add(OP_drop_state) - - /* encode the length for nil maps */ - r := p.pc() - p.add(OP_goto) - p.pin(i) - p.i64(OP_long, 0) - p.pin(j) - p.pin(r) + kt := vt.K + et := vt.V + + /* 6-byte map header */ + p.tag(sp) + p.i64(OP_size_check, 6) + p.i64(OP_byte, int64(kt.Tag())) + p.i64(OP_byte, int64(et.Tag())) + + /* check for nil maps */ + i := p.pc() + p.add(OP_if_nil) + + /* encode the map */ + p.add(OP_map_len) + j := p.pc() + p.add(OP_map_if_empty) + p.add(OP_make_state) + p.rtt(OP_map_begin, vt.S) + k := p.pc() + p.add(OP_map_key) + self.compileItem(p, sp+1, kt, startpc) + p.add(OP_map_value) + self.compileItem(p, sp+1, et, startpc) + p.add(OP_map_next) + p.jmp(OP_map_if_next, k) + p.add(OP_drop_state) + + /* encode the length for nil maps */ + r := p.pc() + p.add(OP_goto) + p.pin(i) + p.i64(OP_long, 0) + p.pin(j) + p.pin(r) } func (self *Compiler) compileSeq(p *Program, sp int, vt *defs.Type, startpc int, verifyUnique bool) { - nb := -1 - et := vt.V - - /* 5-byte set or list header */ - p.tag(sp) - p.i64(OP_size_check, 5) - p.i64(OP_byte, int64(et.Tag())) - p.i64(OP_length, abi.PtrSize) - - /* check for nil slice */ - i := p.pc() - p.add(OP_if_nil) - - /* special case of primitive sets or lists */ - switch et.T { - case defs.T_bool : nb = 1 - case defs.T_i8 : nb = 1 - case defs.T_i16 : nb = 2 - case defs.T_i32 : nb = 4 - case defs.T_i64 : nb = 8 - case defs.T_double : nb = 8 - } - - /* check for uniqueness if needed */ - if verifyUnique { - p.rtt(OP_unique, et.S) - } - - /* check if this is the special case */ - if nb != -1 { - p.dyn(OP_memcpy_be, abi.PtrSize, int64(nb)) - p.pin(i) - return - } - - /* complex sets or lists */ - j := p.pc() - p.add(OP_list_if_empty) - p.add(OP_make_state) - p.add(OP_list_begin) - k := p.pc() - p.add(OP_goto) - r := p.pc() - p.i64(OP_seek, int64(et.S.Size())) - p.pin(k) - self.compileItem(p, sp + 1, et, startpc) - p.add(OP_list_decr) - p.jmp(OP_list_if_next, r) - p.add(OP_drop_state) - p.pin(i) - p.pin(j) + nb := -1 + et := vt.V + + /* 5-byte set or list header */ + p.tag(sp) + p.i64(OP_size_check, 5) + p.i64(OP_byte, int64(et.Tag())) + p.i64(OP_length, abi.PtrSize) + + /* check for nil slice */ + i := p.pc() + p.add(OP_if_nil) + + /* special case of primitive sets or lists */ + switch et.T { + case defs.T_bool: + nb = 1 + case defs.T_i8: + nb = 1 + case defs.T_i16: + nb = 2 + case defs.T_i32: + nb = 4 + case defs.T_i64: + nb = 8 + case defs.T_double: + nb = 8 + } + + /* check for uniqueness if needed */ + if verifyUnique { + p.rtt(OP_unique, et.S) + } + + /* check if this is the special case */ + if nb != -1 { + p.dyn(OP_memcpy_be, abi.PtrSize, int64(nb)) + p.pin(i) + return + } + + /* complex sets or lists */ + j := p.pc() + p.add(OP_list_if_empty) + p.add(OP_make_state) + p.add(OP_list_begin) + k := p.pc() + p.add(OP_goto) + r := p.pc() + p.i64(OP_seek, int64(et.S.Size())) + p.pin(k) + self.compileItem(p, sp+1, et, startpc) + p.add(OP_list_decr) + p.jmp(OP_list_if_next, r) + p.add(OP_drop_state) + p.pin(i) + p.pin(j) } func (self *Compiler) compileItem(p *Program, sp int, vt *defs.Type, startpc int) { - tag := vt.T - elem := vt.V - - /* special handling for pointers */ - if tag != defs.T_pointer { - self.compile(p, sp, vt, startpc) - return - } - - /* must be pointer struct at this point */ - if elem.T != defs.T_struct { - panic("fatal: non-struct pointers within container elements") - } - - /* always add the STOP field for structs */ - i := p.pc() - p.tag(sp) - p.add(OP_if_nil) - p.add(OP_make_state) - p.add(OP_deref) - self.compile(p, sp + 1, elem, startpc) - p.add(OP_drop_state) - j := p.pc() - p.add(OP_goto) - p.pin(i) - p.i64(OP_size_check, 1) - p.i64(OP_byte, 0) - p.pin(j) + tag := vt.T + elem := vt.V + + /* special handling for pointers */ + if tag != defs.T_pointer { + self.compile(p, sp, vt, startpc) + return + } + + /* must be pointer struct at this point */ + if elem.T != defs.T_struct { + panic("fatal: non-struct pointers within container elements") + } + + /* always add the STOP field for structs */ + i := p.pc() + p.tag(sp) + p.add(OP_if_nil) + p.add(OP_make_state) + p.add(OP_deref) + self.compile(p, sp+1, elem, startpc) + p.add(OP_drop_state) + j := p.pc() + p.add(OP_goto) + p.pin(i) + p.i64(OP_size_check, 1) + p.i64(OP_byte, 0) + p.pin(j) } func (self *Compiler) compileStruct(p *Program, sp int, vt *defs.Type, startpc int) { - var err error - var fvs []defs.Field - - /* resolve the field */ - if fvs, err = defs.ResolveFields(vt.S); err != nil { - panic(err) - } - - /* compile every field */ - for _, fv := range fvs { - p.tag(sp) - p.i64(OP_seek, int64(fv.F)) - self.compileStructField(p, sp + 1, fv, startpc) - p.i64(OP_seek, -int64(fv.F)) - } - - /* add the STOP field */ - p.i64(OP_size_check, 1) - p.i64(OP_byte, 0) + var err error + var fvs []defs.Field + + /* resolve the field */ + if fvs, err = defs.ResolveFields(vt.S); err != nil { + panic(err) + } + + /* compile every field */ + for _, fv := range fvs { + p.tag(sp) + p.i64(OP_seek, int64(fv.F)) + self.compileStructField(p, sp+1, fv, startpc) + p.i64(OP_seek, -int64(fv.F)) + } + + /* add the STOP field */ + p.i64(OP_size_check, 1) + p.i64(OP_byte, 0) } func (self *Compiler) compileStructField(p *Program, sp int, fv defs.Field, startpc int) { - switch fv.Type.T { - default: { - panic("fatal: invalid field type: " + fv.Type.String()) - } - - /* non-pointer types */ - case defs.T_bool : fallthrough - case defs.T_i8 : fallthrough - case defs.T_double : fallthrough - case defs.T_i16 : fallthrough - case defs.T_i32 : fallthrough - case defs.T_i64 : fallthrough - case defs.T_string : fallthrough - case defs.T_enum : fallthrough - case defs.T_binary : { - if fv.Default.IsValid() && fv.Spec == defs.Optional { - self.compileStructDefault(p, sp, fv, startpc) - } else { - self.compileStructRequired(p, sp, fv, startpc) - } - } - - /* struct types, only available in hand-written structs */ - case defs.T_struct: { - self.compileStructRequired(p, sp, fv, startpc) - } - - /* sequencial types */ - case defs.T_map : fallthrough - case defs.T_set : fallthrough - case defs.T_list : { - if fv.Spec == defs.Optional { - self.compileStructIterable(p, sp, fv, startpc) - } else { - self.compileStructRequired(p, sp, fv, startpc) - } - } - - /* pointers */ - case defs.T_pointer: { - if fv.Spec == defs.Optional { - self.compileStructOptional(p, sp, fv, startpc) - } else if fv.Type.V.T == defs.T_struct { - self.compileStructPointer(p, sp, fv, startpc) - } else { - panic("fatal: non-optional non-struct pointers") - } - } - } + switch fv.Type.T { + default: + { + panic("fatal: invalid field type: " + fv.Type.String()) + } + + /* non-pointer types */ + case defs.T_bool: + fallthrough + case defs.T_i8: + fallthrough + case defs.T_double: + fallthrough + case defs.T_i16: + fallthrough + case defs.T_i32: + fallthrough + case defs.T_i64: + fallthrough + case defs.T_string: + fallthrough + case defs.T_enum: + fallthrough + case defs.T_binary: + { + if fv.Default.IsValid() && fv.Spec == defs.Optional { + self.compileStructDefault(p, sp, fv, startpc) + } else { + self.compileStructRequired(p, sp, fv, startpc) + } + } + + /* struct types, only available in hand-written structs */ + case defs.T_struct: + { + self.compileStructRequired(p, sp, fv, startpc) + } + + /* sequential types */ + case defs.T_map: + fallthrough + case defs.T_set: + fallthrough + case defs.T_list: + { + if fv.Spec == defs.Optional { + self.compileStructIterable(p, sp, fv, startpc) + } else { + self.compileStructRequired(p, sp, fv, startpc) + } + } + + /* pointers */ + case defs.T_pointer: + { + if fv.Spec == defs.Optional { + self.compileStructOptional(p, sp, fv, startpc) + } else if fv.Type.V.T == defs.T_struct { + self.compileStructPointer(p, sp, fv, startpc) + } else { + panic("fatal: non-optional non-struct pointers") + } + } + } } func (self *Compiler) compileStructDefault(p *Program, sp int, fv defs.Field, startpc int) { - i := p.pc() - t := fv.Type.T - - /* check for default values */ - switch t { - case defs.T_bool : p.dyn(OP_if_eq_imm, 1, bool2i64(fv.Default.Bool())) - case defs.T_i8 : p.dyn(OP_if_eq_imm, 1, fv.Default.Int()) - case defs.T_double : p.dyn(OP_if_eq_imm, 8, int64(math.Float64bits(fv.Default.Float()))) - case defs.T_i16 : p.dyn(OP_if_eq_imm, 2, fv.Default.Int()) - case defs.T_i32 : p.dyn(OP_if_eq_imm, 4, fv.Default.Int()) - case defs.T_i64 : p.dyn(OP_if_eq_imm, 8, fv.Default.Int()) - case defs.T_string : p.str(OP_if_eq_str, fv.Default.String()) - case defs.T_enum : p.dyn(OP_if_eq_imm, 4, fv.Default.Int()) - case defs.T_binary : p.str(OP_if_eq_str, mem2str(fv.Default.Bytes())) - default : panic("unreachable") - } - - /* compile if it's not the default value */ - self.compileStructFieldBegin(p, fv, 3) - self.compile(p, sp, fv.Type, startpc) - p.pin(i) + i := p.pc() + t := fv.Type.T + + /* check for default values */ + switch t { + case defs.T_bool: + p.dyn(OP_if_eq_imm, 1, bool2i64(fv.Default.Bool())) + case defs.T_i8: + p.dyn(OP_if_eq_imm, 1, fv.Default.Int()) + case defs.T_double: + p.dyn(OP_if_eq_imm, 8, int64(math.Float64bits(fv.Default.Float()))) + case defs.T_i16: + p.dyn(OP_if_eq_imm, 2, fv.Default.Int()) + case defs.T_i32: + p.dyn(OP_if_eq_imm, 4, fv.Default.Int()) + case defs.T_i64: + p.dyn(OP_if_eq_imm, 8, fv.Default.Int()) + case defs.T_string: + p.str(OP_if_eq_str, fv.Default.String()) + case defs.T_enum: + p.dyn(OP_if_eq_imm, 4, fv.Default.Int()) + case defs.T_binary: + p.str(OP_if_eq_str, mem2str(fv.Default.Bytes())) + default: + panic("unreachable") + } + + /* compile if it's not the default value */ + self.compileStructFieldBegin(p, fv, 3) + self.compile(p, sp, fv.Type, startpc) + p.pin(i) } func (self *Compiler) compileStructPointer(p *Program, sp int, fv defs.Field, startpc int) { - i := p.pc() - p.add(OP_if_nil) - self.compileStructFieldBegin(p, fv, 4) - p.add(OP_make_state) - p.add(OP_deref) - self.compile(p, sp + 1, fv.Type.V, startpc) - p.add(OP_drop_state) - j := p.pc() - p.add(OP_goto) - p.pin(i) - self.compileStructFieldBegin(p, fv, 4) - p.i64(OP_byte, 0) - p.pin(j) + i := p.pc() + p.add(OP_if_nil) + self.compileStructFieldBegin(p, fv, 4) + p.add(OP_make_state) + p.add(OP_deref) + self.compile(p, sp+1, fv.Type.V, startpc) + p.add(OP_drop_state) + j := p.pc() + p.add(OP_goto) + p.pin(i) + self.compileStructFieldBegin(p, fv, 4) + p.i64(OP_byte, 0) + p.pin(j) } func (self *Compiler) compileStructIterable(p *Program, sp int, fv defs.Field, startpc int) { - i := p.pc() - p.add(OP_if_nil) - self.compileStructFieldBegin(p, fv, 3) - self.compile(p, sp, fv.Type, startpc) - p.pin(i) + i := p.pc() + p.add(OP_if_nil) + self.compileStructFieldBegin(p, fv, 3) + self.compile(p, sp, fv.Type, startpc) + p.pin(i) } func (self *Compiler) compileStructOptional(p *Program, sp int, fv defs.Field, startpc int) { - i := p.pc() - p.add(OP_if_nil) - self.compileStructFieldBegin(p, fv, 3) - p.add(OP_make_state) - p.add(OP_deref) - self.compile(p, sp + 1, fv.Type.V, startpc) - p.add(OP_drop_state) - p.pin(i) + i := p.pc() + p.add(OP_if_nil) + self.compileStructFieldBegin(p, fv, 3) + p.add(OP_make_state) + p.add(OP_deref) + self.compile(p, sp+1, fv.Type.V, startpc) + p.add(OP_drop_state) + p.pin(i) } func (self *Compiler) compileStructRequired(p *Program, sp int, fv defs.Field, startpc int) { - self.compileStructFieldBegin(p, fv, 3) - self.compile(p, sp, fv.Type, startpc) + self.compileStructFieldBegin(p, fv, 3) + self.compile(p, sp, fv.Type, startpc) } func (self *Compiler) compileStructFieldBegin(p *Program, fv defs.Field, nb int64) { - p.i64(OP_size_check, nb) - p.i64(OP_byte, int64(fv.Type.Tag())) - p.i64(OP_word, int64(fv.ID)) + p.i64(OP_size_check, nb) + p.i64(OP_byte, int64(fv.Type.Tag())) + p.i64(OP_word, int64(fv.ID)) } diff --git a/internal/binary/encoder/compiler_measure.go b/internal/binary/encoder/compiler_measure.go index 4f68d91..ff46434 100644 --- a/internal/binary/encoder/compiler_measure.go +++ b/internal/binary/encoder/compiler_measure.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,326 +17,372 @@ package encoder import ( - `math` + "math" - `github.com/cloudwego/frugal/internal/atm/abi` - `github.com/cloudwego/frugal/internal/binary/defs` + "github.com/cloudwego/frugal/internal/atm/abi" + "github.com/cloudwego/frugal/internal/binary/defs" ) func (self *Compiler) measure(p *Program, sp int, vt *defs.Type, startpc int) { - rt := vt.S - tt := vt.T - - /* only recurse on structs */ - if tt != defs.T_struct { - self.measureOne(p, sp, vt, startpc) - return - } - - /* check for loops with inlining depth limit */ - if self.t[rt] || !self.o.CanInline(sp, (p.pc() - startpc) * 2) { - p.rtt(OP_size_defer, rt) - return - } - - /* measure the type recursively */ - self.t[rt] = true - self.measureOne(p, sp, vt, startpc) - delete(self.t, rt) + rt := vt.S + tt := vt.T + + /* only recurse on structs */ + if tt != defs.T_struct { + self.measureOne(p, sp, vt, startpc) + return + } + + /* check for loops with inlining depth limit */ + if self.t[rt] || !self.o.CanInline(sp, (p.pc()-startpc)*2) { + p.rtt(OP_size_defer, rt) + return + } + + /* measure the type recursively */ + self.t[rt] = true + self.measureOne(p, sp, vt, startpc) + delete(self.t, rt) } func (self *Compiler) measureOne(p *Program, sp int, vt *defs.Type, startpc int) { - switch vt.T { - case defs.T_bool : p.i64(OP_size_const, 1) - case defs.T_i8 : p.i64(OP_size_const, 1) - case defs.T_i16 : p.i64(OP_size_const, 2) - case defs.T_i32 : p.i64(OP_size_const, 4) - case defs.T_i64 : p.i64(OP_size_const, 8) - case defs.T_enum : p.i64(OP_size_const, 4) - case defs.T_double : p.i64(OP_size_const, 8) - case defs.T_string : p.i64(OP_size_const, 4); p.dyn(OP_size_dyn, abi.PtrSize, 1) - case defs.T_binary : p.i64(OP_size_const, 4); p.dyn(OP_size_dyn, abi.PtrSize, 1) - case defs.T_map : self.measureMap(p, sp, vt, startpc) - case defs.T_set : self.measureSeq(p, sp, vt, startpc) - case defs.T_list : self.measureSeq(p, sp, vt, startpc) - case defs.T_struct : self.measureStruct(p, sp, vt, startpc) - case defs.T_pointer : self.measurePtr(p, sp, vt, startpc) - default : panic("measureOne: unreachable") - } + switch vt.T { + case defs.T_bool: + p.i64(OP_size_const, 1) + case defs.T_i8: + p.i64(OP_size_const, 1) + case defs.T_i16: + p.i64(OP_size_const, 2) + case defs.T_i32: + p.i64(OP_size_const, 4) + case defs.T_i64: + p.i64(OP_size_const, 8) + case defs.T_enum: + p.i64(OP_size_const, 4) + case defs.T_double: + p.i64(OP_size_const, 8) + case defs.T_string: + p.i64(OP_size_const, 4) + p.dyn(OP_size_dyn, abi.PtrSize, 1) + case defs.T_binary: + p.i64(OP_size_const, 4) + p.dyn(OP_size_dyn, abi.PtrSize, 1) + case defs.T_map: + self.measureMap(p, sp, vt, startpc) + case defs.T_set: + self.measureSeq(p, sp, vt, startpc) + case defs.T_list: + self.measureSeq(p, sp, vt, startpc) + case defs.T_struct: + self.measureStruct(p, sp, vt, startpc) + case defs.T_pointer: + self.measurePtr(p, sp, vt, startpc) + default: + panic("measureOne: unreachable") + } } func (self *Compiler) measurePtr(p *Program, sp int, vt *defs.Type, startpc int) { - i := p.pc() - p.tag(sp) - p.add(OP_if_nil) - p.add(OP_make_state) - p.add(OP_deref) - self.measure(p, sp + 1, vt.V, startpc) - p.add(OP_drop_state) - p.pin(i) + i := p.pc() + p.tag(sp) + p.add(OP_if_nil) + p.add(OP_make_state) + p.add(OP_deref) + self.measure(p, sp+1, vt.V, startpc) + p.add(OP_drop_state) + p.pin(i) } func (self *Compiler) measureMap(p *Program, sp int, vt *defs.Type, startpc int) { - nk := defs.GetSize(vt.K.S) - nv := defs.GetSize(vt.V.S) - - /* 6-byte map header */ - p.tag(sp) - p.i64(OP_size_const, 6) - - /* check for nil maps */ - i := p.pc() - p.add(OP_if_nil) - - /* key and value are both trivially measuable */ - if nk > 0 && nv > 0 { - p.i64(OP_size_map, int64(nk + nv)) - p.pin(i) - return - } - - /* key or value is trivially measuable */ - if nk > 0 { p.i64(OP_size_map, int64(nk)) } - if nv > 0 { p.i64(OP_size_map, int64(nv)) } - - /* complex maps */ - j := p.pc() - p.add(OP_map_if_empty) - p.add(OP_make_state) - p.rtt(OP_map_begin, vt.S) - k := p.pc() - - /* complex keys */ - if nk <= 0 { - p.add(OP_map_key) - self.measureItem(p, sp + 1, vt.K, startpc) - } - - /* complex values */ - if nv <= 0 { - p.add(OP_map_value) - self.measureItem(p, sp + 1, vt.V, startpc) - } - - /* move to the next state */ - p.add(OP_map_next) - p.jmp(OP_map_if_next, k) - p.add(OP_drop_state) - p.pin(i) - p.pin(j) + nk := defs.GetSize(vt.K.S) + nv := defs.GetSize(vt.V.S) + + /* 6-byte map header */ + p.tag(sp) + p.i64(OP_size_const, 6) + + /* check for nil maps */ + i := p.pc() + p.add(OP_if_nil) + + /* key and value are both trivially measuable */ + if nk > 0 && nv > 0 { + p.i64(OP_size_map, int64(nk+nv)) + p.pin(i) + return + } + + /* key or value is trivially measuable */ + if nk > 0 { + p.i64(OP_size_map, int64(nk)) + } + if nv > 0 { + p.i64(OP_size_map, int64(nv)) + } + + /* complex maps */ + j := p.pc() + p.add(OP_map_if_empty) + p.add(OP_make_state) + p.rtt(OP_map_begin, vt.S) + k := p.pc() + + /* complex keys */ + if nk <= 0 { + p.add(OP_map_key) + self.measureItem(p, sp+1, vt.K, startpc) + } + + /* complex values */ + if nv <= 0 { + p.add(OP_map_value) + self.measureItem(p, sp+1, vt.V, startpc) + } + + /* move to the next state */ + p.add(OP_map_next) + p.jmp(OP_map_if_next, k) + p.add(OP_drop_state) + p.pin(i) + p.pin(j) } func (self *Compiler) measureSeq(p *Program, sp int, vt *defs.Type, startpc int) { - et := vt.V - nb := defs.GetSize(et.S) - - /* 5-byte list or set header */ - p.tag(sp) - p.i64(OP_size_const, 5) - - /* check for nil slice */ - i := p.pc() - p.add(OP_if_nil) - - /* element is trivially measuable */ - if nb > 0 { - p.dyn(OP_size_dyn, abi.PtrSize, int64(nb)) - p.pin(i) - return - } - - /* complex lists or sets */ - j := p.pc() - p.add(OP_list_if_empty) - p.add(OP_make_state) - p.add(OP_list_begin) - k := p.pc() - p.add(OP_goto) - r := p.pc() - p.i64(OP_seek, int64(et.S.Size())) - p.pin(k) - self.measureItem(p, sp + 1, et, startpc) - p.add(OP_list_decr) - p.jmp(OP_list_if_next, r) - p.add(OP_drop_state) - p.pin(i) - p.pin(j) + et := vt.V + nb := defs.GetSize(et.S) + + /* 5-byte list or set header */ + p.tag(sp) + p.i64(OP_size_const, 5) + + /* check for nil slice */ + i := p.pc() + p.add(OP_if_nil) + + /* element is trivially measuable */ + if nb > 0 { + p.dyn(OP_size_dyn, abi.PtrSize, int64(nb)) + p.pin(i) + return + } + + /* complex lists or sets */ + j := p.pc() + p.add(OP_list_if_empty) + p.add(OP_make_state) + p.add(OP_list_begin) + k := p.pc() + p.add(OP_goto) + r := p.pc() + p.i64(OP_seek, int64(et.S.Size())) + p.pin(k) + self.measureItem(p, sp+1, et, startpc) + p.add(OP_list_decr) + p.jmp(OP_list_if_next, r) + p.add(OP_drop_state) + p.pin(i) + p.pin(j) } func (self *Compiler) measureItem(p *Program, sp int, vt *defs.Type, startpc int) { - tag := vt.T - elem := vt.V - - /* special handling for pointers */ - if tag != defs.T_pointer { - self.measure(p, sp, vt, startpc) - return - } - - /* must be pointer struct at this point */ - if elem.T != defs.T_struct { - panic("fatal: non-struct pointers within container elements") - } - - /* always add the STOP field for structs */ - i := p.pc() - p.tag(sp) - p.add(OP_if_nil) - p.add(OP_make_state) - p.add(OP_deref) - self.measure(p, sp + 1, elem, startpc) - p.add(OP_drop_state) - j := p.pc() - p.add(OP_goto) - p.pin(i) - p.i64(OP_size_const, 1) - p.pin(j) + tag := vt.T + elem := vt.V + + /* special handling for pointers */ + if tag != defs.T_pointer { + self.measure(p, sp, vt, startpc) + return + } + + /* must be pointer struct at this point */ + if elem.T != defs.T_struct { + panic("fatal: non-struct pointers within container elements") + } + + /* always add the STOP field for structs */ + i := p.pc() + p.tag(sp) + p.add(OP_if_nil) + p.add(OP_make_state) + p.add(OP_deref) + self.measure(p, sp+1, elem, startpc) + p.add(OP_drop_state) + j := p.pc() + p.add(OP_goto) + p.pin(i) + p.i64(OP_size_const, 1) + p.pin(j) } func (self *Compiler) measureStruct(p *Program, sp int, vt *defs.Type, startpc int) { - var err error - var fvs []defs.Field - - /* struct is trivially measuable */ - if nb := defs.GetSize(vt.S); nb > 0 { - p.i64(OP_size_const, int64(nb)) - return - } - - /* resolve the field */ - if fvs, err = defs.ResolveFields(vt.S); err != nil { - panic(err) - } - - /* empty structs */ - if len(fvs) == 0 { - p.i64(OP_size_const, 4) - return - } - - /* 1-byte stop field */ - p.tag(sp) - p.i64(OP_size_const, 1) - - /* measure every field */ - for _, fv := range fvs { - p.i64(OP_seek, int64(fv.F)) - self.measureField(p, sp + 1, fv, startpc) - p.i64(OP_seek, -int64(fv.F)) - } + var err error + var fvs []defs.Field + + /* struct is trivially measuable */ + if nb := defs.GetSize(vt.S); nb > 0 { + p.i64(OP_size_const, int64(nb)) + return + } + + /* resolve the field */ + if fvs, err = defs.ResolveFields(vt.S); err != nil { + panic(err) + } + + /* empty structs */ + if len(fvs) == 0 { + p.i64(OP_size_const, 4) + return + } + + /* 1-byte stop field */ + p.tag(sp) + p.i64(OP_size_const, 1) + + /* measure every field */ + for _, fv := range fvs { + p.i64(OP_seek, int64(fv.F)) + self.measureField(p, sp+1, fv, startpc) + p.i64(OP_seek, -int64(fv.F)) + } } func (self *Compiler) measureField(p *Program, sp int, fv defs.Field, startpc int) { - switch fv.Type.T { - default: { - panic("fatal: invalid field type: " + fv.Type.String()) - } - - /* non-pointer types */ - case defs.T_bool : fallthrough - case defs.T_i8 : fallthrough - case defs.T_double : fallthrough - case defs.T_i16 : fallthrough - case defs.T_i32 : fallthrough - case defs.T_i64 : fallthrough - case defs.T_string : fallthrough - case defs.T_enum : fallthrough - case defs.T_binary : { - if fv.Default.IsValid() && fv.Spec == defs.Optional { - self.measureStructDefault(p, sp, fv, startpc) - } else { - self.measureStructRequired(p, sp, fv, startpc) - } - } - - /* struct types, only available in hand-written structs */ - case defs.T_struct: { - self.measureStructRequired(p, sp, fv, startpc) - } - - /* sequencial types */ - case defs.T_map : fallthrough - case defs.T_set : fallthrough - case defs.T_list : { - if fv.Spec == defs.Optional { - self.measureStructIterable(p, sp, fv, startpc) - } else { - self.measureStructRequired(p, sp, fv, startpc) - } - } - - /* pointers */ - case defs.T_pointer: { - if fv.Spec == defs.Optional { - self.measureStructOptional(p, sp, fv, startpc) - } else if fv.Type.V.T == defs.T_struct { - self.measureStructPointer(p, sp, fv, startpc) - } else { - panic("fatal: non-optional non-struct pointers") - } - } - } + switch fv.Type.T { + default: + { + panic("fatal: invalid field type: " + fv.Type.String()) + } + + /* non-pointer types */ + case defs.T_bool: + fallthrough + case defs.T_i8: + fallthrough + case defs.T_double: + fallthrough + case defs.T_i16: + fallthrough + case defs.T_i32: + fallthrough + case defs.T_i64: + fallthrough + case defs.T_string: + fallthrough + case defs.T_enum: + fallthrough + case defs.T_binary: + { + if fv.Default.IsValid() && fv.Spec == defs.Optional { + self.measureStructDefault(p, sp, fv, startpc) + } else { + self.measureStructRequired(p, sp, fv, startpc) + } + } + + /* struct types, only available in hand-written structs */ + case defs.T_struct: + { + self.measureStructRequired(p, sp, fv, startpc) + } + + /* sequential types */ + case defs.T_map: + fallthrough + case defs.T_set: + fallthrough + case defs.T_list: + { + if fv.Spec == defs.Optional { + self.measureStructIterable(p, sp, fv, startpc) + } else { + self.measureStructRequired(p, sp, fv, startpc) + } + } + + /* pointers */ + case defs.T_pointer: + { + if fv.Spec == defs.Optional { + self.measureStructOptional(p, sp, fv, startpc) + } else if fv.Type.V.T == defs.T_struct { + self.measureStructPointer(p, sp, fv, startpc) + } else { + panic("fatal: non-optional non-struct pointers") + } + } + } } func (self *Compiler) measureStructDefault(p *Program, sp int, fv defs.Field, startpc int) { - i := p.pc() - t := fv.Type.T - - /* check for default values */ - switch t { - case defs.T_bool : p.dyn(OP_if_eq_imm, 1, bool2i64(fv.Default.Bool())) - case defs.T_i8 : p.dyn(OP_if_eq_imm, 1, fv.Default.Int()) - case defs.T_double : p.dyn(OP_if_eq_imm, 8, int64(math.Float64bits(fv.Default.Float()))) - case defs.T_i16 : p.dyn(OP_if_eq_imm, 2, fv.Default.Int()) - case defs.T_i32 : p.dyn(OP_if_eq_imm, 4, fv.Default.Int()) - case defs.T_i64 : p.dyn(OP_if_eq_imm, 8, fv.Default.Int()) - case defs.T_string : p.str(OP_if_eq_str, fv.Default.String()) - case defs.T_enum : p.dyn(OP_if_eq_imm, 4, fv.Default.Int()) - case defs.T_binary : p.str(OP_if_eq_str, mem2str(fv.Default.Bytes())) - default : panic("unreachable") - } - - /* measure if it's not the default value */ - p.i64(OP_size_const, 3) - self.measure(p, sp, fv.Type, startpc) - p.pin(i) + i := p.pc() + t := fv.Type.T + + /* check for default values */ + switch t { + case defs.T_bool: + p.dyn(OP_if_eq_imm, 1, bool2i64(fv.Default.Bool())) + case defs.T_i8: + p.dyn(OP_if_eq_imm, 1, fv.Default.Int()) + case defs.T_double: + p.dyn(OP_if_eq_imm, 8, int64(math.Float64bits(fv.Default.Float()))) + case defs.T_i16: + p.dyn(OP_if_eq_imm, 2, fv.Default.Int()) + case defs.T_i32: + p.dyn(OP_if_eq_imm, 4, fv.Default.Int()) + case defs.T_i64: + p.dyn(OP_if_eq_imm, 8, fv.Default.Int()) + case defs.T_string: + p.str(OP_if_eq_str, fv.Default.String()) + case defs.T_enum: + p.dyn(OP_if_eq_imm, 4, fv.Default.Int()) + case defs.T_binary: + p.str(OP_if_eq_str, mem2str(fv.Default.Bytes())) + default: + panic("unreachable") + } + + /* measure if it's not the default value */ + p.i64(OP_size_const, 3) + self.measure(p, sp, fv.Type, startpc) + p.pin(i) } func (self *Compiler) measureStructPointer(p *Program, sp int, fv defs.Field, startpc int) { - i := p.pc() - p.add(OP_if_nil) - p.i64(OP_size_const, 3) - p.add(OP_make_state) - p.add(OP_deref) - self.measure(p, sp + 1, fv.Type.V, startpc) - p.add(OP_drop_state) - j := p.pc() - p.add(OP_goto) - p.pin(i) - p.i64(OP_size_const, 4) - p.pin(j) + i := p.pc() + p.add(OP_if_nil) + p.i64(OP_size_const, 3) + p.add(OP_make_state) + p.add(OP_deref) + self.measure(p, sp+1, fv.Type.V, startpc) + p.add(OP_drop_state) + j := p.pc() + p.add(OP_goto) + p.pin(i) + p.i64(OP_size_const, 4) + p.pin(j) } func (self *Compiler) measureStructIterable(p *Program, sp int, fv defs.Field, startpc int) { - i := p.pc() - p.add(OP_if_nil) - p.i64(OP_size_const, 3) - self.measure(p, sp, fv.Type, startpc) - p.pin(i) + i := p.pc() + p.add(OP_if_nil) + p.i64(OP_size_const, 3) + self.measure(p, sp, fv.Type, startpc) + p.pin(i) } func (self *Compiler) measureStructOptional(p *Program, sp int, fv defs.Field, startpc int) { - i := p.pc() - p.add(OP_if_nil) - p.i64(OP_size_const, 3) - p.add(OP_make_state) - p.add(OP_deref) - self.measure(p, sp + 1, fv.Type.V, startpc) - p.add(OP_drop_state) - p.pin(i) + i := p.pc() + p.add(OP_if_nil) + p.i64(OP_size_const, 3) + p.add(OP_make_state) + p.add(OP_deref) + self.measure(p, sp+1, fv.Type.V, startpc) + p.add(OP_drop_state) + p.pin(i) } func (self *Compiler) measureStructRequired(p *Program, sp int, fv defs.Field, startpc int) { - p.i64(OP_size_const, 3) - self.measure(p, sp, fv.Type, startpc) + p.i64(OP_size_const, 3) + self.measure(p, sp, fv.Type, startpc) } diff --git a/internal/binary/encoder/compiler_test.go b/internal/binary/encoder/compiler_test.go index 5e76526..322e75c 100644 --- a/internal/binary/encoder/compiler_test.go +++ b/internal/binary/encoder/compiler_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,37 +17,37 @@ package encoder import ( - `reflect` - `testing` + "reflect" + "testing" - `github.com/stretchr/testify/require` + "github.com/stretchr/testify/require" ) type CompilerTest struct { - A bool `frugal:"0,default,bool"` - B int8 `frugal:"1,default,i8"` - C float64 `frugal:"2,default,double"` - D int16 `frugal:"3,default,i16"` - E int32 `frugal:"4,default,i32"` - F int64 `frugal:"5,default,i64"` - G string `frugal:"6,default,string"` - H CompilerTestSubStruct `frugal:"7,default,CompilerTestSubStruct"` - I *CompilerTestSubStruct `frugal:"8,default,CompilerTestSubStruct"` - J map[string]int `frugal:"9,default,map"` - K []string `frugal:"10,default,set"` - L []string `frugal:"11,default,list"` - M []byte `frugal:"12,default,binary"` - N []int8 `frugal:"13,default,set"` - O []int8 `frugal:"14,default,list"` + A bool `frugal:"0,default,bool"` + B int8 `frugal:"1,default,i8"` + C float64 `frugal:"2,default,double"` + D int16 `frugal:"3,default,i16"` + E int32 `frugal:"4,default,i32"` + F int64 `frugal:"5,default,i64"` + G string `frugal:"6,default,string"` + H CompilerTestSubStruct `frugal:"7,default,CompilerTestSubStruct"` + I *CompilerTestSubStruct `frugal:"8,default,CompilerTestSubStruct"` + J map[string]int `frugal:"9,default,map"` + K []string `frugal:"10,default,set"` + L []string `frugal:"11,default,list"` + M []byte `frugal:"12,default,binary"` + N []int8 `frugal:"13,default,set"` + O []int8 `frugal:"14,default,list"` } type CompilerTestSubStruct struct { - X int `frugal:"0,default,i64"` - Y *CompilerTestSubStruct `frugal:"1,default,CompilerTestSubStruct"` + X int `frugal:"0,default,i64"` + Y *CompilerTestSubStruct `frugal:"1,default,CompilerTestSubStruct"` } func TestCompiler_Compile(t *testing.T) { - p, err := CreateCompiler().Compile(reflect.TypeOf(CompilerTest{})) - require.NoError(t, err) - println(p.Disassemble()) + p, err := CreateCompiler().Compile(reflect.TypeOf(CompilerTest{})) + require.NoError(t, err) + println(p.Disassemble()) } diff --git a/internal/binary/encoder/encoder.go b/internal/binary/encoder/encoder.go index 87ca9b6..ed99abe 100644 --- a/internal/binary/encoder/encoder.go +++ b/internal/binary/encoder/encoder.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,121 +17,121 @@ package encoder import ( - `fmt` - `sync/atomic` - `unsafe` - - `github.com/cloudwego/frugal/internal/opts` - `github.com/cloudwego/frugal/internal/rt` - `github.com/cloudwego/frugal/internal/utils` - `github.com/cloudwego/frugal/iov` + "fmt" + "sync/atomic" + "unsafe" + + "github.com/cloudwego/frugal/internal/opts" + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/frugal/internal/utils" + "github.com/cloudwego/frugal/iov" ) -type Encoder func ( - buf unsafe.Pointer, - len int, - mem iov.BufferWriter, - p unsafe.Pointer, - rs *RuntimeState, - st int, +type Encoder func( + buf unsafe.Pointer, + len int, + mem iov.BufferWriter, + p unsafe.Pointer, + rs *RuntimeState, + st int, ) (int, error) var ( - HitCount uint64 = 0 - MissCount uint64 = 0 - TypeCount uint64 = 0 + HitCount uint64 = 0 + MissCount uint64 = 0 + TypeCount uint64 = 0 ) var ( - programCache = utils.CreateProgramCache() + programCache = utils.CreateProgramCache() ) func encode(vt *rt.GoType, buf unsafe.Pointer, len int, mem iov.BufferWriter, p unsafe.Pointer, rs *RuntimeState, st int) (int, error) { - if enc, err := resolve(vt); err != nil { - return -1, err - } else { - return enc(buf, len, mem, p, rs, st) - } + if enc, err := resolve(vt); err != nil { + return -1, err + } else { + return enc(buf, len, mem, p, rs, st) + } } func resolve(vt *rt.GoType) (Encoder, error) { - var err error - var val interface{} - - /* fast-path: type is cached */ - if val = programCache.Get(vt); val != nil { - atomic.AddUint64(&HitCount, 1) - return val.(Encoder), nil - } - - /* record the cache miss, and compile the type */ - atomic.AddUint64(&MissCount, 1) - val, err = programCache.Compute(vt, compile) - - /* check for errors */ - if err != nil { - return nil, err - } - - /* record the successful compilation */ - atomic.AddUint64(&TypeCount, 1) - return val.(Encoder), nil + var err error + var val interface{} + + /* fast-path: type is cached */ + if val = programCache.Get(vt); val != nil { + atomic.AddUint64(&HitCount, 1) + return val.(Encoder), nil + } + + /* record the cache miss, and compile the type */ + atomic.AddUint64(&MissCount, 1) + val, err = programCache.Compute(vt, compile) + + /* check for errors */ + if err != nil { + return nil, err + } + + /* record the successful compilation */ + atomic.AddUint64(&TypeCount, 1) + return val.(Encoder), nil } func compile(vt *rt.GoType) (interface{}, error) { - if pp, err := CreateCompiler().CompileAndFree(vt.Pack()); err != nil { - return nil, err - } else { - return Link(Translate(pp)), nil - } + if pp, err := CreateCompiler().CompileAndFree(vt.Pack()); err != nil { + return nil, err + } else { + return Link(Translate(pp)), nil + } } func mkcompile(opts opts.Options) func(*rt.GoType) (interface{}, error) { - return func(vt *rt.GoType) (interface{}, error) { - if pp, err := CreateCompiler().Apply(opts).CompileAndFree(vt.Pack()); err != nil { - return nil, err - } else { - return Link(Translate(pp)), nil - } - } + return func(vt *rt.GoType) (interface{}, error) { + if pp, err := CreateCompiler().Apply(opts).CompileAndFree(vt.Pack()); err != nil { + return nil, err + } else { + return Link(Translate(pp)), nil + } + } } func Pretouch(vt *rt.GoType, opts opts.Options) error { - if programCache.Get(vt) != nil { - return nil - } else if _, err := programCache.Compute(vt, mkcompile(opts)); err != nil { - return err - } else { - atomic.AddUint64(&TypeCount, 1) - return nil - } + if programCache.Get(vt) != nil { + return nil + } else if _, err := programCache.Compute(vt, mkcompile(opts)); err != nil { + return err + } else { + atomic.AddUint64(&TypeCount, 1) + return nil + } } func EncodedSize(val interface{}) int { - if ret, err := EncodeObject(nil, nil, val); err != nil { - panic(fmt.Errorf("frugal: cannot measure encoded size: %w", err)) - } else { - return ret - } + if ret, err := EncodeObject(nil, nil, val); err != nil { + panic(fmt.Errorf("frugal: cannot measure encoded size: %w", err)) + } else { + return ret + } } func EncodeObject(buf []byte, mem iov.BufferWriter, val interface{}) (ret int, err error) { - rst := newRuntimeState() - efv := rt.UnpackEface(val) - out := (*rt.GoSlice)(unsafe.Pointer(&buf)) - - /* check for indirect types */ - if efv.Type.IsIndirect() { - ret, err = encode(efv.Type, out.Ptr, out.Len, mem, efv.Value, rst, 0) - } else { - /* avoid an extra mallocgc which is expensive for small objects */ - rst.Val = efv.Value - ret, err = encode(efv.Type, out.Ptr, out.Len, mem, unsafe.Pointer(&rst.Val), rst, 0) - /* remove reference to avoid leak since rst will be reused */ - rst.Val = nil - } - - /* return the state into pool */ - freeRuntimeState(rst) - return + rst := newRuntimeState() + efv := rt.UnpackEface(val) + out := (*rt.GoSlice)(unsafe.Pointer(&buf)) + + /* check for indirect types */ + if efv.Type.IsIndirect() { + ret, err = encode(efv.Type, out.Ptr, out.Len, mem, efv.Value, rst, 0) + } else { + /* avoid an extra mallocgc which is expensive for small objects */ + rst.Val = efv.Value + ret, err = encode(efv.Type, out.Ptr, out.Len, mem, unsafe.Pointer(&rst.Val), rst, 0) + /* remove reference to avoid leak since rst will be reused */ + rst.Val = nil + } + + /* return the state into pool */ + freeRuntimeState(rst) + return } diff --git a/internal/binary/encoder/encoder_test.go b/internal/binary/encoder/encoder_test.go index 618409e..b584d69 100644 --- a/internal/binary/encoder/encoder_test.go +++ b/internal/binary/encoder/encoder_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,108 +17,108 @@ package encoder import ( - `bytes` - `encoding/base64` - `testing` + "bytes" + "encoding/base64" + "testing" - `github.com/davecgh/go-spew/spew` - `github.com/stretchr/testify/require` + "github.com/davecgh/go-spew/spew" + "github.com/stretchr/testify/require" ) func TestEncoder_Encode(t *testing.T) { - v := TranslatorTestStruct { - A: true, - B: 0x12, - C: 12.34, - D: 0x3456, - E: 0x12345678, - F: 0x66778899aabbccdd, - G: "hello, world", - H: []byte("testbytebuffer"), - I: []int32{0x11223344, 0x55667788, 3, 4, 5}, - J: map[string]string{"asdf": "qwer", "zxcv": "hjkl"}, - K: map[string]*TranslatorTestStruct{ - "foo": { - B: -1, - }, - }, - L: &(&struct{ x bool }{true}).x, - M: &(&struct{ x int8 }{0x1}).x, - N: &(&struct{ x float64 }{0x12345678}).x, - O: &(&struct{ x int16 }{0x1234}).x, - P: &(&struct{ x int32 }{0x123456}).x, - Q: &(&struct{ x int64 }{0x12345678}).x, - } - nb := EncodedSize(v) - println("Estimated Size:", nb) - buf := make([]byte, nb) - ret, err := EncodeObject(buf, nil, v) - if err != nil { - println("Buffer Shortage:", ret - nb) - require.NoError(t, err) - } - buf = buf[:ret] - spew.Dump(buf) - mm := bytes.NewBufferString("\x80\x01\x00\x01\x00\x00\x00\x01a\x00\x00\x00\x00") - mm.Write(buf) - println("Base64 Encoded Message:", base64.StdEncoding.EncodeToString(mm.Bytes())) + v := TranslatorTestStruct{ + A: true, + B: 0x12, + C: 12.34, + D: 0x3456, + E: 0x12345678, + F: 0x66778899aabbccdd, + G: "hello, world", + H: []byte("testbytebuffer"), + I: []int32{0x11223344, 0x55667788, 3, 4, 5}, + J: map[string]string{"asdf": "qwer", "zxcv": "hjkl"}, + K: map[string]*TranslatorTestStruct{ + "foo": { + B: -1, + }, + }, + L: &(&struct{ x bool }{true}).x, + M: &(&struct{ x int8 }{0x1}).x, + N: &(&struct{ x float64 }{0x12345678}).x, + O: &(&struct{ x int16 }{0x1234}).x, + P: &(&struct{ x int32 }{0x123456}).x, + Q: &(&struct{ x int64 }{0x12345678}).x, + } + nb := EncodedSize(v) + println("Estimated Size:", nb) + buf := make([]byte, nb) + ret, err := EncodeObject(buf, nil, v) + if err != nil { + println("Buffer Shortage:", ret-nb) + require.NoError(t, err) + } + buf = buf[:ret] + spew.Dump(buf) + mm := bytes.NewBufferString("\x80\x01\x00\x01\x00\x00\x00\x01a\x00\x00\x00\x00") + mm.Write(buf) + println("Base64 Encoded Message:", base64.StdEncoding.EncodeToString(mm.Bytes())) } type StructSeekTest struct { - H StructSeekTestSubStruct `frugal:"0,default,StructSeekTestSubStruct"` - O []int8 `frugal:"1,default,list"` + H StructSeekTestSubStruct `frugal:"0,default,StructSeekTestSubStruct"` + O []int8 `frugal:"1,default,list"` } type StructSeekTestSubStruct struct { - X int `frugal:"0,default,i64"` - Y *int `frugal:"1,optional,i64"` + X int `frugal:"0,default,i64"` + Y *int `frugal:"1,optional,i64"` } func TestEncoder_StructSeek(t *testing.T) { - c := StructSeekTest{O: []int8{-61}} - buf := make([]byte, EncodedSize(c)) - _, _ = EncodeObject(buf, nil, c) + c := StructSeekTest{O: []int8{-61}} + buf := make([]byte, EncodedSize(c)) + _, _ = EncodeObject(buf, nil, c) } type ListI8UniqueOuter struct { - Field0 *ListI8UniqueInner `frugal:"0,default,ListI8UniqueInner"` + Field0 *ListI8UniqueInner `frugal:"0,default,ListI8UniqueInner"` } type ListI8UniqueInner struct { - Field12336 []int8 `frugal:"12336,default,set"` + Field12336 []int8 `frugal:"12336,default,set"` } func TestEncoder_ListI8Unique(t *testing.T) { - want := &ListI8UniqueOuter{&ListI8UniqueInner{Field12336: []int8{-61, 67}}} - buf := make([]byte, EncodedSize(want)) - _, err := EncodeObject(buf, nil, want) - if err != nil { - t.Fatal(err) - } + want := &ListI8UniqueOuter{&ListI8UniqueInner{Field12336: []int8{-61, 67}}} + buf := make([]byte, EncodedSize(want)) + _, err := EncodeObject(buf, nil, want) + if err != nil { + t.Fatal(err) + } } type Foo struct { - Message string `frugal:"1,required"` + Message string `frugal:"1,required"` } type ContainerPointerElement struct { - Data map[int64]*Foo `frugal:"1,required"` + Data map[int64]*Foo `frugal:"1,required"` } func TestEncoder_ContainerPointerElement(t *testing.T) { - want := &ContainerPointerElement{map[int64]*Foo{0x1234: nil}} - nb := EncodedSize(want) - println("encoded size =", nb) - buf := make([]byte, nb) - nx, err := EncodeObject(buf, nil, want) - if err != nil { - t.Fatal(err) - } - spew.Dump(buf[:nx]) - require.Equal(t, []byte{ - 0x0d, 0x00, 0x01, 0x0a, 0x0c, 0x00, 0x00, 0x00, 0x01, // field 1: map, len = 1 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x34, // key = (i64) 0x1234 - 0x00, // value = (struct) {} - 0x00, // end - }, buf[:nx]) + want := &ContainerPointerElement{map[int64]*Foo{0x1234: nil}} + nb := EncodedSize(want) + println("encoded size =", nb) + buf := make([]byte, nb) + nx, err := EncodeObject(buf, nil, want) + if err != nil { + t.Fatal(err) + } + spew.Dump(buf[:nx]) + require.Equal(t, []byte{ + 0x0d, 0x00, 0x01, 0x0a, 0x0c, 0x00, 0x00, 0x00, 0x01, // field 1: map, len = 1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x34, // key = (i64) 0x1234 + 0x00, // value = (struct) {} + 0x00, // end + }, buf[:nx]) } diff --git a/internal/binary/encoder/hash.go b/internal/binary/encoder/hash.go index 007137d..4ec3616 100644 --- a/internal/binary/encoder/hash.go +++ b/internal/binary/encoder/hash.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ package encoder import ( - `unsafe` + "unsafe" ) //go:noescape @@ -26,19 +26,19 @@ import ( func strhash(p unsafe.Pointer, h uintptr) uintptr func mkhash(v int) int { - if v != 0 { - return v - } else { - return 1 - } + if v != 0 { + return v + } else { + return 1 + } } func hash64(h uint64) int { - h ^= h >> 33 - h *= 0x9e3779b97f4a7c15 - return mkhash(int(h &^ (1 << 63))) + h ^= h >> 33 + h *= 0x9e3779b97f4a7c15 + return mkhash(int(h &^ (1 << 63))) } func hashstr(p unsafe.Pointer) int { - return mkhash(int(strhash(p, 0) &^ (1 << 63))) + return mkhash(int(strhash(p, 0) &^ (1 << 63))) } diff --git a/internal/binary/encoder/linker.go b/internal/binary/encoder/linker.go index 1c6e618..8cb530c 100644 --- a/internal/binary/encoder/linker.go +++ b/internal/binary/encoder/linker.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,31 +17,31 @@ package encoder import ( - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/utils` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/utils" ) type Linker interface { - Link(p hir.Program) Encoder + Link(p hir.Program) Encoder } var ( - linker Linker - F_encode *hir.CallHandle + linker Linker + F_encode *hir.CallHandle ) func init() { - F_encode = hir.RegisterGCall(encode, emu_gcall_encode) + F_encode = hir.RegisterGCall(encode, emu_gcall_encode) } func Link(p hir.Program) Encoder { - if linker == nil || utils.ForceEmulator { - return link_emu(p) - } else { - return linker.Link(p) - } + if linker == nil || utils.ForceEmulator { + return link_emu(p) + } else { + return linker.Link(p) + } } func SetLinker(v Linker) { - linker = v + linker = v } diff --git a/internal/binary/encoder/linker_amd64.go b/internal/binary/encoder/linker_amd64.go index b15d489..d6a16d1 100644 --- a/internal/binary/encoder/linker_amd64.go +++ b/internal/binary/encoder/linker_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,23 +17,23 @@ package encoder import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/atm/pgen` - `github.com/cloudwego/frugal/internal/loader` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/atm/pgen" + "github.com/cloudwego/frugal/internal/loader" ) type ( - LinkerAMD64 struct{} + LinkerAMD64 struct{} ) func init() { - SetLinker(new(LinkerAMD64)) + SetLinker(new(LinkerAMD64)) } func (LinkerAMD64) Link(p hir.Program) Encoder { - fn := pgen.CreateCodeGen((Encoder)(nil)).Generate(p, 0) - fp := loader.Loader(fn.Code).Load("encoder", fn.Frame) - return *(*Encoder)(unsafe.Pointer(&fp)) -} \ No newline at end of file + fn := pgen.CreateCodeGen((Encoder)(nil)).Generate(p, 0) + fp := loader.Loader(fn.Code).Load("encoder", fn.Frame) + return *(*Encoder)(unsafe.Pointer(&fp)) +} diff --git a/internal/binary/encoder/linker_arm64.go b/internal/binary/encoder/linker_arm64.go index b2c16d8..e952b2d 100644 --- a/internal/binary/encoder/linker_arm64.go +++ b/internal/binary/encoder/linker_arm64.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 ByteDance Inc. + * Copyright 2024 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/internal/binary/encoder/linker_emu.go b/internal/binary/encoder/linker_emu.go index 9310a01..8c37945 100644 --- a/internal/binary/encoder/linker_emu.go +++ b/internal/binary/encoder/linker_emu.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,66 +17,66 @@ package encoder import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/atm/emu` - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` - `github.com/cloudwego/frugal/iov` + "github.com/cloudwego/frugal/internal/atm/emu" + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/frugal/iov" ) func link_emu(prog hir.Program) Encoder { - return func(buf unsafe.Pointer, len int, mem iov.BufferWriter, p unsafe.Pointer, rs *RuntimeState, st int) (ret int, err error) { - ctx := emu.LoadProgram(prog) - exc := (*rt.GoIface)(unsafe.Pointer(&err)) - iop := (*rt.GoIface)(unsafe.Pointer(&mem)) - ctx.Ap(0,buf) - ctx.Au(1, uint64(len)) - ctx.Ap(2, unsafe.Pointer(iop.Itab)) - ctx.Ap(3, iop.Value) - ctx.Ap(4, p) - ctx.Ap(5, unsafe.Pointer(rs)) - ctx.Au(6, uint64(st)) - ctx.Run() - ret = int(ctx.Ru(0)) - exc.Itab = (*rt.GoItab)(ctx.Rp(1)) - exc.Value = ctx.Rp(2) - ctx.Free() - return - } + return func(buf unsafe.Pointer, len int, mem iov.BufferWriter, p unsafe.Pointer, rs *RuntimeState, st int) (ret int, err error) { + ctx := emu.LoadProgram(prog) + exc := (*rt.GoIface)(unsafe.Pointer(&err)) + iop := (*rt.GoIface)(unsafe.Pointer(&mem)) + ctx.Ap(0, buf) + ctx.Au(1, uint64(len)) + ctx.Ap(2, unsafe.Pointer(iop.Itab)) + ctx.Ap(3, iop.Value) + ctx.Ap(4, p) + ctx.Ap(5, unsafe.Pointer(rs)) + ctx.Au(6, uint64(st)) + ctx.Run() + ret = int(ctx.Ru(0)) + exc.Itab = (*rt.GoItab)(ctx.Rp(1)) + exc.Value = ctx.Rp(2) + ctx.Free() + return + } } func emu_wbuf(ctx hir.CallContext, i int) (v iov.BufferWriter) { - (*rt.GoIface)(unsafe.Pointer(&v)).Itab = (*rt.GoItab)(ctx.Ap(i)) - (*rt.GoIface)(unsafe.Pointer(&v)).Value = ctx.Ap(i + 1) - return + (*rt.GoIface)(unsafe.Pointer(&v)).Itab = (*rt.GoItab)(ctx.Ap(i)) + (*rt.GoIface)(unsafe.Pointer(&v)).Value = ctx.Ap(i + 1) + return } func emu_setret(ctx hir.CallContext) func(int, error) { - return func(ret int, err error) { - vv := (*rt.GoIface)(unsafe.Pointer(&err)) - ctx.Ru(0, uint64(ret)) - ctx.Rp(1, unsafe.Pointer(vv.Itab)) - ctx.Rp(2, vv.Value) - } + return func(ret int, err error) { + vv := (*rt.GoIface)(unsafe.Pointer(&err)) + ctx.Ru(0, uint64(ret)) + ctx.Rp(1, unsafe.Pointer(vv.Itab)) + ctx.Rp(2, vv.Value) + } } func emu_encode(ctx hir.CallContext) (int, error) { - return encode( - (*rt.GoType)(ctx.Ap(0)), - ctx.Ap(1), - int(ctx.Au(2)), - emu_wbuf(ctx, 3), - ctx.Ap(5), - (*RuntimeState)(ctx.Ap(6)), - int(ctx.Au(7)), - ) + return encode( + (*rt.GoType)(ctx.Ap(0)), + ctx.Ap(1), + int(ctx.Au(2)), + emu_wbuf(ctx, 3), + ctx.Ap(5), + (*RuntimeState)(ctx.Ap(6)), + int(ctx.Au(7)), + ) } func emu_gcall_encode(ctx hir.CallContext) { - if !ctx.Verify("**i****i", "i**") { - panic("invalid encode call") - } else { - emu_setret(ctx)(emu_encode(ctx)) - } + if !ctx.Verify("**i****i", "i**") { + panic("invalid encode call") + } else { + emu_setret(ctx)(emu_encode(ctx)) + } } diff --git a/internal/binary/encoder/mapiter.go b/internal/binary/encoder/mapiter.go index b80be29..c713ee0 100644 --- a/internal/binary/encoder/mapiter.go +++ b/internal/binary/encoder/mapiter.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,10 +17,10 @@ package encoder import ( - _ `unsafe` + _ "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" ) //go:noescape @@ -35,11 +35,11 @@ func mapiterinit(t *rt.GoMapType, h *rt.GoMap, it *rt.GoMapIterator) //go:nosplit func mapiterstart(t *rt.GoMapType, h *rt.GoMap, it *rt.GoMapIterator) { - *it = rt.GoMapIterator{} - mapiterinit(t, h, it) + *it = rt.GoMapIterator{} + mapiterinit(t, h, it) } var ( - F_mapiternext = hir.RegisterGCall(mapiternext, emu_gcall_mapiternext) - F_mapiterstart = hir.RegisterGCall(mapiterstart, emu_gcall_mapiterstart) + F_mapiternext = hir.RegisterGCall(mapiternext, emu_gcall_mapiternext) + F_mapiterstart = hir.RegisterGCall(mapiterstart, emu_gcall_mapiterstart) ) diff --git a/internal/binary/encoder/mapiter_emu.go b/internal/binary/encoder/mapiter_emu.go index a69e1c1..62613f2 100644 --- a/internal/binary/encoder/mapiter_emu.go +++ b/internal/binary/encoder/mapiter_emu.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,22 +17,22 @@ package encoder import ( - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" ) func emu_gcall_mapiternext(ctx hir.CallContext) { - if !ctx.Verify("*", "") { - panic("invalid mapiternext call") - } else { - mapiternext((*rt.GoMapIterator)(ctx.Ap(0))) - } + if !ctx.Verify("*", "") { + panic("invalid mapiternext call") + } else { + mapiternext((*rt.GoMapIterator)(ctx.Ap(0))) + } } func emu_gcall_mapiterstart(ctx hir.CallContext) { - if !ctx.Verify("***", "") { - panic("invalid mapiterstart call") - } else { - mapiterstart((*rt.GoMapType)(ctx.Ap(0)), (*rt.GoMap)(ctx.Ap(1)), (*rt.GoMapIterator)(ctx.Ap(2))) - } + if !ctx.Verify("***", "") { + panic("invalid mapiterstart call") + } else { + mapiterstart((*rt.GoMapType)(ctx.Ap(0)), (*rt.GoMap)(ctx.Ap(1)), (*rt.GoMapIterator)(ctx.Ap(2))) + } } diff --git a/internal/binary/encoder/opcode.go b/internal/binary/encoder/opcode.go index 01c3d9e..3c8738c 100644 --- a/internal/binary/encoder/opcode.go +++ b/internal/binary/encoder/opcode.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,103 +17,103 @@ package encoder import ( - `fmt` + "fmt" ) type OpCode uint8 const ( - OP_size_check OpCode = iota - OP_size_const - OP_size_dyn - OP_size_map - OP_size_defer - OP_byte - OP_word - OP_long - OP_quad - OP_sint - OP_length - OP_memcpy_be - OP_seek - OP_deref - OP_defer - OP_map_len - OP_map_key - OP_map_next - OP_map_value - OP_map_begin - OP_map_if_next - OP_map_if_empty - OP_list_decr - OP_list_begin - OP_list_if_next - OP_list_if_empty - OP_unique - OP_goto - OP_if_nil - OP_if_hasbuf - OP_if_eq_imm - OP_if_eq_str - OP_make_state - OP_drop_state - OP_halt + OP_size_check OpCode = iota + OP_size_const + OP_size_dyn + OP_size_map + OP_size_defer + OP_byte + OP_word + OP_long + OP_quad + OP_sint + OP_length + OP_memcpy_be + OP_seek + OP_deref + OP_defer + OP_map_len + OP_map_key + OP_map_next + OP_map_value + OP_map_begin + OP_map_if_next + OP_map_if_empty + OP_list_decr + OP_list_begin + OP_list_if_next + OP_list_if_empty + OP_unique + OP_goto + OP_if_nil + OP_if_hasbuf + OP_if_eq_imm + OP_if_eq_str + OP_make_state + OP_drop_state + OP_halt ) -var _OpNames = [256]string { - OP_size_check : "size_check", - OP_size_const : "size_const", - OP_size_dyn : "size_dyn", - OP_size_map : "size_map", - OP_size_defer : "size_defer", - OP_byte : "byte", - OP_word : "word", - OP_long : "long", - OP_quad : "quad", - OP_sint : "sint", - OP_length : "length", - OP_memcpy_be : "memcpy_be", - OP_seek : "seek", - OP_deref : "deref", - OP_defer : "defer", - OP_map_len : "map_len", - OP_map_key : "map_key", - OP_map_next : "map_next", - OP_map_value : "map_value", - OP_map_begin : "map_begin", - OP_map_if_next : "map_if_next", - OP_map_if_empty : "map_if_empty", - OP_list_decr : "list_decr", - OP_list_begin : "list_begin", - OP_list_if_next : "list_if_next", - OP_list_if_empty : "list_if_empty", - OP_unique : "unique", - OP_goto : "goto", - OP_if_nil : "if_nil", - OP_if_hasbuf : "if_hasbuf", - OP_if_eq_imm : "if_eq_imm", - OP_if_eq_str : "if_eq_str", - OP_make_state : "make_state", - OP_drop_state : "drop_state", - OP_halt : "halt", +var _OpNames = [256]string{ + OP_size_check: "size_check", + OP_size_const: "size_const", + OP_size_dyn: "size_dyn", + OP_size_map: "size_map", + OP_size_defer: "size_defer", + OP_byte: "byte", + OP_word: "word", + OP_long: "long", + OP_quad: "quad", + OP_sint: "sint", + OP_length: "length", + OP_memcpy_be: "memcpy_be", + OP_seek: "seek", + OP_deref: "deref", + OP_defer: "defer", + OP_map_len: "map_len", + OP_map_key: "map_key", + OP_map_next: "map_next", + OP_map_value: "map_value", + OP_map_begin: "map_begin", + OP_map_if_next: "map_if_next", + OP_map_if_empty: "map_if_empty", + OP_list_decr: "list_decr", + OP_list_begin: "list_begin", + OP_list_if_next: "list_if_next", + OP_list_if_empty: "list_if_empty", + OP_unique: "unique", + OP_goto: "goto", + OP_if_nil: "if_nil", + OP_if_hasbuf: "if_hasbuf", + OP_if_eq_imm: "if_eq_imm", + OP_if_eq_str: "if_eq_str", + OP_make_state: "make_state", + OP_drop_state: "drop_state", + OP_halt: "halt", } -var _OpBranches = [256]bool { - OP_map_if_next : true, - OP_map_if_empty : true, - OP_list_if_next : true, - OP_list_if_empty : true, - OP_goto : true, - OP_if_nil : true, - OP_if_hasbuf : true, - OP_if_eq_imm : true, - OP_if_eq_str : true, +var _OpBranches = [256]bool{ + OP_map_if_next: true, + OP_map_if_empty: true, + OP_list_if_next: true, + OP_list_if_empty: true, + OP_goto: true, + OP_if_nil: true, + OP_if_hasbuf: true, + OP_if_eq_imm: true, + OP_if_eq_str: true, } func (self OpCode) String() string { - if _OpNames[self] != "" { - return _OpNames[self] - } else { - return fmt.Sprintf("OpCode(%d)", self) - } + if _OpNames[self] != "" { + return _OpNames[self] + } else { + return fmt.Sprintf("OpCode(%d)", self) + } } diff --git a/internal/binary/encoder/optimizer.go b/internal/binary/encoder/optimizer.go index c1a4fe1..31edb20 100644 --- a/internal/binary/encoder/optimizer.go +++ b/internal/binary/encoder/optimizer.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,437 +17,489 @@ package encoder import ( - `encoding/binary` - `fmt` - `sort` - `strings` - `unsafe` - - `github.com/cloudwego/frugal/internal/rt` - `github.com/oleiade/lane` + "encoding/binary" + "fmt" + "sort" + "strings" + "unsafe" + + "github.com/cloudwego/frugal/internal/rt" + "github.com/oleiade/lane" ) type BasicBlock struct { - P Program - Src int - End int - Next *BasicBlock - Link *BasicBlock + P Program + Src int + End int + Next *BasicBlock + Link *BasicBlock } func (self *BasicBlock) Len() int { - return self.End - self.Src + return self.End - self.Src } func (self *BasicBlock) Free() { - q := lane.NewQueue() - m := make(map[*BasicBlock]struct{}) - - /* traverse the graph with BFS */ - for q.Enqueue(self); !q.Empty(); { - v := q.Dequeue() - p := v.(*BasicBlock) - - /* add branch to queue */ - if p.Link != nil { - if _, ok := m[p.Link]; !ok { - q.Enqueue(p.Link) - } - } - - /* clear branch, and add to free list */ - m[p] = struct{}{} - p.Link = nil - } - - /* reset and free all the nodes */ - for p := range m { - p.Next = nil - freeBasicBlock(p) - } + q := lane.NewQueue() + m := make(map[*BasicBlock]struct{}) + + /* traverse the graph with BFS */ + for q.Enqueue(self); !q.Empty(); { + v := q.Dequeue() + p := v.(*BasicBlock) + + /* add branch to queue */ + if p.Link != nil { + if _, ok := m[p.Link]; !ok { + q.Enqueue(p.Link) + } + } + + /* clear branch, and add to free list */ + m[p] = struct{}{} + p.Link = nil + } + + /* reset and free all the nodes */ + for p := range m { + p.Next = nil + freeBasicBlock(p) + } } func (self *BasicBlock) String() string { - n := self.End - self.Src - v := make([]string, n + 1) + n := self.End - self.Src + v := make([]string, n+1) - /* dump every instructions */ - for i := self.Src; i < self.End; i++ { - v[i - self.Src + 1] = " " + self.P[i].Disassemble() - } + /* dump every instructions */ + for i := self.Src; i < self.End; i++ { + v[i-self.Src+1] = " " + self.P[i].Disassemble() + } - /* add the entry label */ - v[0] = fmt.Sprintf("L_%d:", self.Src) - return strings.Join(v, "\n") + /* add the entry label */ + v[0] = fmt.Sprintf("L_%d:", self.Src) + return strings.Join(v, "\n") } type GraphBuilder struct { - Pin map[int]bool - Graph map[int]*BasicBlock + Pin map[int]bool + Graph map[int]*BasicBlock } func (self *GraphBuilder) scan(p Program) { - for _, v := range p { - if _OpBranches[v.Op] { - self.Pin[v.To] = true - } - } + for _, v := range p { + if _OpBranches[v.Op] { + self.Pin[v.To] = true + } + } } func (self *GraphBuilder) block(p Program, i int, bb *BasicBlock) { - bb.Src = i - bb.End = i - - /* traverse down until it hits a branch instruction */ - for i < len(p) && !_OpBranches[p[i].Op] { - i++ - bb.End++ - - /* hit a merge point, merge with existing block */ - if self.Pin[i] { - bb.Next = self.branch(p, i) - return - } - } - - /* end of basic block */ - if i == len(p) { - return - } - - /* also include the branch instruction */ - bb.End++ - bb.Next = self.branch(p, p[i].To) - - /* GOTO instruction doesn't technically "branch", anything - * sits between it and the next branch target are unreachable. */ - if p[i].Op != OP_goto { - bb.Link = bb.Next - bb.Next = self.branch(p, i + 1) - } + bb.Src = i + bb.End = i + + /* traverse down until it hits a branch instruction */ + for i < len(p) && !_OpBranches[p[i].Op] { + i++ + bb.End++ + + /* hit a merge point, merge with existing block */ + if self.Pin[i] { + bb.Next = self.branch(p, i) + return + } + } + + /* end of basic block */ + if i == len(p) { + return + } + + /* also include the branch instruction */ + bb.End++ + bb.Next = self.branch(p, p[i].To) + + /* GOTO instruction doesn't technically "branch", anything + * sits between it and the next branch target are unreachable. */ + if p[i].Op != OP_goto { + bb.Link = bb.Next + bb.Next = self.branch(p, i+1) + } } func (self *GraphBuilder) branch(p Program, i int) *BasicBlock { - var ok bool - var bb *BasicBlock - - /* check for existing basic blocks */ - if bb, ok = self.Graph[i]; ok { - return bb - } - - /* create a new block */ - bb = newBasicBlock() - bb.P, bb.Next, bb.Link = p, nil, nil - - /* process the new block */ - self.Graph[i] = bb - self.block(p, i, bb) - return bb + var ok bool + var bb *BasicBlock + + /* check for existing basic blocks */ + if bb, ok = self.Graph[i]; ok { + return bb + } + + /* create a new block */ + bb = newBasicBlock() + bb.P, bb.Next, bb.Link = p, nil, nil + + /* process the new block */ + self.Graph[i] = bb + self.block(p, i, bb) + return bb } func (self *GraphBuilder) Free() { - rt.MapClear(self.Pin) - rt.MapClear(self.Graph) - freeGraphBuilder(self) + rt.MapClear(self.Pin) + rt.MapClear(self.Graph) + freeGraphBuilder(self) } func (self *GraphBuilder) Build(p Program) *BasicBlock { - self.scan(p) - return self.branch(p, 0) + self.scan(p) + return self.branch(p, 0) } func (self *GraphBuilder) BuildAndFree(p Program) (bb *BasicBlock) { - bb = self.Build(p) - self.Free() - return + bb = self.Build(p) + self.Free() + return } type _OptimizerState struct { - buf []*BasicBlock - refs map[int]int - mask map[*BasicBlock]bool + buf []*BasicBlock + refs map[int]int + mask map[*BasicBlock]bool } func (self *_OptimizerState) visit(bb *BasicBlock) bool { - var mm bool - var ok bool - - /* check for duplication */ - if mm, ok = self.mask[bb]; mm && ok { - return false - } - - /* add to block buffer */ - self.buf = append(self.buf, bb) - self.mask[bb] = true - return true + var mm bool + var ok bool + + /* check for duplication */ + if mm, ok = self.mask[bb]; mm && ok { + return false + } + + /* add to block buffer */ + self.buf = append(self.buf, bb) + self.mask[bb] = true + return true } func Optimize(p Program) Program { - acc := 0 - ret := newProgram() - buf := lane.NewQueue() - ctx := newOptimizerState() - cfg := newGraphBuilder().BuildAndFree(p) - - /* travel with BFS */ - for buf.Enqueue(cfg); !buf.Empty(); { - v := buf.Dequeue() - b := v.(*BasicBlock) - - /* check for duplication, and then mark as visited */ - if !ctx.visit(b) { - continue - } - - /* optimize each block */ - for _, pass := range _PassTab { - pass(b) - } - - /* add conditional branches if any */ - if b.Next != nil { buf.Enqueue(b.Next) } - if b.Link != nil { buf.Enqueue(b.Link) } - } - - /* sort the blocks by entry point */ - sort.Slice(ctx.buf, func(i int, j int) bool { - return ctx.buf[i].Src < ctx.buf[j].Src - }) - - /* remap all the branch locations */ - for _, bb := range ctx.buf { - ctx.refs[bb.Src] = acc - acc += bb.End - bb.Src - } - - /* adjust all the branch targets */ - for _, bb := range ctx.buf { - if end := bb.End; bb.Src != end { - if ins := &bb.P[end - 1]; _OpBranches[ins.Op] { - ins.To = ctx.refs[ins.To] - } - } - } - - /* merge all the basic blocks */ - for _, bb := range ctx.buf { - ret = append(ret, bb.P[bb.Src:bb.End]...) - } - - /* release the original program */ - p.Free() - freeOptimizerState(ctx) - return ret + acc := 0 + ret := newProgram() + buf := lane.NewQueue() + ctx := newOptimizerState() + cfg := newGraphBuilder().BuildAndFree(p) + + /* travel with BFS */ + for buf.Enqueue(cfg); !buf.Empty(); { + v := buf.Dequeue() + b := v.(*BasicBlock) + + /* check for duplication, and then mark as visited */ + if !ctx.visit(b) { + continue + } + + /* optimize each block */ + for _, pass := range _PassTab { + pass(b) + } + + /* add conditional branches if any */ + if b.Next != nil { + buf.Enqueue(b.Next) + } + if b.Link != nil { + buf.Enqueue(b.Link) + } + } + + /* sort the blocks by entry point */ + sort.Slice(ctx.buf, func(i int, j int) bool { + return ctx.buf[i].Src < ctx.buf[j].Src + }) + + /* remap all the branch locations */ + for _, bb := range ctx.buf { + ctx.refs[bb.Src] = acc + acc += bb.End - bb.Src + } + + /* adjust all the branch targets */ + for _, bb := range ctx.buf { + if end := bb.End; bb.Src != end { + if ins := &bb.P[end-1]; _OpBranches[ins.Op] { + ins.To = ctx.refs[ins.To] + } + } + } + + /* merge all the basic blocks */ + for _, bb := range ctx.buf { + ret = append(ret, bb.P[bb.Src:bb.End]...) + } + + /* release the original program */ + p.Free() + freeOptimizerState(ctx) + return ret } -var _PassTab = [...]func(p *BasicBlock) { - _PASS_StaticSizeMerging, - _PASS_SeekMerging, - _PASS_NopElimination, - _PASS_SizeCheckMerging, - _PASS_LiteralMerging, - _PASS_Compacting, +var _PassTab = [...]func(p *BasicBlock){ + _PASS_StaticSizeMerging, + _PASS_SeekMerging, + _PASS_NopElimination, + _PASS_SizeCheckMerging, + _PASS_LiteralMerging, + _PASS_Compacting, } const ( - _NOP OpCode = 0xff + _NOP OpCode = 0xff ) func init() { - _OpNames[_NOP] = "(nop)" + _OpNames[_NOP] = "(nop)" } func checksl(s *[]byte, n int) *rt.GoSlice { - sl := (*rt.GoSlice)(unsafe.Pointer(s)) - sn := sl.Len - - /* check for length */ - if sn + n > sl.Cap { - panic("slice overflow") - } else { - return sl - } + sl := (*rt.GoSlice)(unsafe.Pointer(s)) + sn := sl.Len + + /* check for length */ + if sn+n > sl.Cap { + panic("slice overflow") + } else { + return sl + } } func append1(s *[]byte, v byte) { - sl := checksl(s, 1) - sl.Set(sl.Len, v) - sl.Len++ + sl := checksl(s, 1) + sl.Set(sl.Len, v) + sl.Len++ } func append2(s *[]byte, v uint16) { - sl := checksl(s, 2) - sl.Set(sl.Len + 0, byte(v >> 8)) - sl.Set(sl.Len + 1, byte(v)) - sl.Len += 2 + sl := checksl(s, 2) + sl.Set(sl.Len+0, byte(v>>8)) + sl.Set(sl.Len+1, byte(v)) + sl.Len += 2 } func append4(s *[]byte, v uint32) { - sl := checksl(s, 4) - sl.Set(sl.Len + 0, byte(v >> 24)) - sl.Set(sl.Len + 1, byte(v >> 16)) - sl.Set(sl.Len + 2, byte(v >> 8)) - sl.Set(sl.Len + 3, byte(v)) - sl.Len += 4 + sl := checksl(s, 4) + sl.Set(sl.Len+0, byte(v>>24)) + sl.Set(sl.Len+1, byte(v>>16)) + sl.Set(sl.Len+2, byte(v>>8)) + sl.Set(sl.Len+3, byte(v)) + sl.Len += 4 } func append8(s *[]byte, v uint64) { - sl := checksl(s, 8) - sl.Set(sl.Len + 0, byte(v >> 56)) - sl.Set(sl.Len + 1, byte(v >> 48)) - sl.Set(sl.Len + 2, byte(v >> 40)) - sl.Set(sl.Len + 3, byte(v >> 32)) - sl.Set(sl.Len + 4, byte(v >> 24)) - sl.Set(sl.Len + 5, byte(v >> 16)) - sl.Set(sl.Len + 6, byte(v >> 8)) - sl.Set(sl.Len + 7, byte(v)) - sl.Len += 8 + sl := checksl(s, 8) + sl.Set(sl.Len+0, byte(v>>56)) + sl.Set(sl.Len+1, byte(v>>48)) + sl.Set(sl.Len+2, byte(v>>40)) + sl.Set(sl.Len+3, byte(v>>32)) + sl.Set(sl.Len+4, byte(v>>24)) + sl.Set(sl.Len+5, byte(v>>16)) + sl.Set(sl.Len+6, byte(v>>8)) + sl.Set(sl.Len+7, byte(v)) + sl.Len += 8 } // Static Size Merging Pass: merges constant size instructions as much as possible. func _PASS_StaticSizeMerging(bb *BasicBlock) { - for i := bb.Src; i < bb.End; i++ { - if p := &bb.P[i]; p.Op == OP_size_const { - for r, j := true, i + 1; r && j < bb.End; i, j = i + 1, j + 1 { - switch bb.P[j].Op { - case _NOP : break - case OP_seek : break - case OP_deref : break - case OP_size_dyn : break - case OP_size_const : p.Iv += bb.P[j].Iv; bb.P[j].Op = _NOP - default : r = false - } - } - } - } + for i := bb.Src; i < bb.End; i++ { + if p := &bb.P[i]; p.Op == OP_size_const { + for r, j := true, i+1; r && j < bb.End; i, j = i+1, j+1 { + switch bb.P[j].Op { + case _NOP: + break + case OP_seek: + break + case OP_deref: + break + case OP_size_dyn: + break + case OP_size_const: + p.Iv += bb.P[j].Iv + bb.P[j].Op = _NOP + default: + r = false + } + } + } + } } // Seek Merging Pass: merges seeking instructions as much as possible. func _PASS_SeekMerging(bb *BasicBlock) { - for i := bb.Src; i < bb.End; i++ { - if p := &bb.P[i]; p.Op == OP_seek { - for r, j := true, i + 1; r && j < bb.End; i, j = i + 1, j + 1 { - switch bb.P[j].Op { - case _NOP : break - case OP_seek : p.Iv += bb.P[j].Iv; bb.P[j].Op = _NOP - default : r = false - } - } - } - } + for i := bb.Src; i < bb.End; i++ { + if p := &bb.P[i]; p.Op == OP_seek { + for r, j := true, i+1; r && j < bb.End; i, j = i+1, j+1 { + switch bb.P[j].Op { + case _NOP: + break + case OP_seek: + p.Iv += bb.P[j].Iv + bb.P[j].Op = _NOP + default: + r = false + } + } + } + } } // NOP Elimination Pass: remove instructions that are effectively NOPs (`seek 0`, `size_const 0`) func _PASS_NopElimination(bb *BasicBlock) { - for i := bb.Src; i < bb.End; i++ { - if bb.P[i].Iv == 0 && (bb.P[i].Op == OP_seek || bb.P[i].Op == OP_size_const) { - bb.P[i].Op = _NOP - } - } + for i := bb.Src; i < bb.End; i++ { + if bb.P[i].Iv == 0 && (bb.P[i].Op == OP_seek || bb.P[i].Op == OP_size_const) { + bb.P[i].Op = _NOP + } + } } // Size Check Merging Pass: merges size-checking instructions as much as possible. func _PASS_SizeCheckMerging(bb *BasicBlock) { - for i := bb.Src; i < bb.End; i++ { - if p := &bb.P[i]; p.Op == OP_size_check { - for r, j := true, i + 1; r && j < bb.End; i, j = i + 1, j + 1 { - switch bb.P[j].Op { - case _NOP : break - case OP_byte : break - case OP_word : break - case OP_long : break - case OP_quad : break - case OP_sint : break - case OP_seek : break - case OP_deref : break - case OP_length : break - case OP_memcpy_be : break - case OP_size_check : p.Iv += bb.P[j].Iv; bb.P[j].Op = _NOP - default : r = false - } - } - } - } + for i := bb.Src; i < bb.End; i++ { + if p := &bb.P[i]; p.Op == OP_size_check { + for r, j := true, i+1; r && j < bb.End; i, j = i+1, j+1 { + switch bb.P[j].Op { + case _NOP: + break + case OP_byte: + break + case OP_word: + break + case OP_long: + break + case OP_quad: + break + case OP_sint: + break + case OP_seek: + break + case OP_deref: + break + case OP_length: + break + case OP_memcpy_be: + break + case OP_size_check: + p.Iv += bb.P[j].Iv + bb.P[j].Op = _NOP + default: + r = false + } + } + } + } } -// Literal Merging Pass: merges all consectutive byte, word or long instructions. +// Literal Merging Pass: merges all consecutive byte, word or long instructions. func _PASS_LiteralMerging(bb *BasicBlock) { - p := bb.P - i := bb.Src - - /* scan every instruction */ - for i < bb.End { - iv := p[i] - op := iv.Op - - /* only interested in literal instructions */ - if op < OP_byte || op > OP_quad { - i++ - continue - } - - /* byte merging buffer */ - ip := i - mm := [15]byte{} - sl := mm[:0:cap(mm)] - - /* scan for consecutive bytes */ - loop: for i < bb.End { - iv = p[i] - op = iv.Op - - /* check for OpCode */ - switch op { - case _NOP : i++; continue - case OP_seek : i++; continue - case OP_deref : i++; continue - case OP_byte : append1(&sl, byte(iv.Iv)) - case OP_word : append2(&sl, uint16(iv.Iv)) - case OP_long : append4(&sl, uint32(iv.Iv)) - case OP_quad : append8(&sl, uint64(iv.Iv)) - default : break loop - } - - /* adjust the program counter */ - p[i].Op = _NOP - i++ - - /* commit the buffer if needed */ - for len(sl) >= 8 { - p[ip] = Instr{Op: OP_quad, Iv: int64(binary.BigEndian.Uint64(sl))} - sl = sl[8:] - ip++ - } - - /* move the remaining bytes to the front */ - copy(mm[:], sl) - sl = mm[:len(sl):cap(mm)] - } - - /* add the remaining bytes */ - if len(sl) >= 4 { p[ip] = Instr{Op: OP_long, Iv: int64(binary.BigEndian.Uint32(sl))} ; sl = sl[4:]; ip++ } - if len(sl) >= 2 { p[ip] = Instr{Op: OP_word, Iv: int64(binary.BigEndian.Uint16(sl))} ; sl = sl[2:]; ip++ } - if len(sl) >= 1 { p[ip] = Instr{Op: OP_byte, Iv: int64(sl[0])} ; sl = sl[1:]; ip++ } - } + p := bb.P + i := bb.Src + + /* scan every instruction */ + for i < bb.End { + iv := p[i] + op := iv.Op + + /* only interested in literal instructions */ + if op < OP_byte || op > OP_quad { + i++ + continue + } + + /* byte merging buffer */ + ip := i + mm := [15]byte{} + sl := mm[:0:cap(mm)] + + /* scan for consecutive bytes */ + loop: + for i < bb.End { + iv = p[i] + op = iv.Op + + /* check for OpCode */ + switch op { + case _NOP: + i++ + continue + case OP_seek: + i++ + continue + case OP_deref: + i++ + continue + case OP_byte: + append1(&sl, byte(iv.Iv)) + case OP_word: + append2(&sl, uint16(iv.Iv)) + case OP_long: + append4(&sl, uint32(iv.Iv)) + case OP_quad: + append8(&sl, uint64(iv.Iv)) + default: + break loop + } + + /* adjust the program counter */ + p[i].Op = _NOP + i++ + + /* commit the buffer if needed */ + for len(sl) >= 8 { + p[ip] = Instr{Op: OP_quad, Iv: int64(binary.BigEndian.Uint64(sl))} + sl = sl[8:] + ip++ + } + + /* move the remaining bytes to the front */ + copy(mm[:], sl) + sl = mm[:len(sl):cap(mm)] + } + + /* add the remaining bytes */ + if len(sl) >= 4 { + p[ip] = Instr{Op: OP_long, Iv: int64(binary.BigEndian.Uint32(sl))} + sl = sl[4:] + ip++ + } + if len(sl) >= 2 { + p[ip] = Instr{Op: OP_word, Iv: int64(binary.BigEndian.Uint16(sl))} + sl = sl[2:] + ip++ + } + if len(sl) >= 1 { + p[ip] = Instr{Op: OP_byte, Iv: int64(sl[0])} + sl = sl[1:] + ip++ + } + } } // Compacting Pass: remove all the placeholder NOP instructions inserted in the previous pass. func _PASS_Compacting(bb *BasicBlock) { - var i int - var j int - - /* copy instructins excluding NOPs */ - for i, j = bb.Src, bb.Src; i < bb.End; i++ { - if bb.P[i].Op != _NOP { - bb.P[j] = bb.P[i] - j++ - } - } - - /* update basic block end if needed */ - if i != j { - bb.End = j - } + var i int + var j int + + /* copy instructins excluding NOPs */ + for i, j = bb.Src, bb.Src; i < bb.End; i++ { + if bb.P[i].Op != _NOP { + bb.P[j] = bb.P[i] + j++ + } + } + + /* update basic block end if needed */ + if i != j { + bb.End = j + } } diff --git a/internal/binary/encoder/pools.go b/internal/binary/encoder/pools.go index 25cef22..a4b1b30 100644 --- a/internal/binary/encoder/pools.go +++ b/internal/binary/encoder/pools.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,131 +17,131 @@ package encoder import ( - `reflect` - `sync` + "reflect" + "sync" - `github.com/cloudwego/frugal/internal/opts` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/opts" + "github.com/cloudwego/frugal/internal/rt" ) var ( - programPool sync.Pool - compilerPool sync.Pool - basicBlockPool sync.Pool - graphBuilderPool sync.Pool - runtimeStatePool sync.Pool - optimizerStatePool sync.Pool + programPool sync.Pool + compilerPool sync.Pool + basicBlockPool sync.Pool + graphBuilderPool sync.Pool + runtimeStatePool sync.Pool + optimizerStatePool sync.Pool ) func newProgram() Program { - if v := programPool.Get(); v != nil { - return v.(Program)[:0] - } else { - return make(Program, 0, 16) - } + if v := programPool.Get(); v != nil { + return v.(Program)[:0] + } else { + return make(Program, 0, 16) + } } func freeProgram(p Program) { - programPool.Put(p) + programPool.Put(p) } func newCompiler() *Compiler { - if v := compilerPool.Get(); v == nil { - return allocCompiler() - } else { - return resetCompiler(v.(*Compiler)) - } + if v := compilerPool.Get(); v == nil { + return allocCompiler() + } else { + return resetCompiler(v.(*Compiler)) + } } func freeCompiler(p *Compiler) { - compilerPool.Put(p) + compilerPool.Put(p) } func allocCompiler() *Compiler { - return &Compiler { - o: opts.GetDefaultOptions(), - t: make(map[reflect.Type]bool), - } + return &Compiler{ + o: opts.GetDefaultOptions(), + t: make(map[reflect.Type]bool), + } } func resetCompiler(p *Compiler) *Compiler { - p.o = opts.GetDefaultOptions() - rt.MapClear(p.t) - return p + p.o = opts.GetDefaultOptions() + rt.MapClear(p.t) + return p } func newBasicBlock() *BasicBlock { - if v := basicBlockPool.Get(); v != nil { - return v.(*BasicBlock) - } else { - return new(BasicBlock) - } + if v := basicBlockPool.Get(); v != nil { + return v.(*BasicBlock) + } else { + return new(BasicBlock) + } } func freeBasicBlock(p *BasicBlock) { - basicBlockPool.Put(p) + basicBlockPool.Put(p) } func newGraphBuilder() *GraphBuilder { - if v := graphBuilderPool.Get(); v == nil { - return allocGraphBuilder() - } else { - return resetGraphBuilder(v.(*GraphBuilder)) - } + if v := graphBuilderPool.Get(); v == nil { + return allocGraphBuilder() + } else { + return resetGraphBuilder(v.(*GraphBuilder)) + } } func freeGraphBuilder(p *GraphBuilder) { - graphBuilderPool.Put(p) + graphBuilderPool.Put(p) } func allocGraphBuilder() *GraphBuilder { - return &GraphBuilder { - Pin : make(map[int]bool), - Graph : make(map[int]*BasicBlock), - } + return &GraphBuilder{ + Pin: make(map[int]bool), + Graph: make(map[int]*BasicBlock), + } } func resetGraphBuilder(p *GraphBuilder) *GraphBuilder { - rt.MapClear(p.Pin) - rt.MapClear(p.Graph) - return p + rt.MapClear(p.Pin) + rt.MapClear(p.Graph) + return p } func newRuntimeState() *RuntimeState { - if v := runtimeStatePool.Get(); v != nil { - return v.(*RuntimeState) - } else { - return new(RuntimeState) - } + if v := runtimeStatePool.Get(); v != nil { + return v.(*RuntimeState) + } else { + return new(RuntimeState) + } } func freeRuntimeState(p *RuntimeState) { - runtimeStatePool.Put(p) + runtimeStatePool.Put(p) } func newOptimizerState() *_OptimizerState { - if v := optimizerStatePool.Get(); v == nil { - return allocOptimizerState() - } else { - return resetOptimizerState(v.(*_OptimizerState)) - } + if v := optimizerStatePool.Get(); v == nil { + return allocOptimizerState() + } else { + return resetOptimizerState(v.(*_OptimizerState)) + } } func freeOptimizerState(p *_OptimizerState) { - optimizerStatePool.Put(p) + optimizerStatePool.Put(p) } func allocOptimizerState() *_OptimizerState { - return &_OptimizerState { - buf : make([]*BasicBlock, 0, 16), - refs : make(map[int]int), - mask : make(map[*BasicBlock]bool), - } + return &_OptimizerState{ + buf: make([]*BasicBlock, 0, 16), + refs: make(map[int]int), + mask: make(map[*BasicBlock]bool), + } } func resetOptimizerState(p *_OptimizerState) *_OptimizerState { - p.buf = p.buf[:0] - rt.MapClear(p.refs) - rt.MapClear(p.mask) - return p + p.buf = p.buf[:0] + rt.MapClear(p.refs) + rt.MapClear(p.mask) + return p } diff --git a/internal/binary/encoder/state.go b/internal/binary/encoder/state.go index 5b224c5..7b2d6b2 100644 --- a/internal/binary/encoder/state.go +++ b/internal/binary/encoder/state.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,43 +17,43 @@ package encoder import ( - `math` - `unsafe` + "math" + "unsafe" - `github.com/cloudwego/frugal/internal/binary/defs` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/binary/defs" + "github.com/cloudwego/frugal/internal/rt" ) const ( - LnOffset = int64(unsafe.Offsetof(StateItem{}.Ln)) - MiOffset = int64(unsafe.Offsetof(StateItem{}.Mi)) - WpOffset = int64(unsafe.Offsetof(StateItem{}.Wp)) - BmOffset = int64(unsafe.Offsetof(RuntimeState{}.Bm)) + LnOffset = int64(unsafe.Offsetof(StateItem{}.Ln)) + MiOffset = int64(unsafe.Offsetof(StateItem{}.Mi)) + WpOffset = int64(unsafe.Offsetof(StateItem{}.Wp)) + BmOffset = int64(unsafe.Offsetof(RuntimeState{}.Bm)) ) const ( - MiKeyOffset = int64(unsafe.Offsetof(rt.GoMapIterator{}.K)) - MiValueOffset = int64(unsafe.Offsetof(rt.GoMapIterator{}.V)) + MiKeyOffset = int64(unsafe.Offsetof(rt.GoMapIterator{}.K)) + MiValueOffset = int64(unsafe.Offsetof(rt.GoMapIterator{}.V)) ) const ( - StateMax = (defs.StackSize - 1) * StateSize - StateSize = int64(unsafe.Sizeof(StateItem{})) + StateMax = (defs.StackSize - 1) * StateSize + StateSize = int64(unsafe.Sizeof(StateItem{})) ) const ( - RangeUint8 = math.MaxUint8 + 1 - RangeUint16 = math.MaxUint16 + 1 + RangeUint8 = math.MaxUint8 + 1 + RangeUint16 = math.MaxUint16 + 1 ) type StateItem struct { - Ln uintptr - Wp unsafe.Pointer - Mi rt.GoMapIterator + Ln uintptr + Wp unsafe.Pointer + Mi rt.GoMapIterator } type RuntimeState struct { - St [defs.StackSize]StateItem // Must be the first field. - Bm [1024]uint64 // Bitmap, used for uniqueness check of set and set. - Val unsafe.Pointer // Pointer to the object to be encoded + St [defs.StackSize]StateItem // Must be the first field. + Bm [1024]uint64 // Bitmap, used for uniqueness check of set and set. + Val unsafe.Pointer // Pointer to the object to be encoded } diff --git a/internal/binary/encoder/translator.go b/internal/binary/encoder/translator.go index 4c45d85..550a096 100644 --- a/internal/binary/encoder/translator.go +++ b/internal/binary/encoder/translator.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,15 +17,15 @@ package encoder import ( - `fmt` - `os` - `reflect` - - `github.com/cloudwego/frugal/internal/atm/abi` - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/binary/defs` - `github.com/cloudwego/frugal/internal/rt` - `github.com/cloudwego/frugal/internal/utils` + "fmt" + "os" + "reflect" + + "github.com/cloudwego/frugal/internal/atm/abi" + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/binary/defs" + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/frugal/internal/utils" ) /** Function Prototype @@ -44,13 +44,13 @@ import ( */ const ( - ARG_buf = 0 - ARG_len = 1 - ARG_mem_itab = 2 - ARG_mem_data = 3 - ARG_p = 4 - ARG_rs = 5 - ARG_st = 6 + ARG_buf = 0 + ARG_len = 1 + ARG_mem_itab = 2 + ARG_mem_data = 3 + ARG_p = 4 + ARG_rs = 5 + ARG_st = 6 ) /** Register Allocations @@ -67,575 +67,626 @@ const ( */ const ( - WP = hir.P1 - RP = hir.P2 - RS = hir.P3 - ET = hir.P4 // may also be used as a temporary pointer register - EP = hir.P5 // may also be used as a temporary pointer register + WP = hir.P1 + RP = hir.P2 + RS = hir.P3 + ET = hir.P4 // may also be used as a temporary pointer register + EP = hir.P5 // may also be used as a temporary pointer register ) const ( - RL = hir.R2 - RC = hir.R3 - ST = hir.R4 + RL = hir.R2 + RC = hir.R3 + ST = hir.R4 ) const ( - TP = hir.P0 - TR = hir.R0 - UR = hir.R1 + TP = hir.P0 + TR = hir.R0 + UR = hir.R1 ) const ( - LB_halt = "_halt" - LB_error = "_error" - LB_nomem = "_nomem" - LB_overflow = "_overflow" - LB_duplicated = "_duplicated" + LB_halt = "_halt" + LB_error = "_error" + LB_nomem = "_nomem" + LB_overflow = "_overflow" + LB_duplicated = "_duplicated" ) var ( - _N_page = int64(os.Getpagesize()) - _E_nomem = fmt.Errorf("frugal: buffer is too small") - _E_overflow = fmt.Errorf("frugal: encoder stack overflow") - _E_duplicated = fmt.Errorf("frugal: duplicated element within sets") + _N_page = int64(os.Getpagesize()) + _E_nomem = fmt.Errorf("frugal: buffer is too small") + _E_overflow = fmt.Errorf("frugal: encoder stack overflow") + _E_duplicated = fmt.Errorf("frugal: duplicated element within sets") ) func Translate(s Program) hir.Program { - p := hir.CreateBuilder() - prologue (p) - program (p, s) - epilogue (p) - errors (p) - return p.Build() + p := hir.CreateBuilder() + prologue(p) + program(p, s) + epilogue(p) + errors(p) + return p.Build() } func errors(p *hir.Builder) { - p.Label (LB_nomem) - p.MOV (UR, RL) - p.IP (&_E_nomem, TP) - p.JMP ("_basic_error") - p.Label (LB_overflow) - p.IP (&_E_overflow, TP) - p.JMP ("_basic_error") - p.Label (LB_duplicated) - p.IP (&_E_duplicated, TP) - p.Label ("_basic_error") - p.LP (TP, 0, ET) - p.LP (TP, 8, EP) - p.JMP (LB_error) + p.Label(LB_nomem) + p.MOV(UR, RL) + p.IP(&_E_nomem, TP) + p.JMP("_basic_error") + p.Label(LB_overflow) + p.IP(&_E_overflow, TP) + p.JMP("_basic_error") + p.Label(LB_duplicated) + p.IP(&_E_duplicated, TP) + p.Label("_basic_error") + p.LP(TP, 0, ET) + p.LP(TP, 8, EP) + p.JMP(LB_error) } func program(p *hir.Builder, s Program) { - for i, v := range s { - p.Mark(i) - translators[v.Op](p, v) - } + for i, v := range s { + p.Mark(i) + translators[v.Op](p, v) + } } func prologue(p *hir.Builder) { - p.LDAP (ARG_buf, RP) - p.LDAQ (ARG_len, RC) - p.LDAP (ARG_p, WP) - p.LDAP (ARG_rs, RS) - p.LDAQ (ARG_st, ST) - p.MOV (hir.Rz, UR) - p.MOV (hir.Rz, RL) + p.LDAP(ARG_buf, RP) + p.LDAQ(ARG_len, RC) + p.LDAP(ARG_p, WP) + p.LDAP(ARG_rs, RS) + p.LDAQ(ARG_st, ST) + p.MOV(hir.Rz, UR) + p.MOV(hir.Rz, RL) } func epilogue(p *hir.Builder) { - p.Label (LB_halt) - p.MOVP (hir.Pn, ET) - p.MOVP (hir.Pn, EP) - p.Label (LB_error) - p.RET (). - R0 (RL). - R1 (ET). - R2 (EP) -} - -var translators = [256]func(*hir.Builder, Instr) { - OP_size_check : translate_OP_size_check, - OP_size_const : translate_OP_size_const, - OP_size_dyn : translate_OP_size_dyn, - OP_size_map : translate_OP_size_map, - OP_size_defer : translate_OP_size_defer, - OP_byte : translate_OP_byte, - OP_word : translate_OP_word, - OP_long : translate_OP_long, - OP_quad : translate_OP_quad, - OP_sint : translate_OP_sint, - OP_length : translate_OP_length, - OP_memcpy_be : translate_OP_memcpy_be, - OP_seek : translate_OP_seek, - OP_deref : translate_OP_deref, - OP_defer : translate_OP_defer, - OP_map_len : translate_OP_map_len, - OP_map_key : translate_OP_map_key, - OP_map_next : translate_OP_map_next, - OP_map_value : translate_OP_map_value, - OP_map_begin : translate_OP_map_begin, - OP_map_if_next : translate_OP_map_if_next, - OP_map_if_empty : translate_OP_map_if_empty, - OP_list_decr : translate_OP_list_decr, - OP_list_begin : translate_OP_list_begin, - OP_list_if_next : translate_OP_list_if_next, - OP_list_if_empty : translate_OP_list_if_empty, - OP_unique : translate_OP_unique, - OP_goto : translate_OP_goto, - OP_if_nil : translate_OP_if_nil, - OP_if_hasbuf : translate_OP_if_hasbuf, - OP_if_eq_imm : translate_OP_if_eq_imm, - OP_if_eq_str : translate_OP_if_eq_str, - OP_make_state : translate_OP_make_state, - OP_drop_state : translate_OP_drop_state, - OP_halt : translate_OP_halt, + p.Label(LB_halt) + p.MOVP(hir.Pn, ET) + p.MOVP(hir.Pn, EP) + p.Label(LB_error) + p.RET(). + R0(RL). + R1(ET). + R2(EP) +} + +var translators = [256]func(*hir.Builder, Instr){ + OP_size_check: translate_OP_size_check, + OP_size_const: translate_OP_size_const, + OP_size_dyn: translate_OP_size_dyn, + OP_size_map: translate_OP_size_map, + OP_size_defer: translate_OP_size_defer, + OP_byte: translate_OP_byte, + OP_word: translate_OP_word, + OP_long: translate_OP_long, + OP_quad: translate_OP_quad, + OP_sint: translate_OP_sint, + OP_length: translate_OP_length, + OP_memcpy_be: translate_OP_memcpy_be, + OP_seek: translate_OP_seek, + OP_deref: translate_OP_deref, + OP_defer: translate_OP_defer, + OP_map_len: translate_OP_map_len, + OP_map_key: translate_OP_map_key, + OP_map_next: translate_OP_map_next, + OP_map_value: translate_OP_map_value, + OP_map_begin: translate_OP_map_begin, + OP_map_if_next: translate_OP_map_if_next, + OP_map_if_empty: translate_OP_map_if_empty, + OP_list_decr: translate_OP_list_decr, + OP_list_begin: translate_OP_list_begin, + OP_list_if_next: translate_OP_list_if_next, + OP_list_if_empty: translate_OP_list_if_empty, + OP_unique: translate_OP_unique, + OP_goto: translate_OP_goto, + OP_if_nil: translate_OP_if_nil, + OP_if_hasbuf: translate_OP_if_hasbuf, + OP_if_eq_imm: translate_OP_if_eq_imm, + OP_if_eq_str: translate_OP_if_eq_str, + OP_make_state: translate_OP_make_state, + OP_drop_state: translate_OP_drop_state, + OP_halt: translate_OP_halt, } func translate_OP_size_check(p *hir.Builder, v Instr) { - p.ADDI (RL, v.Iv, UR) - p.BLTU (RC, UR, LB_nomem) + p.ADDI(RL, v.Iv, UR) + p.BLTU(RC, UR, LB_nomem) } func translate_OP_size_const(p *hir.Builder, v Instr) { - p.ADDI (RL, v.Iv, RL) + p.ADDI(RL, v.Iv, RL) } func translate_OP_size_dyn(p *hir.Builder, v Instr) { - p.LQ (WP, int64(v.Uv), TR) - p.MULI (TR, v.Iv, TR) - p.ADD (RL, TR, RL) + p.LQ(WP, int64(v.Uv), TR) + p.MULI(TR, v.Iv, TR) + p.ADD(RL, TR, RL) } func translate_OP_size_map(p *hir.Builder, v Instr) { - p.LP (WP, 0, TP) - p.LQ (TP, 0, TR) - p.MULI (TR, v.Iv, TR) - p.ADD (RL, TR, RL) + p.LP(WP, 0, TP) + p.LQ(TP, 0, TR) + p.MULI(TR, v.Iv, TR) + p.ADD(RL, TR, RL) } func translate_OP_size_defer(p *hir.Builder, v Instr) { - p.IP (v.Vt(), TP) - p.GCALL (F_encode). - A0 (TP). - A1 (hir.Pn). - A2 (hir.Rz). - A3 (hir.Pn). - A4 (hir.Pn). - A5 (WP). - A6 (RS). - A7 (ST). - R0 (TR). - R1 (ET). - R2 (EP) - p.BNEP (ET, hir.Pn, LB_error) - p.ADD (RL, TR, RL) + p.IP(v.Vt(), TP) + p.GCALL(F_encode). + A0(TP). + A1(hir.Pn). + A2(hir.Rz). + A3(hir.Pn). + A4(hir.Pn). + A5(WP). + A6(RS). + A7(ST). + R0(TR). + R1(ET). + R2(EP) + p.BNEP(ET, hir.Pn, LB_error) + p.ADD(RL, TR, RL) } func translate_OP_byte(p *hir.Builder, v Instr) { - p.ADDP (RP, RL, TP) - p.ADDI (RL, 1, RL) - p.IB (int8(v.Iv), TR) - p.SB (TR, TP, 0) + p.ADDP(RP, RL, TP) + p.ADDI(RL, 1, RL) + p.IB(int8(v.Iv), TR) + p.SB(TR, TP, 0) } func translate_OP_word(p *hir.Builder, v Instr) { - p.ADDP (RP, RL, TP) - p.ADDI (RL, 2, RL) - p.IW (bswap16(v.Iv), TR) - p.SW (TR, TP, 0) + p.ADDP(RP, RL, TP) + p.ADDI(RL, 2, RL) + p.IW(bswap16(v.Iv), TR) + p.SW(TR, TP, 0) } func translate_OP_long(p *hir.Builder, v Instr) { - p.ADDP (RP, RL, TP) - p.ADDI (RL, 4, RL) - p.IL (bswap32(v.Iv), TR) - p.SL (TR, TP, 0) + p.ADDP(RP, RL, TP) + p.ADDI(RL, 4, RL) + p.IL(bswap32(v.Iv), TR) + p.SL(TR, TP, 0) } func translate_OP_quad(p *hir.Builder, v Instr) { - p.ADDP (RP, RL, TP) - p.ADDI (RL, 8, RL) - p.IQ (bswap64(v.Iv), TR) - p.SQ (TR, TP, 0) + p.ADDP(RP, RL, TP) + p.ADDI(RL, 8, RL) + p.IQ(bswap64(v.Iv), TR) + p.SQ(TR, TP, 0) } func translate_OP_sint(p *hir.Builder, v Instr) { - p.ADDP (RP, RL, TP) - p.ADDI (RL, v.Iv, RL) - - /* check for copy size */ - switch v.Iv { - case 1 : p.LB(WP, 0, TR); p.SB(TR, TP, 0) - case 2 : p.LW(WP, 0, TR); p.SWAPW(TR, TR); p.SW(TR, TP, 0) - case 4 : p.LL(WP, 0, TR); p.SWAPL(TR, TR); p.SL(TR, TP, 0) - case 8 : p.LQ(WP, 0, TR); p.SWAPQ(TR, TR); p.SQ(TR, TP, 0) - default : panic("can only convert 1, 2, 4 or 8 bytes at a time") - } + p.ADDP(RP, RL, TP) + p.ADDI(RL, v.Iv, RL) + + /* check for copy size */ + switch v.Iv { + case 1: + p.LB(WP, 0, TR) + p.SB(TR, TP, 0) + case 2: + p.LW(WP, 0, TR) + p.SWAPW(TR, TR) + p.SW(TR, TP, 0) + case 4: + p.LL(WP, 0, TR) + p.SWAPL(TR, TR) + p.SL(TR, TP, 0) + case 8: + p.LQ(WP, 0, TR) + p.SWAPQ(TR, TR) + p.SQ(TR, TP, 0) + default: + panic("can only convert 1, 2, 4 or 8 bytes at a time") + } } func translate_OP_length(p *hir.Builder, v Instr) { - p.LL (WP, v.Iv, TR) - p.SWAPL (TR, TR) - p.ADDP (RP, RL, TP) - p.ADDI (RL, 4, RL) - p.SL (TR, TP, 0) + p.LL(WP, v.Iv, TR) + p.SWAPL(TR, TR) + p.ADDP(RP, RL, TP) + p.ADDI(RL, 4, RL) + p.SL(TR, TP, 0) } func translate_OP_memcpy_1(p *hir.Builder) { - p.IQ (_N_page, UR) - p.BGEU (UR, TR, "_do_copy_{n}") - p.LDAP (ARG_mem_itab, ET) - p.LDAP (ARG_mem_data, EP) - p.BEQP (EP, hir.Pn, "_do_copy_{n}") - p.SUB (RC, RL, UR) - p.ICALL (ET, EP, utils.FnWrite). - A0 (TP). - A1 (TR). - A2 (TR). - A3 (UR). - R0 (ET). - R1 (EP) - p.BNEP (ET, hir.Pn, LB_error) - p.JMP ("_done_{n}") - p.Label ("_do_copy_{n}") - p.ADD (RL, TR, UR) - p.BLTU (RC, UR, LB_nomem) - p.ADDP (RP, RL, EP) - p.MOV (UR, RL) - p.BCOPY (TP, TR, EP) - p.Label ("_done_{n}") + p.IQ(_N_page, UR) + p.BGEU(UR, TR, "_do_copy_{n}") + p.LDAP(ARG_mem_itab, ET) + p.LDAP(ARG_mem_data, EP) + p.BEQP(EP, hir.Pn, "_do_copy_{n}") + p.SUB(RC, RL, UR) + p.ICALL(ET, EP, utils.FnWrite). + A0(TP). + A1(TR). + A2(TR). + A3(UR). + R0(ET). + R1(EP) + p.BNEP(ET, hir.Pn, LB_error) + p.JMP("_done_{n}") + p.Label("_do_copy_{n}") + p.ADD(RL, TR, UR) + p.BLTU(RC, UR, LB_nomem) + p.ADDP(RP, RL, EP) + p.MOV(UR, RL) + p.BCOPY(TP, TR, EP) + p.Label("_done_{n}") } func translate_OP_memcpy_be(p *hir.Builder, v Instr) { - p.LQ (WP, int64(v.Uv), TR) - p.BEQ (TR, hir.Rz, "_done_{n}") - p.LP (WP, 0, TP) - - /* special case: unit of a single byte */ - if v.Iv == 1 { - translate_OP_memcpy_1(p) - return - } - - /* adjust the buffer length */ - p.MULI (TR, v.Iv, UR) - p.ADD (RL, UR, UR) - p.BLTU (RC, UR, LB_nomem) - p.ADDP (RP, RL, EP) - p.MOV (UR, RL) - p.Label ("_loop_{n}") - p.BEQ (TR, hir.Rz, "_done_{n}") - - /* load-swap-store sequence */ - switch v.Iv { - case 2 : p.LW(TP, 0, UR); p.SWAPW(UR, UR); p.SW(UR, EP, 0) - case 4 : p.LL(TP, 0, UR); p.SWAPL(UR, UR); p.SL(UR, EP, 0) - case 8 : p.LQ(TP, 0, UR); p.SWAPQ(UR, UR); p.SQ(UR, EP, 0) - default : panic("can only swap 2, 4 or 8 bytes at a time") - } - - /* update loop counter */ - p.SUBI (TR, 1, TR) - p.ADDPI (TP, v.Iv, TP) - p.ADDPI (EP, v.Iv, EP) - p.JMP ("_loop_{n}") - p.Label ("_done_{n}") + p.LQ(WP, int64(v.Uv), TR) + p.BEQ(TR, hir.Rz, "_done_{n}") + p.LP(WP, 0, TP) + + /* special case: unit of a single byte */ + if v.Iv == 1 { + translate_OP_memcpy_1(p) + return + } + + /* adjust the buffer length */ + p.MULI(TR, v.Iv, UR) + p.ADD(RL, UR, UR) + p.BLTU(RC, UR, LB_nomem) + p.ADDP(RP, RL, EP) + p.MOV(UR, RL) + p.Label("_loop_{n}") + p.BEQ(TR, hir.Rz, "_done_{n}") + + /* load-swap-store sequence */ + switch v.Iv { + case 2: + p.LW(TP, 0, UR) + p.SWAPW(UR, UR) + p.SW(UR, EP, 0) + case 4: + p.LL(TP, 0, UR) + p.SWAPL(UR, UR) + p.SL(UR, EP, 0) + case 8: + p.LQ(TP, 0, UR) + p.SWAPQ(UR, UR) + p.SQ(UR, EP, 0) + default: + panic("can only swap 2, 4 or 8 bytes at a time") + } + + /* update loop counter */ + p.SUBI(TR, 1, TR) + p.ADDPI(TP, v.Iv, TP) + p.ADDPI(EP, v.Iv, EP) + p.JMP("_loop_{n}") + p.Label("_done_{n}") } func translate_OP_seek(p *hir.Builder, v Instr) { - p.ADDPI (WP, v.Iv, WP) + p.ADDPI(WP, v.Iv, WP) } func translate_OP_deref(p *hir.Builder, _ Instr) { - p.LP (WP, 0, WP) + p.LP(WP, 0, WP) } func translate_OP_defer(p *hir.Builder, v Instr) { - p.IP (v.Vt(), TP) - p.LDAP (ARG_mem_itab, ET) - p.LDAP (ARG_mem_data, EP) - p.SUB (RC, RL, TR) - p.ADDP (RP, RL, RP) - p.GCALL (F_encode). - A0 (TP). - A1 (RP). - A2 (TR). - A3 (ET). - A4 (EP). - A5 (WP). - A6 (RS). - A7 (ST). - R0 (TR). - R1 (ET). - R2 (EP) - p.SUBP (RP, RL, RP) - p.BNEP (ET, hir.Pn, LB_error) - p.ADD (RL, TR, RL) + p.IP(v.Vt(), TP) + p.LDAP(ARG_mem_itab, ET) + p.LDAP(ARG_mem_data, EP) + p.SUB(RC, RL, TR) + p.ADDP(RP, RL, RP) + p.GCALL(F_encode). + A0(TP). + A1(RP). + A2(TR). + A3(ET). + A4(EP). + A5(WP). + A6(RS). + A7(ST). + R0(TR). + R1(ET). + R2(EP) + p.SUBP(RP, RL, RP) + p.BNEP(ET, hir.Pn, LB_error) + p.ADD(RL, TR, RL) } func translate_OP_map_len(p *hir.Builder, _ Instr) { - p.LP (WP, 0, TP) - p.LQ (TP, 0, TR) - p.SWAPL (TR, TR) - p.ADDP (RP, RL, TP) - p.ADDI (RL, 4, RL) - p.SL (TR, TP, 0) + p.LP(WP, 0, TP) + p.LQ(TP, 0, TR) + p.SWAPL(TR, TR) + p.ADDP(RP, RL, TP) + p.ADDI(RL, 4, RL) + p.SL(TR, TP, 0) } func translate_OP_map_key(p *hir.Builder, _ Instr) { - p.ADDP (RS, ST, TP) - p.LP (TP, MiOffset + MiKeyOffset, WP) + p.ADDP(RS, ST, TP) + p.LP(TP, MiOffset+MiKeyOffset, WP) } func translate_OP_map_next(p *hir.Builder, _ Instr) { - p.ADDP (RS, ST, TP) - p.ADDPI (TP, MiOffset, TP) - p.GCALL (F_mapiternext).A0(TP) + p.ADDP(RS, ST, TP) + p.ADDPI(TP, MiOffset, TP) + p.GCALL(F_mapiternext).A0(TP) } func translate_OP_map_value(p *hir.Builder, _ Instr) { - p.ADDP (RS, ST, TP) - p.LP (TP, MiOffset + MiValueOffset, WP) + p.ADDP(RS, ST, TP) + p.LP(TP, MiOffset+MiValueOffset, WP) } func translate_OP_map_begin(p *hir.Builder, v Instr) { - p.IP (v.Vt(), ET) - p.LP (WP, 0, EP) - p.ADDP (RS, ST, TP) - p.ADDPI (TP, MiOffset, TP) - p.GCALL (F_mapiterstart). - A0 (ET). - A1 (EP). - A2 (TP) + p.IP(v.Vt(), ET) + p.LP(WP, 0, EP) + p.ADDP(RS, ST, TP) + p.ADDPI(TP, MiOffset, TP) + p.GCALL(F_mapiterstart). + A0(ET). + A1(EP). + A2(TP) } func translate_OP_map_if_next(p *hir.Builder, v Instr) { - p.ADDP (RS, ST, TP) - p.LP (TP, MiOffset + MiKeyOffset, TP) - p.BNEP (TP, hir.Pn, p.At(v.To)) + p.ADDP(RS, ST, TP) + p.LP(TP, MiOffset+MiKeyOffset, TP) + p.BNEP(TP, hir.Pn, p.At(v.To)) } func translate_OP_map_if_empty(p *hir.Builder, v Instr) { - p.LP (WP, 0, TP) - p.LQ (TP, 0, TR) - p.BEQ (TR, hir.Rz, p.At(v.To)) + p.LP(WP, 0, TP) + p.LQ(TP, 0, TR) + p.BEQ(TR, hir.Rz, p.At(v.To)) } func translate_OP_list_decr(p *hir.Builder, _ Instr) { - p.ADDP (RS, ST, TP) - p.LQ (TP, LnOffset, TR) - p.SUBI (TR, 1, TR) - p.SQ (TR, TP, LnOffset) + p.ADDP(RS, ST, TP) + p.LQ(TP, LnOffset, TR) + p.SUBI(TR, 1, TR) + p.SQ(TR, TP, LnOffset) } func translate_OP_list_begin(p *hir.Builder, _ Instr) { - p.LQ (WP, abi.PtrSize, TR) - p.LP (WP, 0, WP) - p.ADDP (RS, ST, TP) - p.SQ (TR, TP, LnOffset) + p.LQ(WP, abi.PtrSize, TR) + p.LP(WP, 0, WP) + p.ADDP(RS, ST, TP) + p.SQ(TR, TP, LnOffset) } func translate_OP_list_if_next(p *hir.Builder, v Instr) { - p.ADDP (RS, ST, TP) - p.LQ (TP, LnOffset, TR) - p.BNE (TR, hir.Rz, p.At(v.To)) + p.ADDP(RS, ST, TP) + p.LQ(TP, LnOffset, TR) + p.BNE(TR, hir.Rz, p.At(v.To)) } func translate_OP_list_if_empty(p *hir.Builder, v Instr) { - p.LQ (WP, abi.PtrSize, TR) - p.BEQ (TR, hir.Rz, p.At(v.To)) + p.LQ(WP, abi.PtrSize, TR) + p.BEQ(TR, hir.Rz, p.At(v.To)) } func translate_OP_unique(p *hir.Builder, v Instr) { - p.IB (2, UR) - p.LQ (WP, abi.PtrSize, TR) - p.BLTU (TR, UR, "_ok_{n}") - translate_OP_unique_type(p, v.Vt()) - p.Label ("_ok_{n}") + p.IB(2, UR) + p.LQ(WP, abi.PtrSize, TR) + p.BLTU(TR, UR, "_ok_{n}") + translate_OP_unique_type(p, v.Vt()) + p.Label("_ok_{n}") } func translate_OP_unique_type(p *hir.Builder, vt *rt.GoType) { - switch vt.Kind() { - case reflect.Bool : translate_OP_unique_b(p) - case reflect.Int : translate_OP_unique_int(p) - case reflect.Int8 : translate_OP_unique_i8(p) - case reflect.Int16 : translate_OP_unique_i16(p) - case reflect.Int32 : translate_OP_unique_i32(p) - case reflect.Int64 : translate_OP_unique_i64(p) - case reflect.Float64 : translate_OP_unique_i64(p) - case reflect.Map : break - case reflect.Ptr : break - case reflect.Slice : break - case reflect.String : translate_OP_unique_str(p) - case reflect.Struct : break - default : panic("unique: invalid type: " + vt.String()) - } + switch vt.Kind() { + case reflect.Bool: + translate_OP_unique_b(p) + case reflect.Int: + translate_OP_unique_int(p) + case reflect.Int8: + translate_OP_unique_i8(p) + case reflect.Int16: + translate_OP_unique_i16(p) + case reflect.Int32: + translate_OP_unique_i32(p) + case reflect.Int64: + translate_OP_unique_i64(p) + case reflect.Float64: + translate_OP_unique_i64(p) + case reflect.Map: + break + case reflect.Ptr: + break + case reflect.Slice: + break + case reflect.String: + translate_OP_unique_str(p) + case reflect.Struct: + break + default: + panic("unique: invalid type: " + vt.String()) + } } func translate_OP_unique_b(p *hir.Builder) { - p.BLTU (UR, TR, LB_duplicated) - p.LP (WP, 0, TP) - p.LB (TP, 0, TR) - p.LB (TP, 1, UR) - p.BEQ (TR, UR, LB_duplicated) + p.BLTU(UR, TR, LB_duplicated) + p.LP(WP, 0, TP) + p.LB(TP, 0, TR) + p.LB(TP, 1, UR) + p.BEQ(TR, UR, LB_duplicated) } func translate_OP_unique_i8(p *hir.Builder) { - p.IQ (RangeUint8, UR) - p.BLTU (UR, TR, LB_duplicated) - translate_OP_unique_small(p, RangeUint8 / 8, 1, p.LB) + p.IQ(RangeUint8, UR) + p.BLTU(UR, TR, LB_duplicated) + translate_OP_unique_small(p, RangeUint8/8, 1, p.LB) } func translate_OP_unique_i16(p *hir.Builder) { - p.IQ (RangeUint16, UR) - p.BLTU (UR, TR, LB_duplicated) - translate_OP_unique_small(p, RangeUint16 / 8, 2, p.LW) + p.IQ(RangeUint16, UR) + p.BLTU(UR, TR, LB_duplicated) + translate_OP_unique_small(p, RangeUint16/8, 2, p.LW) } func translate_OP_unique_int(p *hir.Builder) { - switch defs.IntSize { - case 4 : translate_OP_unique_i32(p) - case 8 : translate_OP_unique_i64(p) - default : panic("invalid int size") - } + switch defs.IntSize { + case 4: + translate_OP_unique_i32(p) + case 8: + translate_OP_unique_i64(p) + default: + panic("invalid int size") + } } func translate_OP_unique_small(p *hir.Builder, nb int64, dv int64, ld func(hir.PointerRegister, int64, hir.GenericRegister) *hir.Ir) { - p.ADDPI (RS, BmOffset, ET) - p.BZERO (nb, ET) - p.LP (WP, 0, EP) - p.JMP ("_first_{n}") - p.Label ("_loop_{n}") - p.ADDPI (EP, dv, EP) - p.Label ("_first_{n}") - ld (EP, 0, RC) - p.SHRI (RC, 3, UR) - p.ANDI (RC, 0x3f, RC) - p.ANDI (UR, ^0x07, UR) - p.ADDP (ET, UR, TP) - p.LQ (TP, 0, UR) - p.BTS (RC, UR, RC) - p.SQ (UR, TP, 0) - p.BNE (RC, hir.Rz, LB_duplicated) - p.SUBI (TR, 1, TR) - p.BNE (TR, hir.Rz, "_loop_{n}") - p.LDAQ (ARG_len, RC) + p.ADDPI(RS, BmOffset, ET) + p.BZERO(nb, ET) + p.LP(WP, 0, EP) + p.JMP("_first_{n}") + p.Label("_loop_{n}") + p.ADDPI(EP, dv, EP) + p.Label("_first_{n}") + ld(EP, 0, RC) + p.SHRI(RC, 3, UR) + p.ANDI(RC, 0x3f, RC) + p.ANDI(UR, ^0x07, UR) + p.ADDP(ET, UR, TP) + p.LQ(TP, 0, UR) + p.BTS(RC, UR, RC) + p.SQ(UR, TP, 0) + p.BNE(RC, hir.Rz, LB_duplicated) + p.SUBI(TR, 1, TR) + p.BNE(TR, hir.Rz, "_loop_{n}") + p.LDAQ(ARG_len, RC) } func translate_OP_unique_i32(p *hir.Builder) { - p.LP (WP, 0, TP) - p.GCALL (F_unique32). - A0 (TP). - A1 (TR). - R0 (TR) - p.BNE (TR, hir.Rz, LB_duplicated) + p.LP(WP, 0, TP) + p.GCALL(F_unique32). + A0(TP). + A1(TR). + R0(TR) + p.BNE(TR, hir.Rz, LB_duplicated) } func translate_OP_unique_i64(p *hir.Builder) { - p.LP (WP, 0, TP) - p.GCALL (F_unique64). - A0 (TP). - A1 (TR). - R0 (TR) - p.BNE (TR, hir.Rz, LB_duplicated) + p.LP(WP, 0, TP) + p.GCALL(F_unique64). + A0(TP). + A1(TR). + R0(TR) + p.BNE(TR, hir.Rz, LB_duplicated) } func translate_OP_unique_str(p *hir.Builder) { - p.LP (WP, 0, TP) - p.GCALL (F_uniquestr). - A0 (TP). - A1 (TR). - R0 (TR) - p.BNE (TR, hir.Rz, LB_duplicated) + p.LP(WP, 0, TP) + p.GCALL(F_uniquestr). + A0(TP). + A1(TR). + R0(TR) + p.BNE(TR, hir.Rz, LB_duplicated) } func translate_OP_goto(p *hir.Builder, v Instr) { - p.JMP (p.At(v.To)) + p.JMP(p.At(v.To)) } func translate_OP_if_nil(p *hir.Builder, v Instr) { - p.LP (WP, 0, TP) - p.BEQP (TP, hir.Pn, p.At(v.To)) + p.LP(WP, 0, TP) + p.BEQP(TP, hir.Pn, p.At(v.To)) } func translate_OP_if_hasbuf(p *hir.Builder, v Instr) { - p.BNEP (RP, hir.Pn, p.At(v.To)) + p.BNEP(RP, hir.Pn, p.At(v.To)) } func translate_OP_if_eq_imm(p *hir.Builder, v Instr) { - switch v.Uv { - case 1 : p.LB(WP, 0, TR); p.IB( int8(v.Iv), UR); p.BEQ(TR, UR, p.At(v.To)) - case 2 : p.LW(WP, 0, TR); p.IW(int16(v.Iv), UR); p.BEQ(TR, UR, p.At(v.To)) - case 4 : p.LL(WP, 0, TR); p.IL(int32(v.Iv), UR); p.BEQ(TR, UR, p.At(v.To)) - case 8 : p.LQ(WP, 0, TR); p.IQ( v.Iv , UR); p.BEQ(TR, UR, p.At(v.To)) - default : panic("invalid imm size") - } + switch v.Uv { + case 1: + p.LB(WP, 0, TR) + p.IB(int8(v.Iv), UR) + p.BEQ(TR, UR, p.At(v.To)) + case 2: + p.LW(WP, 0, TR) + p.IW(int16(v.Iv), UR) + p.BEQ(TR, UR, p.At(v.To)) + case 4: + p.LL(WP, 0, TR) + p.IL(int32(v.Iv), UR) + p.BEQ(TR, UR, p.At(v.To)) + case 8: + p.LQ(WP, 0, TR) + p.IQ(v.Iv, UR) + p.BEQ(TR, UR, p.At(v.To)) + default: + panic("invalid imm size") + } } func translate_OP_if_eq_str(p *hir.Builder, v Instr) { - nb := v.Iv - to := p.At(v.To) - - /* load the string length */ - p.IQ (v.Iv, TR) - p.LQ (WP, abi.PtrSize, UR) - - /* empty string */ - if v.Iv == 0 { - p.BEQ(TR, UR, to) - return - } - - /* compare the string pointers */ - p.BNE (TR, UR, "_neq_{n}") - p.IP (v.Pr, TP) - p.LP (WP, 0, EP) - p.BEQP (TP, EP, to) - - /* compare the content, 4-byte loop */ - for nb >= 4 { - p.LL (EP, v.Iv - nb, TR) - p.IL (v.Long(v.Iv - nb), UR) - p.BNE (TR, UR, "_neq_{n}") - nb -= 4 - } - - /* compare the content, 2-byte test */ - if nb >= 2 { - p.LW (EP, v.Iv - nb, TR) - p.IW (v.Word(v.Iv - nb), UR) - p.BNE (TR, UR, "_neq_{n}") - nb -= 2 - } - - /* compare the content, the last byte */ - if nb != 0 { - p.LB (EP, v.Iv - 1, TR) - p.IB (v.Byte(v.Iv - 1), UR) - p.BNE (TR, UR, "_neq_{n}") - } - - /* two string are equal */ - p.JMP (to) - p.Label ("_neq_{n}") + nb := v.Iv + to := p.At(v.To) + + /* load the string length */ + p.IQ(v.Iv, TR) + p.LQ(WP, abi.PtrSize, UR) + + /* empty string */ + if v.Iv == 0 { + p.BEQ(TR, UR, to) + return + } + + /* compare the string pointers */ + p.BNE(TR, UR, "_neq_{n}") + p.IP(v.Pr, TP) + p.LP(WP, 0, EP) + p.BEQP(TP, EP, to) + + /* compare the content, 4-byte loop */ + for nb >= 4 { + p.LL(EP, v.Iv-nb, TR) + p.IL(v.Long(v.Iv-nb), UR) + p.BNE(TR, UR, "_neq_{n}") + nb -= 4 + } + + /* compare the content, 2-byte test */ + if nb >= 2 { + p.LW(EP, v.Iv-nb, TR) + p.IW(v.Word(v.Iv-nb), UR) + p.BNE(TR, UR, "_neq_{n}") + nb -= 2 + } + + /* compare the content, the last byte */ + if nb != 0 { + p.LB(EP, v.Iv-1, TR) + p.IB(v.Byte(v.Iv-1), UR) + p.BNE(TR, UR, "_neq_{n}") + } + + /* two string are equal */ + p.JMP(to) + p.Label("_neq_{n}") } func translate_OP_make_state(p *hir.Builder, _ Instr) { - p.IQ (StateMax, TR) - p.BGEU (ST, TR, LB_overflow) - p.ADDP (RS, ST, TP) - p.SP (WP, TP, WpOffset) - p.ADDI (ST, StateSize, ST) + p.IQ(StateMax, TR) + p.BGEU(ST, TR, LB_overflow) + p.ADDP(RS, ST, TP) + p.SP(WP, TP, WpOffset) + p.ADDI(ST, StateSize, ST) } func translate_OP_drop_state(p *hir.Builder, _ Instr) { - p.SUBI (ST, StateSize, ST) - p.ADDP (RS, ST, TP) - p.LP (TP, WpOffset, WP) - p.SP (hir.Pn, TP, WpOffset) + p.SUBI(ST, StateSize, ST) + p.ADDP(RS, ST, TP) + p.LP(TP, WpOffset, WP) + p.SP(hir.Pn, TP, WpOffset) } func translate_OP_halt(p *hir.Builder, _ Instr) { - p.JMP (LB_halt) + p.JMP(LB_halt) } diff --git a/internal/binary/encoder/translator_test.go b/internal/binary/encoder/translator_test.go index 8b5a116..1d173c5 100644 --- a/internal/binary/encoder/translator_test.go +++ b/internal/binary/encoder/translator_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,36 +17,36 @@ package encoder import ( - `reflect` - `testing` + "reflect" + "testing" - `github.com/stretchr/testify/require` + "github.com/stretchr/testify/require" ) type TranslatorTestStruct struct { - A bool `frugal:"0,default,bool"` - B int8 `frugal:"1,default,i8"` - C float64 `frugal:"2,default,double"` - D int16 `frugal:"3,default,i16"` - E int32 `frugal:"4,default,i32"` - F int64 `frugal:"5,default,i64"` - G string `frugal:"6,default,string"` - H []byte `frugal:"7,default,binary"` - I []int32 `frugal:"8,default,list"` - J map[string]string `frugal:"9,default,map"` - K map[string]*TranslatorTestStruct `frugal:"65,default,map"` - L *bool `frugal:"11,optional,bool"` - M *int8 `frugal:"12,optional,i8"` - N *float64 `frugal:"13,optional,double"` - O *int16 `frugal:"14,optional,i16"` - P *int32 `frugal:"15,optional,i32"` - Q *int64 `frugal:"16,optional,i64"` + A bool `frugal:"0,default,bool"` + B int8 `frugal:"1,default,i8"` + C float64 `frugal:"2,default,double"` + D int16 `frugal:"3,default,i16"` + E int32 `frugal:"4,default,i32"` + F int64 `frugal:"5,default,i64"` + G string `frugal:"6,default,string"` + H []byte `frugal:"7,default,binary"` + I []int32 `frugal:"8,default,list"` + J map[string]string `frugal:"9,default,map"` + K map[string]*TranslatorTestStruct `frugal:"65,default,map"` + L *bool `frugal:"11,optional,bool"` + M *int8 `frugal:"12,optional,i8"` + N *float64 `frugal:"13,optional,double"` + O *int16 `frugal:"14,optional,i16"` + P *int32 `frugal:"15,optional,i32"` + Q *int64 `frugal:"16,optional,i64"` } func TestTranslator_Translate(t *testing.T) { - var v TranslatorTestStruct - p, err := CreateCompiler().Compile(reflect.TypeOf(v)) - require.NoError(t, err) - tr := Translate(p) - println(tr.Disassemble()) + var v TranslatorTestStruct + p, err := CreateCompiler().Compile(reflect.TypeOf(v)) + require.NoError(t, err) + tr := Translate(p) + println(tr.Disassemble()) } diff --git a/internal/binary/encoder/unique.go b/internal/binary/encoder/unique.go index d7c3666..9d53ed0 100644 --- a/internal/binary/encoder/unique.go +++ b/internal/binary/encoder/unique.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,74 +17,74 @@ package encoder import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" ) const ( - _N_u32 = unsafe.Sizeof(uint32(0)) - _N_u64 = unsafe.Sizeof(uint64(0)) - _N_str = unsafe.Sizeof(rt.GoString{}) + _N_u32 = unsafe.Sizeof(uint32(0)) + _N_u64 = unsafe.Sizeof(uint64(0)) + _N_str = unsafe.Sizeof(rt.GoString{}) ) func u32at(p unsafe.Pointer, i int) uint32 { - return *(*uint32)(unsafe.Pointer(uintptr(p) + uintptr(i) * _N_u32)) + return *(*uint32)(unsafe.Pointer(uintptr(p) + uintptr(i)*_N_u32)) } func u64at(p unsafe.Pointer, i int) uint64 { - return *(*uint64)(unsafe.Pointer(uintptr(p) + uintptr(i) * _N_u64)) + return *(*uint64)(unsafe.Pointer(uintptr(p) + uintptr(i)*_N_u64)) } func straddr(p unsafe.Pointer, i int) unsafe.Pointer { - return unsafe.Pointer(uintptr(p) + uintptr(i) * _N_str) + return unsafe.Pointer(uintptr(p) + uintptr(i)*_N_str) } func unique32(p unsafe.Pointer, nb int) bool { - dup := false - bmp := newBucket(nb * 2) + dup := false + bmp := newBucket(nb * 2) - /* put all the items */ - for i := 0; !dup && i < nb; i++ { - dup = bucketAppend64(bmp, uint64(u32at(p, i))) - } + /* put all the items */ + for i := 0; !dup && i < nb; i++ { + dup = bucketAppend64(bmp, uint64(u32at(p, i))) + } - /* free the bucket */ - freeBucket(bmp) - return dup + /* free the bucket */ + freeBucket(bmp) + return dup } func unique64(p unsafe.Pointer, nb int) bool { - dup := false - bmp := newBucket(nb * 2) + dup := false + bmp := newBucket(nb * 2) - /* put all the items */ - for i := 0; !dup && i < nb; i++ { - dup = bucketAppend64(bmp, u64at(p, i)) - } + /* put all the items */ + for i := 0; !dup && i < nb; i++ { + dup = bucketAppend64(bmp, u64at(p, i)) + } - /* free the bucket */ - freeBucket(bmp) - return dup + /* free the bucket */ + freeBucket(bmp) + return dup } func uniquestr(p unsafe.Pointer, nb int) bool { - dup := false - bmp := newBucket(nb * 2) + dup := false + bmp := newBucket(nb * 2) - /* put all the items */ - for i := 0; !dup && i < nb; i++ { - dup = bucketAppendStr(bmp, straddr(p, i)) - } + /* put all the items */ + for i := 0; !dup && i < nb; i++ { + dup = bucketAppendStr(bmp, straddr(p, i)) + } - /* free the bucket */ - freeBucket(bmp) - return dup + /* free the bucket */ + freeBucket(bmp) + return dup } var ( - F_unique32 = hir.RegisterGCall(unique32, emu_gcall_unique32) - F_unique64 = hir.RegisterGCall(unique64, emu_gcall_unique64) - F_uniquestr = hir.RegisterGCall(uniquestr, emu_gcall_uniquestr) + F_unique32 = hir.RegisterGCall(unique32, emu_gcall_unique32) + F_unique64 = hir.RegisterGCall(unique64, emu_gcall_unique64) + F_uniquestr = hir.RegisterGCall(uniquestr, emu_gcall_uniquestr) ) diff --git a/internal/binary/encoder/unique_emu.go b/internal/binary/encoder/unique_emu.go index e54f6ca..ba37851 100644 --- a/internal/binary/encoder/unique_emu.go +++ b/internal/binary/encoder/unique_emu.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,37 +17,37 @@ package encoder import ( - `github.com/cloudwego/frugal/internal/atm/hir` + "github.com/cloudwego/frugal/internal/atm/hir" ) func bool2u64(v bool) uint64 { - if v { - return 1 - } else { - return 0 - } + if v { + return 1 + } else { + return 0 + } } func emu_gcall_unique32(ctx hir.CallContext) { - if !ctx.Verify("*i", "i") { - panic("invalid unique32 call") - } else { - ctx.Ru(0, bool2u64(unique32(ctx.Ap(0), int(ctx.Au(1))))) - } + if !ctx.Verify("*i", "i") { + panic("invalid unique32 call") + } else { + ctx.Ru(0, bool2u64(unique32(ctx.Ap(0), int(ctx.Au(1))))) + } } func emu_gcall_unique64(ctx hir.CallContext) { - if !ctx.Verify("*i", "i") { - panic("invalid unique64 call") - } else { - ctx.Ru(0, bool2u64(unique64(ctx.Ap(0), int(ctx.Au(1))))) - } + if !ctx.Verify("*i", "i") { + panic("invalid unique64 call") + } else { + ctx.Ru(0, bool2u64(unique64(ctx.Ap(0), int(ctx.Au(1))))) + } } func emu_gcall_uniquestr(ctx hir.CallContext) { - if !ctx.Verify("*i", "i") { - panic("invalid uniquestr call") - } else { - ctx.Ru(0, bool2u64(uniquestr(ctx.Ap(0), int(ctx.Au(1))))) - } + if !ctx.Verify("*i", "i") { + panic("invalid uniquestr call") + } else { + ctx.Ru(0, bool2u64(uniquestr(ctx.Ap(0), int(ctx.Au(1))))) + } } diff --git a/internal/binary/encoder/utils.go b/internal/binary/encoder/utils.go index 528b078..76bde29 100644 --- a/internal/binary/encoder/utils.go +++ b/internal/binary/encoder/utils.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,30 +17,30 @@ package encoder import ( - `math/bits` - `unsafe` + "math/bits" + "unsafe" ) func bswap16(v int64) int16 { - return int16(bits.ReverseBytes16(uint16(v))) + return int16(bits.ReverseBytes16(uint16(v))) } func bswap32(v int64) int32 { - return int32(bits.ReverseBytes32(uint32(v))) + return int32(bits.ReverseBytes32(uint32(v))) } func bswap64(v int64) int64 { - return int64(bits.ReverseBytes64(uint64(v))) + return int64(bits.ReverseBytes64(uint64(v))) } func mem2str(v []byte) string { - return *(*string)(unsafe.Pointer(&v)) + return *(*string)(unsafe.Pointer(&v)) } func bool2i64(v bool) int64 { - if v { - return 1 - } else { - return 0 - } + if v { + return 1 + } else { + return 0 + } } diff --git a/internal/cpu/features.go b/internal/cpu/features.go index aa547d3..bf071f5 100644 --- a/internal/cpu/features.go +++ b/internal/cpu/features.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,9 @@ package cpu import ( - `github.com/klauspost/cpuid/v2` + "github.com/klauspost/cpuid/v2" ) var ( - HasMOVBE = cpuid.CPU.Has(cpuid.MOVBE) + HasMOVBE = cpuid.CPU.Has(cpuid.MOVBE) ) diff --git a/internal/loader/funcdata.go b/internal/loader/funcdata.go index b23e180..43ddb90 100644 --- a/internal/loader/funcdata.go +++ b/internal/loader/funcdata.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,16 +17,16 @@ package loader import ( - `sync/atomic` - `unsafe` + "sync/atomic" + "unsafe" - `github.com/cloudwego/frugal/internal/utils` + "github.com/cloudwego/frugal/internal/utils" ) const ( - _PCDATA_UnsafePoint = 0 - _PCDATA_StackMapIndex = 1 - _PCDATA_UnsafePointUnsafe = -2 + _PCDATA_UnsafePoint = 0 + _PCDATA_StackMapIndex = 1 + _PCDATA_UnsafePointUnsafe = -2 ) //go:linkname lastmoduledatap runtime.lastmoduledatap @@ -37,70 +37,70 @@ var lastmoduledatap *_ModuleData func moduledataverify1(_ *_ModuleData) var ( - /* retains local reference of all modules to bypass gc */ - modList = utils.ListNode{} + /* retains local reference of all modules to bypass gc */ + modList = utils.ListNode{} ) func toZigzag(v int) int { - return (v << 1) ^ (v >> 31) + return (v << 1) ^ (v >> 31) } func encodeFirst(v int) []byte { - return encodeValue(v + 1) + return encodeValue(v + 1) } func encodeValue(v int) []byte { - return encodeVariant(toZigzag(v)) + return encodeVariant(toZigzag(v)) } func encodeVariant(v int) []byte { - var u int - var r []byte - - /* split every 7 bits */ - for v > 127 { - u = v & 0x7f - v = v >> 7 - r = append(r, byte(u) | 0x80) - } - - /* check for last one */ - if v == 0 { - return r - } - - /* add the last one */ - r = append(r, byte(v)) - return r + var u int + var r []byte + + /* split every 7 bits */ + for v > 127 { + u = v & 0x7f + v = v >> 7 + r = append(r, byte(u)|0x80) + } + + /* check for last one */ + if v == 0 { + return r + } + + /* add the last one */ + r = append(r, byte(v)) + return r } func registerModule(mod *_ModuleData) { - modList.Prepend(unsafe.Pointer(mod)) - registerModuleLockFree(&lastmoduledatap, mod) + modList.Prepend(unsafe.Pointer(mod)) + registerModuleLockFree(&lastmoduledatap, mod) } func registerModuleLockFree(tail **_ModuleData, mod *_ModuleData) { - for { - oldTail := loadModule(tail) - if casModule(tail, oldTail, mod) { - storeModule(&oldTail.next, mod) - break - } - } + for { + oldTail := loadModule(tail) + if casModule(tail, oldTail, mod) { + storeModule(&oldTail.next, mod) + break + } + } } func loadModule(p **_ModuleData) *_ModuleData { - return (*_ModuleData)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p)))) + return (*_ModuleData)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p)))) } func storeModule(p **_ModuleData, value *_ModuleData) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(value)) + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(value)) } func casModule(p **_ModuleData, oldValue *_ModuleData, newValue *_ModuleData) bool { - return atomic.CompareAndSwapPointer( - (*unsafe.Pointer)(unsafe.Pointer(p)), - unsafe.Pointer(oldValue), - unsafe.Pointer(newValue), - ) + return atomic.CompareAndSwapPointer( + (*unsafe.Pointer)(unsafe.Pointer(p)), + unsafe.Pointer(oldValue), + unsafe.Pointer(newValue), + ) } diff --git a/internal/loader/funcdata_go116_117.go b/internal/loader/funcdata_go116_117.go index 126c4c9..e073dbc 100644 --- a/internal/loader/funcdata_go116_117.go +++ b/internal/loader/funcdata_go116_117.go @@ -1,7 +1,8 @@ +//go:build go1.16 && !go1.18 // +build go1.16,!go1.18 /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,212 +20,212 @@ package loader import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/rt` - `github.com/cloudwego/frugal/internal/utils` + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/frugal/internal/utils" ) type _Func struct { - entry uintptr - nameoff int32 - args int32 - deferreturn uint32 - pcsp uint32 - pcfile uint32 - pcln uint32 - npcdata uint32 - cuOffset uint32 - funcID uint8 - _ [2]byte - nfuncdata uint8 - pcdata [2]uint32 - argptrs uintptr - localptrs uintptr + entry uintptr + nameoff int32 + args int32 + deferreturn uint32 + pcsp uint32 + pcfile uint32 + pcln uint32 + npcdata uint32 + cuOffset uint32 + funcID uint8 + _ [2]byte + nfuncdata uint8 + pcdata [2]uint32 + argptrs uintptr + localptrs uintptr } type _FuncTab struct { - entry uintptr - funcoff uintptr + entry uintptr + funcoff uintptr } type _PCHeader struct { - magic uint32 - pad1, pad2 uint8 - minLC uint8 - ptrSize uint8 - nfunc int - nfiles uint - funcnameOffset uintptr - cuOffset uintptr - filetabOffset uintptr - pctabOffset uintptr - pclnOffset uintptr + magic uint32 + pad1, pad2 uint8 + minLC uint8 + ptrSize uint8 + nfunc int + nfiles uint + funcnameOffset uintptr + cuOffset uintptr + filetabOffset uintptr + pctabOffset uintptr + pclnOffset uintptr } type _BitVector struct { - n int32 - bytedata *uint8 + n int32 + bytedata *uint8 } type _ModuleData struct { - pcHeader *_PCHeader - funcnametab []byte - cutab []uint32 - filetab []byte - pctab []byte - pclntable []_Func - ftab []_FuncTab - findfunctab *_FindFuncBucket - minpc, maxpc uintptr - text, etext uintptr - noptrdata, enoptrdata uintptr - data, edata uintptr - bss, ebss uintptr - noptrbss, enoptrbss uintptr - end, gcdata, gcbss uintptr - types, etypes uintptr - textsectmap [][3]uintptr - typelinks []int32 - itablinks []unsafe.Pointer - ptab [][2]int32 - pluginpath string - pkghashes []struct{} - modulename string - modulehashes []struct{} - hasmain uint8 - gcdatamask, gcbssmask _BitVector - typemap map[int32]unsafe.Pointer - bad bool - next *_ModuleData + pcHeader *_PCHeader + funcnametab []byte + cutab []uint32 + filetab []byte + pctab []byte + pclntable []_Func + ftab []_FuncTab + findfunctab *_FindFuncBucket + minpc, maxpc uintptr + text, etext uintptr + noptrdata, enoptrdata uintptr + data, edata uintptr + bss, ebss uintptr + noptrbss, enoptrbss uintptr + end, gcdata, gcbss uintptr + types, etypes uintptr + textsectmap [][3]uintptr + typelinks []int32 + itablinks []unsafe.Pointer + ptab [][2]int32 + pluginpath string + pkghashes []struct{} + modulename string + modulehashes []struct{} + hasmain uint8 + gcdatamask, gcbssmask _BitVector + typemap map[int32]unsafe.Pointer + bad bool + next *_ModuleData } type _FindFuncBucket struct { - idx uint32 - subbuckets [16]byte + idx uint32 + subbuckets [16]byte } const minfunc = 16 // minimum function size const pcbucketsize = 256 * minfunc // size of bucket in the pc->func lookup table -var modHeader = &_PCHeader { - magic : 0xfffffffa, - minLC : 1, - nfunc : 1, - ptrSize : 4 << (^uintptr(0) >> 63), +var modHeader = &_PCHeader{ + magic: 0xfffffffa, + minLC: 1, + nfunc: 1, + ptrSize: 4 << (^uintptr(0) >> 63), } var ( - emptyByte byte + emptyByte byte - /* retain local reference of all buckets to bypass gc */ - bucketList = &utils.ListNode{} + /* retain local reference of all buckets to bypass gc */ + bucketList = &utils.ListNode{} ) func registerFunction(name string, pc uintptr, size uintptr, frame rt.Frame) { - var pbase uintptr - var sbase uintptr - - /* PC ranges */ - minpc := pc - maxpc := pc + size - pctab := make([]byte, 1) - ffunc := make([]_FindFuncBucket, size / pcbucketsize + 1) - - /* initialize the find function buckets */ - for i := range ffunc { - ffunc[i].idx = 1 - } - - /* define the PC-SP ranges */ - for i, r := range frame.SpTab { - nb := r.Nb - ds := int(r.Sp - sbase) - - /* check for remaining size */ - if nb == 0 { - if i == len(frame.SpTab) - 1 { - nb = size - pbase - } else { - panic("invalid PC-SP tab") - } - } - - /* check for the first entry */ - if i == 0 { - pctab = append(pctab, encodeFirst(ds)...) - } else { - pctab = append(pctab, encodeValue(ds)...) - } - - /* encode the length */ - sbase = r.Sp - pbase = pbase + nb - pctab = append(pctab, encodeVariant(int(nb))...) - } - - /* pin the find function bucket */ - ftab := &ffunc[0] - pctab = append(pctab, 0) - bucketList.Prepend(unsafe.Pointer(ftab)) - - /* function entry */ - fn := _Func { - entry : pc, - nameoff : 1, - args : int32(frame.ArgSize), - pcsp : 1, - npcdata : 2, - nfuncdata : 2, - cuOffset : 1, - argptrs : frame.ArgPtrs.Pin(), - localptrs : frame.LocalPtrs.Pin(), - } - - /* mark the entire function as a single line of code */ - fn.pcln = uint32(len(pctab)) - fn.pcfile = uint32(len(pctab)) - pctab = append(pctab, encodeFirst(1)...) - pctab = append(pctab, encodeVariant(int(size))...) - pctab = append(pctab, 0) - - /* set the entire function to use stack map 0 */ - fn.pcdata[_PCDATA_StackMapIndex] = uint32(len(pctab)) - pctab = append(pctab, encodeFirst(0)...) - pctab = append(pctab, encodeVariant(int(size))...) - pctab = append(pctab, 0) - - /* mark the entire function as unsafe to async-preempt */ - fn.pcdata[_PCDATA_UnsafePoint] = uint32(len(pctab)) - pctab = append(pctab, encodeFirst(_PCDATA_UnsafePointUnsafe)...) - pctab = append(pctab, encodeVariant(int(size))...) - pctab = append(pctab, 0) - - /* function table */ - tab := []_FuncTab { - {entry: pc}, - {entry: pc}, - {entry: maxpc}, - } - - /* module data */ - mod := &_ModuleData { - pcHeader : modHeader, - funcnametab : append(append([]byte{0}, name...), 0), - cutab : []uint32{0, 0, 1}, - filetab : []byte("\x00(jit-generated)\x00"), - pctab : pctab, - pclntable : []_Func{fn}, - ftab : tab, - findfunctab : ftab, - minpc : minpc, - maxpc : maxpc, - modulename : name, - gcdata : uintptr(unsafe.Pointer(&emptyByte)), - gcbss : uintptr(unsafe.Pointer(&emptyByte)), - } - - /* verify and register the new module */ - moduledataverify1(mod) - registerModule(mod) + var pbase uintptr + var sbase uintptr + + /* PC ranges */ + minpc := pc + maxpc := pc + size + pctab := make([]byte, 1) + ffunc := make([]_FindFuncBucket, size/pcbucketsize+1) + + /* initialize the find function buckets */ + for i := range ffunc { + ffunc[i].idx = 1 + } + + /* define the PC-SP ranges */ + for i, r := range frame.SpTab { + nb := r.Nb + ds := int(r.Sp - sbase) + + /* check for remaining size */ + if nb == 0 { + if i == len(frame.SpTab)-1 { + nb = size - pbase + } else { + panic("invalid PC-SP tab") + } + } + + /* check for the first entry */ + if i == 0 { + pctab = append(pctab, encodeFirst(ds)...) + } else { + pctab = append(pctab, encodeValue(ds)...) + } + + /* encode the length */ + sbase = r.Sp + pbase = pbase + nb + pctab = append(pctab, encodeVariant(int(nb))...) + } + + /* pin the find function bucket */ + ftab := &ffunc[0] + pctab = append(pctab, 0) + bucketList.Prepend(unsafe.Pointer(ftab)) + + /* function entry */ + fn := _Func{ + entry: pc, + nameoff: 1, + args: int32(frame.ArgSize), + pcsp: 1, + npcdata: 2, + nfuncdata: 2, + cuOffset: 1, + argptrs: frame.ArgPtrs.Pin(), + localptrs: frame.LocalPtrs.Pin(), + } + + /* mark the entire function as a single line of code */ + fn.pcln = uint32(len(pctab)) + fn.pcfile = uint32(len(pctab)) + pctab = append(pctab, encodeFirst(1)...) + pctab = append(pctab, encodeVariant(int(size))...) + pctab = append(pctab, 0) + + /* set the entire function to use stack map 0 */ + fn.pcdata[_PCDATA_StackMapIndex] = uint32(len(pctab)) + pctab = append(pctab, encodeFirst(0)...) + pctab = append(pctab, encodeVariant(int(size))...) + pctab = append(pctab, 0) + + /* mark the entire function as unsafe to async-preempt */ + fn.pcdata[_PCDATA_UnsafePoint] = uint32(len(pctab)) + pctab = append(pctab, encodeFirst(_PCDATA_UnsafePointUnsafe)...) + pctab = append(pctab, encodeVariant(int(size))...) + pctab = append(pctab, 0) + + /* function table */ + tab := []_FuncTab{ + {entry: pc}, + {entry: pc}, + {entry: maxpc}, + } + + /* module data */ + mod := &_ModuleData{ + pcHeader: modHeader, + funcnametab: append(append([]byte{0}, name...), 0), + cutab: []uint32{0, 0, 1}, + filetab: []byte("\x00(jit-generated)\x00"), + pctab: pctab, + pclntable: []_Func{fn}, + ftab: tab, + findfunctab: ftab, + minpc: minpc, + maxpc: maxpc, + modulename: name, + gcdata: uintptr(unsafe.Pointer(&emptyByte)), + gcbss: uintptr(unsafe.Pointer(&emptyByte)), + } + + /* verify and register the new module */ + moduledataverify1(mod) + registerModule(mod) } diff --git a/internal/loader/funcdata_go118_121.go b/internal/loader/funcdata_go118_121.go index e8baf20..60adfbc 100644 --- a/internal/loader/funcdata_go118_121.go +++ b/internal/loader/funcdata_go118_121.go @@ -2,7 +2,7 @@ // +build go1.18,!go1.23 /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,174 +20,173 @@ package loader import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/rt` - `github.com/cloudwego/frugal/internal/utils` + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/frugal/internal/utils" ) type _FuncTab struct { - entry uint32 - funcoff uint32 + entry uint32 + funcoff uint32 } type _PCHeader struct { - magic uint32 - pad1, pad2 uint8 - minLC uint8 - ptrSize uint8 - nfunc int - nfiles uint - textStart uintptr - funcnameOffset uintptr - cuOffset uintptr - filetabOffset uintptr - pctabOffset uintptr - pclnOffset uintptr + magic uint32 + pad1, pad2 uint8 + minLC uint8 + ptrSize uint8 + nfunc int + nfiles uint + textStart uintptr + funcnameOffset uintptr + cuOffset uintptr + filetabOffset uintptr + pctabOffset uintptr + pclnOffset uintptr } type _BitVector struct { - n int32 // # of bits - bytedata *uint8 + n int32 // # of bits + bytedata *uint8 } type _FindFuncBucket struct { - idx uint32 - subbuckets [16]byte + idx uint32 + subbuckets [16]byte } const minfunc = 16 const pcbucketsize = 256 * minfunc var ( - emptyByte byte + emptyByte byte - /* retain local reference of all buckets to bypass gc */ - bucketList = &utils.ListNode{} + /* retain local reference of all buckets to bypass gc */ + bucketList = &utils.ListNode{} ) func registerFunction(name string, pc uintptr, size uintptr, frame rt.Frame) { - var pbase uintptr - var sbase uintptr - - /* PC ranges */ - minpc := pc - maxpc := pc + size - pctab := make([]byte, 1) - ffunc := make([]_FindFuncBucket, size / pcbucketsize + 1) - - /* define the PC-SP ranges */ - for i, r := range frame.SpTab { - nb := r.Nb - ds := int(r.Sp - sbase) - - /* check for remaining size */ - if nb == 0 { - if i == len(frame.SpTab) - 1 { - nb = size - pbase - } else { - panic("invalid PC-SP tab") - } - } - - /* check for the first entry */ - if i == 0 { - pctab = append(pctab, encodeFirst(ds)...) - } else { - pctab = append(pctab, encodeValue(ds)...) - } - - /* encode the length */ - sbase = r.Sp - pbase = pbase + nb - pctab = append(pctab, encodeVariant(int(nb))...) - } - - /* pin the find function bucket */ - ftab := &ffunc[0] - pctab = append(pctab, 0) - bucketList.Prepend(unsafe.Pointer(ftab)) - - - /* pin the pointer maps */ - argptrs := frame.ArgPtrs.Pin() - localptrs := frame.LocalPtrs.Pin() - - /* find the lower base */ - if argptrs < localptrs { - pbase = argptrs - } else { - pbase = localptrs - } - - /* function entry */ - fn := _Func { - entryOff : 0, - nameoff : 1, - args : int32(frame.ArgSize), - pcsp : 1, - npcdata : 2, - cuOffset : 1, - nfuncdata : 2, - argptrs : uint32(argptrs - pbase), - localptrs : uint32(localptrs - pbase), - } - - /* mark the entire function as a single line of code */ - fn.pcln = uint32(len(pctab)) - fn.pcfile = uint32(len(pctab)) - pctab = append(pctab, encodeFirst(1)...) - pctab = append(pctab, encodeVariant(int(size))...) - pctab = append(pctab, 0) - - /* set the entire function to use stack map 0 */ - fn.pcdata[_PCDATA_StackMapIndex] = uint32(len(pctab)) - pctab = append(pctab, encodeFirst(0)...) - pctab = append(pctab, encodeVariant(int(size))...) - pctab = append(pctab, 0) - - /* mark the entire function as unsafe to async-preempt */ - fn.pcdata[_PCDATA_UnsafePoint] = uint32(len(pctab)) - pctab = append(pctab, encodeFirst(_PCDATA_UnsafePointUnsafe)...) - pctab = append(pctab, encodeVariant(int(size))...) - pctab = append(pctab, 0) - - /* module header */ - hdr := &_PCHeader { - magic : _ModuleMagic, - minLC : 1, - nfunc : 1, - ptrSize : 4 << (^uintptr(0) >> 63), - textStart : minpc, - } - - /* function table */ - tab := []_FuncTab { - { entry: 0 }, - { entry: uint32(size) }, - } - - /* module data */ - mod := &_ModuleData { - pcHeader : hdr, - funcnametab : append(append([]byte{0}, name...), 0), - cutab : []uint32{0, 0, 1}, - filetab : []byte("\x00(jit-generated)\x00"), - pctab : pctab, - pclntable : ((*[unsafe.Sizeof(_Func{})]byte)(unsafe.Pointer(&fn)))[:], - ftab : tab, - findfunctab : uintptr(unsafe.Pointer(ftab)), - minpc : minpc, - maxpc : maxpc, - text : minpc, - etext : maxpc, - modulename : name, - gcdata : uintptr(unsafe.Pointer(&emptyByte)), - gcbss : uintptr(unsafe.Pointer(&emptyByte)), - gofunc : pbase, - } - - /* verify and register the new module */ - moduledataverify1(mod) - registerModule(mod) + var pbase uintptr + var sbase uintptr + + /* PC ranges */ + minpc := pc + maxpc := pc + size + pctab := make([]byte, 1) + ffunc := make([]_FindFuncBucket, size/pcbucketsize+1) + + /* define the PC-SP ranges */ + for i, r := range frame.SpTab { + nb := r.Nb + ds := int(r.Sp - sbase) + + /* check for remaining size */ + if nb == 0 { + if i == len(frame.SpTab)-1 { + nb = size - pbase + } else { + panic("invalid PC-SP tab") + } + } + + /* check for the first entry */ + if i == 0 { + pctab = append(pctab, encodeFirst(ds)...) + } else { + pctab = append(pctab, encodeValue(ds)...) + } + + /* encode the length */ + sbase = r.Sp + pbase = pbase + nb + pctab = append(pctab, encodeVariant(int(nb))...) + } + + /* pin the find function bucket */ + ftab := &ffunc[0] + pctab = append(pctab, 0) + bucketList.Prepend(unsafe.Pointer(ftab)) + + /* pin the pointer maps */ + argptrs := frame.ArgPtrs.Pin() + localptrs := frame.LocalPtrs.Pin() + + /* find the lower base */ + if argptrs < localptrs { + pbase = argptrs + } else { + pbase = localptrs + } + + /* function entry */ + fn := _Func{ + entryOff: 0, + nameoff: 1, + args: int32(frame.ArgSize), + pcsp: 1, + npcdata: 2, + cuOffset: 1, + nfuncdata: 2, + argptrs: uint32(argptrs - pbase), + localptrs: uint32(localptrs - pbase), + } + + /* mark the entire function as a single line of code */ + fn.pcln = uint32(len(pctab)) + fn.pcfile = uint32(len(pctab)) + pctab = append(pctab, encodeFirst(1)...) + pctab = append(pctab, encodeVariant(int(size))...) + pctab = append(pctab, 0) + + /* set the entire function to use stack map 0 */ + fn.pcdata[_PCDATA_StackMapIndex] = uint32(len(pctab)) + pctab = append(pctab, encodeFirst(0)...) + pctab = append(pctab, encodeVariant(int(size))...) + pctab = append(pctab, 0) + + /* mark the entire function as unsafe to async-preempt */ + fn.pcdata[_PCDATA_UnsafePoint] = uint32(len(pctab)) + pctab = append(pctab, encodeFirst(_PCDATA_UnsafePointUnsafe)...) + pctab = append(pctab, encodeVariant(int(size))...) + pctab = append(pctab, 0) + + /* module header */ + hdr := &_PCHeader{ + magic: _ModuleMagic, + minLC: 1, + nfunc: 1, + ptrSize: 4 << (^uintptr(0) >> 63), + textStart: minpc, + } + + /* function table */ + tab := []_FuncTab{ + {entry: 0}, + {entry: uint32(size)}, + } + + /* module data */ + mod := &_ModuleData{ + pcHeader: hdr, + funcnametab: append(append([]byte{0}, name...), 0), + cutab: []uint32{0, 0, 1}, + filetab: []byte("\x00(jit-generated)\x00"), + pctab: pctab, + pclntable: ((*[unsafe.Sizeof(_Func{})]byte)(unsafe.Pointer(&fn)))[:], + ftab: tab, + findfunctab: uintptr(unsafe.Pointer(ftab)), + minpc: minpc, + maxpc: maxpc, + text: minpc, + etext: maxpc, + modulename: name, + gcdata: uintptr(unsafe.Pointer(&emptyByte)), + gcbss: uintptr(unsafe.Pointer(&emptyByte)), + gofunc: pbase, + } + + /* verify and register the new module */ + moduledataverify1(mod) + registerModule(mod) } diff --git a/internal/loader/funcdata_invalid.go b/internal/loader/funcdata_invalid.go index 2ab96d7..1871f24 100644 --- a/internal/loader/funcdata_invalid.go +++ b/internal/loader/funcdata_invalid.go @@ -2,7 +2,7 @@ // +build !go1.16 go1.23 /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,14 +20,14 @@ package loader import ( - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) // triggers a compilation error const ( - _ = panic("Unsupported Go version. Supported versions are 1.16 ~ 1.21") + _ = panic("Unsupported Go version. Supported versions are 1.16 ~ 1.21") ) func registerFunction(_ string, _ uintptr, _ uintptr, _ rt.Frame) { - panic("Unsupported Go version. Supported versions are 1.16 ~ 1.21") + panic("Unsupported Go version. Supported versions are 1.16 ~ 1.21") } diff --git a/internal/loader/loader.go b/internal/loader/loader.go index 7c06579..fc12c8d 100644 --- a/internal/loader/loader.go +++ b/internal/loader/loader.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,69 +17,69 @@ package loader import ( - `fmt` - `os` - `sync/atomic` - `syscall` - `unsafe` + "fmt" + "os" + "sync/atomic" + "syscall" + "unsafe" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) const ( - MAP_BASE = 0x7ff00000000 + MAP_BASE = 0x7ff00000000 ) const ( - _AP = syscall.MAP_ANON | syscall.MAP_PRIVATE - _RX = syscall.PROT_READ | syscall.PROT_EXEC - _RW = syscall.PROT_READ | syscall.PROT_WRITE + _AP = syscall.MAP_ANON | syscall.MAP_PRIVATE + _RX = syscall.PROT_READ | syscall.PROT_EXEC + _RW = syscall.PROT_READ | syscall.PROT_WRITE ) type ( - Loader []byte - Function unsafe.Pointer + Loader []byte + Function unsafe.Pointer ) var ( - FnCount uint32 - LoadSize uintptr - LoadBase uintptr = MAP_BASE + FnCount uint32 + LoadSize uintptr + LoadBase uintptr = MAP_BASE ) func mkptr(m uintptr) unsafe.Pointer { - return *(*unsafe.Pointer)(unsafe.Pointer(&m)) + return *(*unsafe.Pointer)(unsafe.Pointer(&m)) } func alignUp(n uintptr, a int) uintptr { - return (n + uintptr(a) - 1) &^ (uintptr(a) - 1) + return (n + uintptr(a) - 1) &^ (uintptr(a) - 1) } func (self Loader) Load(fn string, frame rt.Frame) (f Function) { - var mm uintptr - var er syscall.Errno + var mm uintptr + var er syscall.Errno - /* align the size to pages */ - nf := uintptr(len(self)) - nb := alignUp(nf, os.Getpagesize()) - fp := atomic.AddUintptr(&LoadBase, nb) - nb + /* align the size to pages */ + nf := uintptr(len(self)) + nb := alignUp(nf, os.Getpagesize()) + fp := atomic.AddUintptr(&LoadBase, nb) - nb - /* allocate a block of memory */ - if mm, _, er = syscall.Syscall6(syscall.SYS_MMAP, fp, nb, _RW, _AP, 0, 0); er != 0 { - panic(er) - } + /* allocate a block of memory */ + if mm, _, er = syscall.Syscall6(syscall.SYS_MMAP, fp, nb, _RW, _AP, 0, 0); er != 0 { + panic(er) + } - /* copy code into the memory, and register the function */ - copy(rt.BytesFrom(mkptr(mm), len(self), int(nb)), self) - registerFunction(fmt.Sprintf("(frugal).%s_%x", fn, mm), mm, nf, frame) + /* copy code into the memory, and register the function */ + copy(rt.BytesFrom(mkptr(mm), len(self), int(nb)), self) + registerFunction(fmt.Sprintf("(frugal).%s_%x", fn, mm), mm, nf, frame) - /* make it executable */ - if _, _, err := syscall.Syscall(syscall.SYS_MPROTECT, mm, nb, _RX); err != 0 { - panic(err) - } + /* make it executable */ + if _, _, err := syscall.Syscall(syscall.SYS_MPROTECT, mm, nb, _RX); err != 0 { + panic(err) + } - /* record statistics */ - atomic.AddUint32(&FnCount, 1) - atomic.AddUintptr(&LoadSize, nb) - return Function(&mm) + /* record statistics */ + atomic.AddUint32(&FnCount, 1) + atomic.AddUintptr(&LoadSize, nb) + return Function(&mm) } diff --git a/internal/loader/loader_amd64_test.go b/internal/loader/loader_amd64_test.go index b1101ca..959ee24 100644 --- a/internal/loader/loader_amd64_test.go +++ b/internal/loader/loader_amd64_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,31 +17,31 @@ package loader import ( - `fmt` - `reflect` - `runtime` - `testing` - `unsafe` + "fmt" + "reflect" + "runtime" + "testing" + "unsafe" - `github.com/cloudwego/iasm/x86_64` - `github.com/cloudwego/frugal/internal/rt` - `github.com/stretchr/testify/assert` - `github.com/stretchr/testify/require` - `golang.org/x/arch/x86/x86asm` + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/iasm/x86_64" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/arch/x86/x86asm" ) type funcInfo struct { - *_Func - datap *_ModuleData + *_Func + datap *_ModuleData } func (self funcInfo) entry() uintptr { - if runtime.Version() <= "go1.17" { - return *(*uintptr)(unsafe.Pointer(self._Func)) - } - off := uintptr(*(*uint32)(unsafe.Pointer(self._Func))) - off += self.datap.text - return off + if runtime.Version() <= "go1.17" { + return *(*uintptr)(unsafe.Pointer(self._Func)) + } + off := uintptr(*(*uint32)(unsafe.Pointer(self._Func))) + off += self.datap.text + return off } //go:linkname findfunc runtime.findfunc @@ -53,65 +53,67 @@ func findfunc(pc uintptr) funcInfo func pcdatavalue2(f funcInfo, table uint32, targetpc uintptr) (int32, uintptr) func TestLoader_Load(t *testing.T) { - var src string - var asm x86_64.Assembler - if runtime.Version() < "go1.17" { src += ` + var src string + var asm x86_64.Assembler + if runtime.Version() < "go1.17" { + src += ` movq 8(%rsp), %rax` - } - src += ` + } + src += ` movq $1234, (%rax) ret` - require.NoError(t, asm.Assemble(src)) - v0 := 0 - cc := asm.Code() - fp := Loader(cc).Load("test", rt.Frame{}) - (*(*func(*int))(unsafe.Pointer(&fp)))(&v0) - pc := *(*uintptr)(fp) - assert.Equal(t, 1234, v0) - assert.Equal(t, fmt.Sprintf("(frugal).test_%x", pc), runtime.FuncForPC(pc).Name()) - file, line := runtime.FuncForPC(pc).FileLine(pc + 1) - assert.Equal(t, "(jit-generated)", file) - assert.Equal(t, 1, line) - smi, startpc := pcdatavalue2(findfunc(pc), _PCDATA_StackMapIndex, pc + uintptr(len(cc)) - 1) - assert.Equal(t, int32(0), smi) - assert.Equal(t, pc, startpc) - aup, startpc2 := pcdatavalue2(findfunc(pc), _PCDATA_UnsafePoint, pc + uintptr(len(cc)) - 1) - assert.Equal(t, int32(_PCDATA_UnsafePointUnsafe), aup) - assert.Equal(t, pc, startpc2) + require.NoError(t, asm.Assemble(src)) + v0 := 0 + cc := asm.Code() + fp := Loader(cc).Load("test", rt.Frame{}) + (*(*func(*int))(unsafe.Pointer(&fp)))(&v0) + pc := *(*uintptr)(fp) + assert.Equal(t, 1234, v0) + assert.Equal(t, fmt.Sprintf("(frugal).test_%x", pc), runtime.FuncForPC(pc).Name()) + file, line := runtime.FuncForPC(pc).FileLine(pc + 1) + assert.Equal(t, "(jit-generated)", file) + assert.Equal(t, 1, line) + smi, startpc := pcdatavalue2(findfunc(pc), _PCDATA_StackMapIndex, pc+uintptr(len(cc))-1) + assert.Equal(t, int32(0), smi) + assert.Equal(t, pc, startpc) + aup, startpc2 := pcdatavalue2(findfunc(pc), _PCDATA_UnsafePoint, pc+uintptr(len(cc))-1) + assert.Equal(t, int32(_PCDATA_UnsafePointUnsafe), aup) + assert.Equal(t, pc, startpc2) } func mkpointer() *int { - ret := new(int) - *ret = 1234 - runtime.SetFinalizer(ret, func(_ *int) { - println("ret has been recycled") - }) - println("ret is allocated") - return ret + ret := new(int) + *ret = 1234 + runtime.SetFinalizer(ret, func(_ *int) { + println("ret has been recycled") + }) + println("ret is allocated") + return ret } func collect() { - println("start collecting") - for i := 1; i < 1000; i++ { - runtime.GC() - } - println("done collecting") + println("start collecting") + for i := 1; i < 1000; i++ { + runtime.GC() + } + println("done collecting") } func TestLoader_StackMap(t *testing.T) { - var asm x86_64.Assembler - var smb rt.StackMapBuilder - src := ` + var asm x86_64.Assembler + var smb rt.StackMapBuilder + src := ` subq $24, %rsp movq %rbp, 16(%rsp) leaq 16(%rsp), %rbp movq $` + fmt.Sprintf("%p", mkpointer) + `, %r12 callq %r12` - if runtime.Version() < "go1.17" { src += ` + if runtime.Version() < "go1.17" { + src += ` movq (%rsp), %rax` - } - src += ` + } + src += ` movq %rax, 8(%rsp) movq $0x123, (%rsp) movq $` + fmt.Sprintf("%p", collect) + `, %r12 @@ -119,28 +121,28 @@ func TestLoader_StackMap(t *testing.T) { movq 16(%rsp), %rbp addq $24, %rsp ret` - require.NoError(t, asm.Assemble(src)) - smb.AddField(true) - cc := asm.Code() - smb1 := new(rt.StackMapBuilder) - smb1.AddField(false) - fp := Loader(cc).Load("test_with_stackmap", rt.Frame { - SpTab: []rt.Stack { - { Sp: 0, Nb: 4 }, - { Sp: 24, Nb: uintptr(len(cc) - 5) }, - { Sp: 0, Nb: 0 }, - }, - ArgSize : 0, - ArgPtrs : smb1.Build(), // Build() should be called on non-empty Builder, - // otherwise mallocgc will allocate less than enough - // which leads to fatal error when run with -race - LocalPtrs : smb.Build(), - }) - dumpfunction(*(*func())(unsafe.Pointer(&fp))) - println("enter function") - (*(*func())(unsafe.Pointer(&fp)))() - println("leave function") - collect() + require.NoError(t, asm.Assemble(src)) + smb.AddField(true) + cc := asm.Code() + smb1 := new(rt.StackMapBuilder) + smb1.AddField(false) + fp := Loader(cc).Load("test_with_stackmap", rt.Frame{ + SpTab: []rt.Stack{ + {Sp: 0, Nb: 4}, + {Sp: 24, Nb: uintptr(len(cc) - 5)}, + {Sp: 0, Nb: 0}, + }, + ArgSize: 0, + ArgPtrs: smb1.Build(), // Build() should be called on non-empty Builder, + // otherwise mallocgc will allocate less than enough + // which leads to fatal error when run with -race + LocalPtrs: smb.Build(), + }) + dumpfunction(*(*func())(unsafe.Pointer(&fp))) + println("enter function") + (*(*func())(unsafe.Pointer(&fp)))() + println("leave function") + collect() } //go:linkname step runtime.step @@ -148,55 +150,55 @@ func TestLoader_StackMap(t *testing.T) { func step(p []byte, pc *uintptr, val *int32, first bool) (newp []byte, ok bool) func dumpfunction(f interface{}) { - fp := rt.FuncAddr(f) - fn := findfunc(uintptr(fp)) - var name string - if runtime.Version() >= "go1.16" { - name = "pctab" - } else { - name = "pclntable" - } - datap := reflect.ValueOf(fn.datap) - ff, ok := datap.Type().Elem().FieldByName(name) - if !ok { - panic("no such field: pctab") - } - p := (*(*[]byte)(unsafe.Pointer(uintptr(unsafe.Pointer(fn.datap)) + ff.Offset)))[fn.pcsp:] - pc := fn.entry() - val := int32(-1) - lastpc := uintptr(0) - for { - var ok bool - lastpc = pc - p, ok = step(p, &pc, &val, pc == fn.entry()) - if !ok { - break - } - fmt.Printf("%#x = %#x\n", lastpc, val) - } - pc = 0 - lastpc -= fn.entry() - for pc <= lastpc { - pp := unsafe.Pointer(uintptr(fp) + pc) - fx := runtime.FuncForPC(uintptr(pp)) - if fx.Name() != "" && fx.Entry() == uintptr(pp) { - println("----", fx.Name(), "----") - } - ins, err := x86asm.Decode(rt.BytesFrom(pp, 15, 15), 64) - if err != nil { - panic(err) - } - fmt.Printf("%#x %s\n", uintptr(pp), x86asm.GNUSyntax(ins, uint64(uintptr(pp)), func(u uint64) (string, uint64) { - v := runtime.FuncForPC(uintptr(u)) - if v == nil { - return "", 0 - } - return v.Name(), uint64(v.Entry()) - })) - pc += uintptr(ins.Len) - } + fp := rt.FuncAddr(f) + fn := findfunc(uintptr(fp)) + var name string + if runtime.Version() >= "go1.16" { + name = "pctab" + } else { + name = "pclntable" + } + datap := reflect.ValueOf(fn.datap) + ff, ok := datap.Type().Elem().FieldByName(name) + if !ok { + panic("no such field: pctab") + } + p := (*(*[]byte)(unsafe.Pointer(uintptr(unsafe.Pointer(fn.datap)) + ff.Offset)))[fn.pcsp:] + pc := fn.entry() + val := int32(-1) + lastpc := uintptr(0) + for { + var ok bool + lastpc = pc + p, ok = step(p, &pc, &val, pc == fn.entry()) + if !ok { + break + } + fmt.Printf("%#x = %#x\n", lastpc, val) + } + pc = 0 + lastpc -= fn.entry() + for pc <= lastpc { + pp := unsafe.Pointer(uintptr(fp) + pc) + fx := runtime.FuncForPC(uintptr(pp)) + if fx.Name() != "" && fx.Entry() == uintptr(pp) { + println("----", fx.Name(), "----") + } + ins, err := x86asm.Decode(rt.BytesFrom(pp, 15, 15), 64) + if err != nil { + panic(err) + } + fmt.Printf("%#x %s\n", uintptr(pp), x86asm.GNUSyntax(ins, uint64(uintptr(pp)), func(u uint64) (string, uint64) { + v := runtime.FuncForPC(uintptr(u)) + if v == nil { + return "", 0 + } + return v.Name(), uint64(v.Entry()) + })) + pc += uintptr(ins.Len) + } } func TestLoader_PCSPDelta(t *testing.T) { - dumpfunction(moduledataverify1) + dumpfunction(moduledataverify1) } diff --git a/internal/opts/limits.go b/internal/opts/limits.go index 3cb78e7..c7d5b03 100644 --- a/internal/opts/limits.go +++ b/internal/opts/limits.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,28 +17,28 @@ package opts import ( - `os` - `strconv` + "os" + "strconv" ) const ( - _DefaultMaxInlineDepth = 2 // cutoff at 2 levels of inlining - _DefaultMaxInlineILSize = 50000 // cutoff at 50k of IL instructions + _DefaultMaxInlineDepth = 2 // cutoff at 2 levels of inlining + _DefaultMaxInlineILSize = 50000 // cutoff at 50k of IL instructions ) var ( - MaxInlineDepth = parseOrDefault("FRUGAL_MAX_INLINE_DEPTH", _DefaultMaxInlineDepth, 1) - MaxInlineILSize = parseOrDefault("FRUGAL_MAX_INLINE_IL_SIZE", _DefaultMaxInlineILSize, 256) + MaxInlineDepth = parseOrDefault("FRUGAL_MAX_INLINE_DEPTH", _DefaultMaxInlineDepth, 1) + MaxInlineILSize = parseOrDefault("FRUGAL_MAX_INLINE_IL_SIZE", _DefaultMaxInlineILSize, 256) ) func parseOrDefault(key string, def int, min int) int { - if env := os.Getenv(key); env == "" { - return def - } else if val, err := strconv.ParseUint(env, 0, 64); err != nil { - panic("frugal: invalid value for " + key) - } else if ret := int(val); ret <= min { - panic("frugal: value too small for " + key) - } else { - return ret - } + if env := os.Getenv(key); env == "" { + return def + } else if val, err := strconv.ParseUint(env, 0, 64); err != nil { + panic("frugal: invalid value for " + key) + } else if ret := int(val); ret <= min { + panic("frugal: value too small for " + key) + } else { + return ret + } } diff --git a/internal/opts/options.go b/internal/opts/options.go index 8731ce5..e17612f 100644 --- a/internal/opts/options.go +++ b/internal/opts/options.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,23 +17,23 @@ package opts type Options struct { - MaxInlineDepth int - MaxInlineILSize int - MaxPretouchDepth int + MaxInlineDepth int + MaxInlineILSize int + MaxPretouchDepth int } func (self *Options) CanInline(sp int, pc int) bool { - return (self.MaxInlineDepth > sp || self.MaxInlineDepth == 0) && (self.MaxInlineILSize > pc || self.MaxInlineILSize == 0) + return (self.MaxInlineDepth > sp || self.MaxInlineDepth == 0) && (self.MaxInlineILSize > pc || self.MaxInlineILSize == 0) } func (self *Options) CanPretouch(d int) bool { - return self.MaxPretouchDepth > d || self.MaxPretouchDepth == 0 + return self.MaxPretouchDepth > d || self.MaxPretouchDepth == 0 } func GetDefaultOptions() Options { - return Options { - MaxInlineDepth : MaxInlineDepth, - MaxInlineILSize : MaxInlineILSize, - MaxPretouchDepth : 0, - } + return Options{ + MaxInlineDepth: MaxInlineDepth, + MaxInlineILSize: MaxInlineILSize, + MaxPretouchDepth: 0, + } } diff --git a/internal/rt/asm.s b/internal/rt/asm.s index a8c6cc1..9c99547 100644 --- a/internal/rt/asm.s +++ b/internal/rt/asm.s @@ -1,4 +1,4 @@ -// Copyright 2022 ByteDance Inc. +// Copyright 2022 CloudWeGo Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/internal/rt/escape.go b/internal/rt/escape.go index 7a5a159..f7134e6 100644 --- a/internal/rt/escape.go +++ b/internal/rt/escape.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ package rt import ( - `unsafe` + "unsafe" ) // NoEscape hides a pointer from escape analysis. NoEscape is @@ -25,9 +25,10 @@ import ( // output depends on the input. NoEscape is inlined and currently // compiles down to zero instructions. // USE CAREFULLY! +// //go:nosplit //goland:noinspection GoVetUnsafePointer func NoEscape(p unsafe.Pointer) unsafe.Pointer { - x := uintptr(p) - return unsafe.Pointer(x ^ 0) + x := uintptr(p) + return unsafe.Pointer(x ^ 0) } diff --git a/internal/rt/exports.go b/internal/rt/exports.go index 120cd19..3157e41 100644 --- a/internal/rt/exports.go +++ b/internal/rt/exports.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ package rt import ( - `unsafe` + "unsafe" ) //go:noescape @@ -47,6 +47,6 @@ func resolveTextOff(p unsafe.Pointer, off GoTextOffset) unsafe.Pointer //go:nosplit func MapClear(m interface{}) { - v := UnpackEface(m) - mapclear(v.Type, v.Value) + v := UnpackEface(m) + mapclear(v.Type, v.Value) } diff --git a/internal/rt/frame.go b/internal/rt/frame.go index b1e1463..5f92ea7 100644 --- a/internal/rt/frame.go +++ b/internal/rt/frame.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,13 +17,13 @@ package rt type Stack struct { - Sp uintptr - Nb uintptr + Sp uintptr + Nb uintptr } type Frame struct { - SpTab []Stack - ArgSize uintptr - ArgPtrs *StackMap - LocalPtrs *StackMap + SpTab []Stack + ArgSize uintptr + ArgPtrs *StackMap + LocalPtrs *StackMap } diff --git a/internal/rt/funcname.go b/internal/rt/funcname.go index 253e9ca..c5db24a 100644 --- a/internal/rt/funcname.go +++ b/internal/rt/funcname.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,17 +17,17 @@ package rt import ( - `fmt` - `runtime` - `unsafe` + "fmt" + "runtime" + "unsafe" ) func FuncName(p unsafe.Pointer) string { - if fn := runtime.FuncForPC(uintptr(p)); fn == nil { - return "???" - } else if fp := fn.Entry(); fp == uintptr(p) { - return fn.Name() - } else { - return fmt.Sprintf("%s+%#x", fn.Name(), uintptr(p) - fp) - } + if fn := runtime.FuncForPC(uintptr(p)); fn == nil { + return "???" + } else if fp := fn.Entry(); fp == uintptr(p) { + return fn.Name() + } else { + return fmt.Sprintf("%s+%#x", fn.Name(), uintptr(p)-fp) + } } diff --git a/internal/rt/iface.go b/internal/rt/iface.go index 62b2e0c..80e752d 100644 --- a/internal/rt/iface.go +++ b/internal/rt/iface.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,31 +17,31 @@ package rt import ( - `reflect` + "reflect" ) type Method struct { - Id int - Vt *GoType + Id int + Vt *GoType } func AsMethod(id int, vt *GoType) Method { - return Method { - Id: id, - Vt: vt, - } + return Method{ + Id: id, + Vt: vt, + } } func GetMethod(tp interface{}, name string) Method { - if tp == nil { - panic("value must be an interface pointer") - } else if vt := reflect.TypeOf(tp); vt.Kind() != reflect.Ptr { - panic("value must be an interface pointer") - } else if et := vt.Elem(); et.Kind() != reflect.Interface { - panic("value must be an interface pointer") - } else if mm, ok := et.MethodByName(name); !ok { - panic("interface " + vt.Elem().String() + " does not have method " + name) - } else { - return AsMethod(mm.Index, UnpackType(et)) - } + if tp == nil { + panic("value must be an interface pointer") + } else if vt := reflect.TypeOf(tp); vt.Kind() != reflect.Ptr { + panic("value must be an interface pointer") + } else if et := vt.Elem(); et.Kind() != reflect.Interface { + panic("value must be an interface pointer") + } else if mm, ok := et.MethodByName(name); !ok { + panic("interface " + vt.Elem().String() + " does not have method " + name) + } else { + return AsMethod(mm.Index, UnpackType(et)) + } } diff --git a/internal/rt/iface_test.go b/internal/rt/iface_test.go index e5f2f99..7273160 100644 --- a/internal/rt/iface_test.go +++ b/internal/rt/iface_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,33 +17,33 @@ package rt import ( - `testing` + "testing" ) type FooIface interface { - FooMethod() + FooMethod() } type FooValue struct { - X int + X int } func (self FooValue) FooMethod() { - println(self.X) + println(self.X) } type FooPointer struct { - X int + X int } func (self *FooPointer) FooMethod() { - println(self.X) + println(self.X) } func TestIface_Invoke(t *testing.T) { - var v FooIface - v = FooValue{X: 100} - v.FooMethod() - v = &FooPointer{X: 200} - v.FooMethod() + var v FooIface + v = FooValue{X: 100} + v.FooMethod() + v = &FooPointer{X: 200} + v.FooMethod() } diff --git a/internal/rt/runtime.go b/internal/rt/runtime.go index c137b8b..de4eb32 100644 --- a/internal/rt/runtime.go +++ b/internal/rt/runtime.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,345 +17,356 @@ package rt import ( - `reflect` - `unsafe` + "reflect" + "unsafe" ) const ( - MaxFastMap = 128 + MaxFastMap = 128 ) const ( - T_uncommon = 1 << 0 + T_uncommon = 1 << 0 ) const ( - F_direct = 1 << 5 - F_kind_mask = (1 << 5) - 1 + F_direct = 1 << 5 + F_kind_mask = (1 << 5) - 1 ) var ( - reflectRtypeItab = findReflectRtypeItab() + reflectRtypeItab = findReflectRtypeItab() ) type ( - GoNameOffset int32 - GoTypeOffset int32 - GoTextOffset int32 + GoNameOffset int32 + GoTypeOffset int32 + GoTextOffset int32 ) func (self GoNameOffset) Resolve(vt *GoType) GoName { - if self == -1 { - return GoName{} - } else { - return resolveNameOff(unsafe.Pointer(vt), self) - } + if self == -1 { + return GoName{} + } else { + return resolveNameOff(unsafe.Pointer(vt), self) + } } func (self GoTypeOffset) Resolve(vt *GoType) *GoType { - if self == -1 { - return nil - } else { - return resolveTypeOff(unsafe.Pointer(vt), self) - } + if self == -1 { + return nil + } else { + return resolveTypeOff(unsafe.Pointer(vt), self) + } } func (self GoTextOffset) Resolve(vt *GoType) unsafe.Pointer { - if self == -1 { - return nil - } else { - return resolveTextOff(unsafe.Pointer(vt), self) - } + if self == -1 { + return nil + } else { + return resolveTextOff(unsafe.Pointer(vt), self) + } } type GoName struct { - b *[1 << 16]byte + b *[1 << 16]byte } func (self GoName) read(p int) (i int, v int) { - for i, v = 0, 0;; i++ { - if v += int(self.b[p + i] & 0x7f) << (7 * i); self.b[p + i] & 0x80 == 0 { + for i, v = 0, 0; ; i++ { + if v += int(self.b[p+i]&0x7f) << (7 * i); self.b[p+i]&0x80 == 0 { return i + 1, v } - } + } } func (self GoName) mkstr(i int, n int) (s string) { - (*GoString)(unsafe.Pointer(&s)).Len = n - (*GoString)(unsafe.Pointer(&s)).Ptr = unsafe.Pointer(&self.b[i]) - return + (*GoString)(unsafe.Pointer(&s)).Len = n + (*GoString)(unsafe.Pointer(&s)).Ptr = unsafe.Pointer(&self.b[i]) + return } func (self GoName) Name() string { - if self.b == nil { - return "" - } else if i, n := self.read(1); n == 0 { - return "" - } else { - return self.mkstr(i + 1, n) - } + if self.b == nil { + return "" + } else if i, n := self.read(1); n == 0 { + return "" + } else { + return self.mkstr(i+1, n) + } } func (self GoName) IsExported() bool { - return self.b != nil && self.b[0] & (1 << 0) != 0 + return self.b != nil && self.b[0]&(1<<0) != 0 } type GoType struct { - Size uintptr - PtrData uintptr - Hash uint32 - Flags uint8 - Align uint8 - FieldAlign uint8 - KindFlags uint8 - Equal func(unsafe.Pointer, unsafe.Pointer) bool - GCData *byte - Str GoNameOffset - PtrToSelf GoTypeOffset + Size uintptr + PtrData uintptr + Hash uint32 + Flags uint8 + Align uint8 + FieldAlign uint8 + KindFlags uint8 + Equal func(unsafe.Pointer, unsafe.Pointer) bool + GCData *byte + Str GoNameOffset + PtrToSelf GoTypeOffset } func (self *GoType) mtab() []GoMethod { - switch self.Kind() { - case reflect.Ptr : return (*struct { GoPtrType ; GoUncommonType })(unsafe.Pointer(self)).Methods() - case reflect.Struct : return (*struct { GoStructType ; GoUncommonType })(unsafe.Pointer(self)).Methods() - default : return nil - } + switch self.Kind() { + case reflect.Ptr: + return (*struct { + GoPtrType + GoUncommonType + })(unsafe.Pointer(self)).Methods() + case reflect.Struct: + return (*struct { + GoStructType + GoUncommonType + })(unsafe.Pointer(self)).Methods() + default: + return nil + } } func (self *GoType) Kind() reflect.Kind { - return reflect.Kind(self.KindFlags & F_kind_mask) + return reflect.Kind(self.KindFlags & F_kind_mask) } func (self *GoType) Pack() (t reflect.Type) { - (*GoIface)(unsafe.Pointer(&t)).Itab = reflectRtypeItab - (*GoIface)(unsafe.Pointer(&t)).Value = unsafe.Pointer(self) - return + (*GoIface)(unsafe.Pointer(&t)).Itab = reflectRtypeItab + (*GoIface)(unsafe.Pointer(&t)).Value = unsafe.Pointer(self) + return } func (self *GoType) String() string { - return self.Pack().String() + return self.Pack().String() } func (self *GoType) Methods() []GoMethod { - if !self.IsUncommon() { - return nil - } else { - return self.mtab() - } + if !self.IsUncommon() { + return nil + } else { + return self.mtab() + } } func (self *GoType) IsUncommon() bool { - return (self.Flags & T_uncommon) != 0 + return (self.Flags & T_uncommon) != 0 } func (self *GoType) IsIndirect() bool { - return (self.KindFlags & F_direct) == 0 + return (self.KindFlags & F_direct) == 0 } type GoPtrType struct { - GoType - Elem *GoType + GoType + Elem *GoType } type GoMapType struct { - GoType - Key *GoType - Elem *GoType - Bucket *GoType - Hasher func(unsafe.Pointer, uintptr) uintptr - KeySize uint8 - ElemSize uint8 - BucketSize uint16 - Flags uint32 + GoType + Key *GoType + Elem *GoType + Bucket *GoType + Hasher func(unsafe.Pointer, uintptr) uintptr + KeySize uint8 + ElemSize uint8 + BucketSize uint16 + Flags uint32 } func (self *GoMapType) IsFastMap() bool { - return self.Elem.Size <= MaxFastMap + return self.Elem.Size <= MaxFastMap } type GoSliceType struct { - GoType - Elem *GoType + GoType + Elem *GoType } type GoStructType struct { - GoType - PkgPath GoName - Fields []GoStructField + GoType + PkgPath GoName + Fields []GoStructField } type GoMethod struct { - Name GoNameOffset - Type GoTypeOffset - IFunc GoTextOffset - TFunc GoTextOffset + Name GoNameOffset + Type GoTypeOffset + IFunc GoTextOffset + TFunc GoTextOffset } type GoStructField struct { - Name GoName - Type *GoType - OffsetEmbed uintptr + Name GoName + Type *GoType + OffsetEmbed uintptr } type GoUncommonType struct { - PkgPath int32 - NumMethod uint16 - NumExported uint16 - MethodOffset uint32 - _ uint32 + PkgPath int32 + NumMethod uint16 + NumExported uint16 + MethodOffset uint32 + _ uint32 } func (self *GoUncommonType) mbuf() unsafe.Pointer { - return unsafe.Pointer(uintptr(unsafe.Pointer(self)) + uintptr(self.MethodOffset)) + return unsafe.Pointer(uintptr(unsafe.Pointer(self)) + uintptr(self.MethodOffset)) } func (self *GoUncommonType) Methods() []GoMethod { - if self.NumMethod == 0 { - return nil - } else { - return (*[1 << 16]GoMethod)(self.mbuf())[:self.NumMethod:self.NumMethod] - } + if self.NumMethod == 0 { + return nil + } else { + return (*[1 << 16]GoMethod)(self.mbuf())[:self.NumMethod:self.NumMethod] + } } type GoItab struct { - it unsafe.Pointer - vt *GoType - hv uint32 - _ [4]byte - fn [1]uintptr + it unsafe.Pointer + vt *GoType + hv uint32 + _ [4]byte + fn [1]uintptr } const ( - GoItabFuncBase = unsafe.Offsetof(GoItab{}.fn) + GoItabFuncBase = unsafe.Offsetof(GoItab{}.fn) ) type GoIface struct { - Itab *GoItab - Value unsafe.Pointer + Itab *GoItab + Value unsafe.Pointer } type GoEface struct { - Type *GoType - Value unsafe.Pointer + Type *GoType + Value unsafe.Pointer } type GoSlice struct { - Ptr unsafe.Pointer - Len int - Cap int + Ptr unsafe.Pointer + Len int + Cap int } func (self GoSlice) Set(i int, v byte) { - *(*byte)(unsafe.Pointer(uintptr(self.Ptr) + uintptr(i))) = v + *(*byte)(unsafe.Pointer(uintptr(self.Ptr) + uintptr(i))) = v } type GoString struct { - Ptr unsafe.Pointer - Len int + Ptr unsafe.Pointer + Len int } type GoMap struct { - Count int - Flags uint8 - B uint8 - Overflow uint16 - Hash0 uint32 - Buckets unsafe.Pointer - OldBuckets unsafe.Pointer - Evacuate uintptr - Extra unsafe.Pointer + Count int + Flags uint8 + B uint8 + Overflow uint16 + Hash0 uint32 + Buckets unsafe.Pointer + OldBuckets unsafe.Pointer + Evacuate uintptr + Extra unsafe.Pointer } type GoMapIterator struct { - K unsafe.Pointer - V unsafe.Pointer - T *GoMapType - H *GoMap - Buckets unsafe.Pointer - Bptr *unsafe.Pointer - Overflow *[]unsafe.Pointer - OldOverflow *[]unsafe.Pointer - StartBucket uintptr - Offset uint8 - Wrapped bool - B uint8 - I uint8 - Bucket uintptr - CheckBucket uintptr + K unsafe.Pointer + V unsafe.Pointer + T *GoMapType + H *GoMap + Buckets unsafe.Pointer + Bptr *unsafe.Pointer + Overflow *[]unsafe.Pointer + OldOverflow *[]unsafe.Pointer + StartBucket uintptr + Offset uint8 + Wrapped bool + B uint8 + I uint8 + Bucket uintptr + CheckBucket uintptr } func (self *GoMapIterator) Next() bool { - mapiternext(self) - return self.K != nil + mapiternext(self) + return self.K != nil } func IsPtr(t *GoType) bool { - return t.Kind() == reflect.Ptr || t.Kind() == reflect.UnsafePointer + return t.Kind() == reflect.Ptr || t.Kind() == reflect.UnsafePointer } func PtrElem(t *GoType) *GoType { - if t.Kind() != reflect.Ptr { - panic("t is not a ptr") - } else { - return (*GoPtrType)(unsafe.Pointer(t)).Elem - } + if t.Kind() != reflect.Ptr { + panic("t is not a ptr") + } else { + return (*GoPtrType)(unsafe.Pointer(t)).Elem + } } func MapType(t *GoType) *GoMapType { - if t.Kind() != reflect.Map { - panic("t is not a map") - } else { - return (*GoMapType)(unsafe.Pointer(t)) - } + if t.Kind() != reflect.Map { + panic("t is not a map") + } else { + return (*GoMapType)(unsafe.Pointer(t)) + } } func Dereference(t *GoType) *GoType { - for t.Kind() == reflect.Ptr { t = PtrElem(t) } - return t + for t.Kind() == reflect.Ptr { + t = PtrElem(t) + } + return t } func FuncAddr(f interface{}) unsafe.Pointer { - if vv := UnpackEface(f); vv.Type.Kind() != reflect.Func { - panic("f is not a function") - } else { - return *(*unsafe.Pointer)(vv.Value) - } + if vv := UnpackEface(f); vv.Type.Kind() != reflect.Func { + panic("f is not a function") + } else { + return *(*unsafe.Pointer)(vv.Value) + } } func SliceElem(t *GoType) *GoType { - if t.Kind() != reflect.Slice { - panic("t is not a slice") - } else { - return (*GoSliceType)(unsafe.Pointer(t)).Elem - } + if t.Kind() != reflect.Slice { + panic("t is not a slice") + } else { + return (*GoSliceType)(unsafe.Pointer(t)).Elem + } } func BytesFrom(p unsafe.Pointer, n int, c int) (r []byte) { - (*GoSlice)(unsafe.Pointer(&r)).Ptr = p - (*GoSlice)(unsafe.Pointer(&r)).Len = n - (*GoSlice)(unsafe.Pointer(&r)).Cap = c - return + (*GoSlice)(unsafe.Pointer(&r)).Ptr = p + (*GoSlice)(unsafe.Pointer(&r)).Len = n + (*GoSlice)(unsafe.Pointer(&r)).Cap = c + return } func StringPtr(s string) unsafe.Pointer { - return (*GoString)(unsafe.Pointer(&s)).Ptr + return (*GoString)(unsafe.Pointer(&s)).Ptr } func StringFrom(p unsafe.Pointer, n int) (r string) { - (*GoString)(unsafe.Pointer(&r)).Ptr = p - (*GoString)(unsafe.Pointer(&r)).Len = n - return + (*GoString)(unsafe.Pointer(&r)).Ptr = p + (*GoString)(unsafe.Pointer(&r)).Len = n + return } func UnpackType(t reflect.Type) *GoType { - return (*GoType)((*GoIface)(unsafe.Pointer(&t)).Value) + return (*GoType)((*GoIface)(unsafe.Pointer(&t)).Value) } func UnpackEface(v interface{}) GoEface { - return *(*GoEface)(unsafe.Pointer(&v)) + return *(*GoEface)(unsafe.Pointer(&v)) } func findReflectRtypeItab() *GoItab { - v := reflect.TypeOf(struct{}{}) - return (*GoIface)(unsafe.Pointer(&v)).Itab + v := reflect.TypeOf(struct{}{}) + return (*GoIface)(unsafe.Pointer(&v)).Itab } diff --git a/internal/rt/stackmap.go b/internal/rt/stackmap.go index c298fdb..b9a9fff 100644 --- a/internal/rt/stackmap.go +++ b/internal/rt/stackmap.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,126 +17,126 @@ package rt import ( - `fmt` - `strings` - `sync` - `unsafe` + "fmt" + "strings" + "sync" + "unsafe" ) type Bitmap struct { - N int - B []byte + N int + B []byte } func (self *Bitmap) grow() { - if self.N >= len(self.B) * 8 { - self.B = append(self.B, 0) - } + if self.N >= len(self.B)*8 { + self.B = append(self.B, 0) + } } func (self *Bitmap) mark(i int, bv int) { - if bv != 0 { - self.B[i / 8] |= 1 << (i % 8) - } else { - self.B[i / 8] &^= 1 << (i % 8) - } + if bv != 0 { + self.B[i/8] |= 1 << (i % 8) + } else { + self.B[i/8] &^= 1 << (i % 8) + } } func (self *Bitmap) Set(i int, bv int) { - if i >= self.N { - panic("bitmap: invalid bit position") - } else { - self.mark(i, bv) - } + if i >= self.N { + panic("bitmap: invalid bit position") + } else { + self.mark(i, bv) + } } func (self *Bitmap) Append(bv int) { - self.grow() - self.mark(self.N, bv) - self.N++ + self.grow() + self.mark(self.N, bv) + self.N++ } func (self *Bitmap) AppendMany(n int, bv int) { - for i := 0; i < n; i++ { - self.Append(bv) - } + for i := 0; i < n; i++ { + self.Append(bv) + } } var ( - _stackMapLock = sync.Mutex{} - _stackMapCache = make(map[*StackMap]struct{}) + _stackMapLock = sync.Mutex{} + _stackMapCache = make(map[*StackMap]struct{}) ) type BitVec struct { - N uintptr - B unsafe.Pointer + N uintptr + B unsafe.Pointer } func (self BitVec) Bit(i uintptr) byte { - return (*(*byte)(unsafe.Pointer(uintptr(self.B) + i / 8)) >> (i % 8)) & 1 + return (*(*byte)(unsafe.Pointer(uintptr(self.B) + i/8)) >> (i % 8)) & 1 } func (self BitVec) String() string { - var i uintptr - var v []string + var i uintptr + var v []string - /* add each bit */ - for i = 0; i < self.N; i++ { - v = append(v, fmt.Sprintf("%d", self.Bit(i))) - } + /* add each bit */ + for i = 0; i < self.N; i++ { + v = append(v, fmt.Sprintf("%d", self.Bit(i))) + } - /* join them together */ - return fmt.Sprintf( - "BitVec { %s }", - strings.Join(v, ", "), - ) + /* join them together */ + return fmt.Sprintf( + "BitVec { %s }", + strings.Join(v, ", "), + ) } type StackMap struct { - N int32 - L int32 - B [1]byte + N int32 + L int32 + B [1]byte } func (self *StackMap) add() { - _stackMapLock.Lock() - _stackMapCache[self] = struct{}{} - _stackMapLock.Unlock() + _stackMapLock.Lock() + _stackMapCache[self] = struct{}{} + _stackMapLock.Unlock() } func (self *StackMap) Pin() uintptr { - self.add() - return uintptr(unsafe.Pointer(self)) + self.add() + return uintptr(unsafe.Pointer(self)) } func (self *StackMap) Get(i int32) BitVec { - return BitVec { - N: uintptr(self.L), - B: unsafe.Pointer(uintptr(unsafe.Pointer(&self.B)) + uintptr(i * ((self.L + 7) >> 3))), - } + return BitVec{ + N: uintptr(self.L), + B: unsafe.Pointer(uintptr(unsafe.Pointer(&self.B)) + uintptr(i*((self.L+7)>>3))), + } } func (self *StackMap) String() string { - sb := strings.Builder{} - sb.WriteString("StackMap {") + sb := strings.Builder{} + sb.WriteString("StackMap {") - /* dump every stack map */ - for i := int32(0); i < self.N; i++ { - sb.WriteRune('\n') - sb.WriteString(" " + self.Get(i).String()) - } + /* dump every stack map */ + for i := int32(0); i < self.N; i++ { + sb.WriteRune('\n') + sb.WriteString(" " + self.Get(i).String()) + } - /* close the stackmap */ - sb.WriteString("\n}") - return sb.String() + /* close the stackmap */ + sb.WriteString("\n}") + return sb.String() } var ( - byteType = UnpackEface(byte(0)).Type + byteType = UnpackEface(byte(0)).Type ) const ( - _StackMapSize = unsafe.Sizeof(StackMap{}) + _StackMapSize = unsafe.Sizeof(StackMap{}) ) //go:linkname mallocgc runtime.mallocgc @@ -144,32 +144,32 @@ const ( func mallocgc(nb uintptr, vt *GoType, zero bool) unsafe.Pointer type StackMapBuilder struct { - b Bitmap + b Bitmap } func (self *StackMapBuilder) Build() (p *StackMap) { - nb := len(self.b.B) - bm := mallocgc(_StackMapSize + uintptr(nb) - 1, byteType, false) + nb := len(self.b.B) + bm := mallocgc(_StackMapSize+uintptr(nb)-1, byteType, false) - /* initialize as 1 bitmap of N bits */ - p = (*StackMap)(bm) - p.N, p.L = 1, int32(self.b.N) - copy(BytesFrom(unsafe.Pointer(&p.B), nb, nb), self.b.B) - return + /* initialize as 1 bitmap of N bits */ + p = (*StackMap)(bm) + p.N, p.L = 1, int32(self.b.N) + copy(BytesFrom(unsafe.Pointer(&p.B), nb, nb), self.b.B) + return } func (self *StackMapBuilder) AddField(ptr bool) { - if ptr { - self.b.Append(1) - } else { - self.b.Append(0) - } + if ptr { + self.b.Append(1) + } else { + self.b.Append(0) + } } func (self *StackMapBuilder) AddFields(n int, ptr bool) { - if ptr { - self.b.AppendMany(n, 1) - } else { - self.b.AppendMany(n, 0) - } + if ptr { + self.b.AppendMany(n, 1) + } else { + self.b.AppendMany(n, 0) + } } diff --git a/internal/rt/stackmap_test.go b/internal/rt/stackmap_test.go index 0b319ef..f7bf4a5 100644 --- a/internal/rt/stackmap_test.go +++ b/internal/rt/stackmap_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,35 +17,35 @@ package rt import ( - `fmt` - `reflect` - `testing` - `unsafe` + "fmt" + "reflect" + "testing" + "unsafe" ) const ( - _FUNCDATA_ArgsPointerMaps = 0 - _FUNCDATA_LocalsPointerMaps = 1 + _FUNCDATA_ArgsPointerMaps = 0 + _FUNCDATA_LocalsPointerMaps = 1 ) type funcInfo struct { - fn unsafe.Pointer - datap unsafe.Pointer + fn unsafe.Pointer + datap unsafe.Pointer } type bitvector struct { - n int32 // # of bits - bytedata *uint8 + n int32 // # of bits + bytedata *uint8 } //go:nosplit func addb(p *byte, n uintptr) *byte { - return (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + n)) + return (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + n)) } func (bv *bitvector) ptrbit(i uintptr) uint8 { - b := *(addb(bv.bytedata, i/8)) - return (b >> (i % 8)) & 1 + b := *(addb(bv.bytedata, i/8)) + return (b >> (i % 8)) & 1 } //go:linkname findfunc runtime.findfunc @@ -58,51 +58,51 @@ func funcdata(_ funcInfo, _ uint8) unsafe.Pointer func stackmapdata(_ *StackMap, _ int32) bitvector func stackMap(f interface{}) (*StackMap, *StackMap) { - fv := reflect.ValueOf(f) - if fv.Kind() != reflect.Func { - panic("f must be reflect.Func kind!") - } - fi := findfunc(fv.Pointer()) - args := funcdata(fi, uint8(_FUNCDATA_ArgsPointerMaps)) - locals := funcdata(fi, uint8(_FUNCDATA_LocalsPointerMaps)) - return (*StackMap)(args), (*StackMap)(locals) + fv := reflect.ValueOf(f) + if fv.Kind() != reflect.Func { + panic("f must be reflect.Func kind!") + } + fi := findfunc(fv.Pointer()) + args := funcdata(fi, uint8(_FUNCDATA_ArgsPointerMaps)) + locals := funcdata(fi, uint8(_FUNCDATA_LocalsPointerMaps)) + return (*StackMap)(args), (*StackMap)(locals) } func dumpstackmap(m *StackMap) { - for i := int32(0); i < m.N; i++ { - fmt.Printf("bitmap #%d/%d: ", i, m.L) - bv := stackmapdata(m, i) - for j := int32(0); j < bv.n; j++ { - fmt.Printf("%d ", bv.ptrbit(uintptr(j))) - } - fmt.Printf("\n") - } + for i := int32(0); i < m.N; i++ { + fmt.Printf("bitmap #%d/%d: ", i, m.L) + bv := stackmapdata(m, i) + for j := int32(0); j < bv.n; j++ { + fmt.Printf("%d ", bv.ptrbit(uintptr(j))) + } + fmt.Printf("\n") + } } var keepalive struct { - s string - i int - vp unsafe.Pointer - sb interface{} - fv uint64 + s string + i int + vp unsafe.Pointer + sb interface{} + fv uint64 } func stackmaptestfunc(s string, i int, vp unsafe.Pointer, sb interface{}, fv uint64) (x *uint64, y string, z *int) { - z = new(int) - x = new(uint64) - y = s + "asdf" - keepalive.s = s - keepalive.i = i - keepalive.vp = vp - keepalive.sb = sb - keepalive.fv = fv - return + z = new(int) + x = new(uint64) + y = s + "asdf" + keepalive.s = s + keepalive.i = i + keepalive.vp = vp + keepalive.sb = sb + keepalive.fv = fv + return } func TestStackMap_Dump(t *testing.T) { - args, locals := stackMap(stackmaptestfunc) - println("--- args ---") - dumpstackmap(args) - println("--- locals ---") - dumpstackmap(locals) + args, locals := stackMap(stackmaptestfunc) + println("--- args ---") + dumpstackmap(args) + println("--- locals ---") + dumpstackmap(locals) } diff --git a/internal/utils/buffer.go b/internal/utils/buffer.go index f5be131..25a7da0 100644 --- a/internal/utils/buffer.go +++ b/internal/utils/buffer.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,12 +17,12 @@ package utils import ( - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` - `github.com/cloudwego/frugal/iov` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/frugal/iov" ) var FnWrite = hir.RegisterICall( - rt.GetMethod((*iov.BufferWriter)(nil), "WriteDirect"), - emu_icall_ZeroCopyWriter_WriteDirect, + rt.GetMethod((*iov.BufferWriter)(nil), "WriteDirect"), + emu_icall_ZeroCopyWriter_WriteDirect, ) diff --git a/internal/utils/buffer_emu.go b/internal/utils/buffer_emu.go index 3257da4..18a546a 100644 --- a/internal/utils/buffer_emu.go +++ b/internal/utils/buffer_emu.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,33 +17,33 @@ package utils import ( - `unsafe` + "unsafe" - `github.com/cloudwego/frugal/internal/atm/hir` - `github.com/cloudwego/frugal/internal/rt` - `github.com/cloudwego/frugal/iov` + "github.com/cloudwego/frugal/internal/atm/hir" + "github.com/cloudwego/frugal/internal/rt" + "github.com/cloudwego/frugal/iov" ) func emu_wbuf(ctx hir.CallContext) (v iov.BufferWriter) { - (*rt.GoIface)(unsafe.Pointer(&v)).Itab = ctx.Itab() - (*rt.GoIface)(unsafe.Pointer(&v)).Value = ctx.Data() - return + (*rt.GoIface)(unsafe.Pointer(&v)).Itab = ctx.Itab() + (*rt.GoIface)(unsafe.Pointer(&v)).Value = ctx.Data() + return } func emu_bytes(ctx hir.CallContext, i int) (v []byte) { - return rt.BytesFrom(ctx.Ap(i), int(ctx.Au(i + 1)), int(ctx.Au(i + 2))) + return rt.BytesFrom(ctx.Ap(i), int(ctx.Au(i+1)), int(ctx.Au(i+2))) } func emu_seterr(ctx hir.CallContext, err error) { - vv := (*rt.GoIface)(unsafe.Pointer(&err)) - ctx.Rp(0, unsafe.Pointer(vv.Itab)) - ctx.Rp(1, vv.Value) + vv := (*rt.GoIface)(unsafe.Pointer(&err)) + ctx.Rp(0, unsafe.Pointer(vv.Itab)) + ctx.Rp(1, vv.Value) } func emu_icall_ZeroCopyWriter_WriteDirect(ctx hir.CallContext) { - if !ctx.Verify("*iii", "**") { - panic("invalid ZeroCopyWriter.WriteDirect call") - } else { - emu_seterr(ctx, emu_wbuf(ctx).WriteDirect(emu_bytes(ctx, 0), int(ctx.Au(3)))) - } + if !ctx.Verify("*iii", "**") { + panic("invalid ZeroCopyWriter.WriteDirect call") + } else { + emu_seterr(ctx, emu_wbuf(ctx).WriteDirect(emu_bytes(ctx, 0), int(ctx.Au(3)))) + } } diff --git a/internal/utils/errors.go b/internal/utils/errors.go index 2986e12..2ac976d 100644 --- a/internal/utils/errors.go +++ b/internal/utils/errors.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,51 +17,51 @@ package utils import ( - `fmt` - `reflect` + "fmt" + "reflect" ) type TypeError struct { - Note string - Type reflect.Type + Note string + Type reflect.Type } func (self TypeError) Error() string { - return fmt.Sprintf("TypeError(%s): %s", self.Type, self.Note) + return fmt.Sprintf("TypeError(%s): %s", self.Type, self.Note) } type SyntaxError struct { - Pos int - Src string - Reason string + Pos int + Src string + Reason string } func (self SyntaxError) Error() string { - return fmt.Sprintf("Syntax error at position %d: %s", self.Pos, self.Reason) + return fmt.Sprintf("Syntax error at position %d: %s", self.Pos, self.Reason) } func EType(vt reflect.Type, note string) TypeError { - return TypeError { - Type: vt, - Note: note, - } + return TypeError{ + Type: vt, + Note: note, + } } func ESyntax(pos int, src string, reason string) SyntaxError { - return SyntaxError { - Pos : pos, - Src : src, - Reason : reason, - } + return SyntaxError{ + Pos: pos, + Src: src, + Reason: reason, + } } func ESetList(pos int, src string, vt reflect.Type) SyntaxError { - return ESyntax(pos, src, fmt.Sprintf(`ambiguous type between set<%s> and list<%s>, please specify in the "frugal" tag`, vt, vt)) + return ESyntax(pos, src, fmt.Sprintf(`ambiguous type between set<%s> and list<%s>, please specify in the "frugal" tag`, vt, vt)) } func EUseOther(vt reflect.Type, alt string) TypeError { - return TypeError { - Type: vt, - Note: fmt.Sprintf("Thrift does not support %s, use %s instead", vt, alt), - } + return TypeError{ + Type: vt, + Note: fmt.Sprintf("Thrift does not support %s, use %s instead", vt, alt), + } } diff --git a/internal/utils/flags.go b/internal/utils/flags.go index 5a1cea8..e47ffd8 100644 --- a/internal/utils/flags.go +++ b/internal/utils/flags.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ package utils import ( - `os` + "os" ) var ForceEmulator = os.Getenv("FRUGAL_BACKEND") == "emu" diff --git a/internal/utils/ints.go b/internal/utils/ints.go index b5bddcc..059e196 100644 --- a/internal/utils/ints.go +++ b/internal/utils/ints.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,9 @@ package utils func MaxInt(a int, b int) int { - if a > b { - return a - } else { - return b - } + if a > b { + return a + } else { + return b + } } diff --git a/internal/utils/pcache.go b/internal/utils/pcache.go index f4270f7..8f67312 100644 --- a/internal/utils/pcache.go +++ b/internal/utils/pcache.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,156 +17,156 @@ package utils import ( - `sync` - `sync/atomic` - `unsafe` + "sync" + "sync/atomic" + "unsafe" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) /** Program Map **/ const ( - LoadFactor = 0.5 - InitCapacity = 4096 // must be a power of 2 + LoadFactor = 0.5 + InitCapacity = 4096 // must be a power of 2 ) type ProgramMap struct { - n uint64 - m uint32 - b []ProgramEntry + n uint64 + m uint32 + b []ProgramEntry } type ProgramEntry struct { - vt *rt.GoType - fn interface{} + vt *rt.GoType + fn interface{} } func newProgramMap() *ProgramMap { - return &ProgramMap { - n: 0, - m: InitCapacity - 1, - b: make([]ProgramEntry, InitCapacity), - } + return &ProgramMap{ + n: 0, + m: InitCapacity - 1, + b: make([]ProgramEntry, InitCapacity), + } } func (self *ProgramMap) get(vt *rt.GoType) interface{} { - i := self.m + 1 - p := vt.Hash & self.m - - /* linear probing */ - for ; i > 0; i-- { - if b := self.b[p]; b.vt == vt { - return b.fn - } else if b.vt == nil { - break - } else { - p = (p + 1) & self.m - } - } - - /* not found */ - return nil + i := self.m + 1 + p := vt.Hash & self.m + + /* linear probing */ + for ; i > 0; i-- { + if b := self.b[p]; b.vt == vt { + return b.fn + } else if b.vt == nil { + break + } else { + p = (p + 1) & self.m + } + } + + /* not found */ + return nil } func (self *ProgramMap) add(vt *rt.GoType, fn interface{}) *ProgramMap { - var f float64 - var p *ProgramMap - - /* check for load factor */ - if f = float64(atomic.LoadUint64(&self.n) + 1) / float64(self.m + 1); f <= LoadFactor { - p = self.copy() - } else { - p = self.rehash() - } - - /* insert the value */ - p.insert(vt, fn) - return p + var f float64 + var p *ProgramMap + + /* check for load factor */ + if f = float64(atomic.LoadUint64(&self.n)+1) / float64(self.m+1); f <= LoadFactor { + p = self.copy() + } else { + p = self.rehash() + } + + /* insert the value */ + p.insert(vt, fn) + return p } func (self *ProgramMap) copy() *ProgramMap { - p := new(ProgramMap) - p.n = self.n - p.m = self.m - p.b = make([]ProgramEntry, len(self.b)) - copy(p.b, self.b) - return p + p := new(ProgramMap) + p.n = self.n + p.m = self.m + p.b = make([]ProgramEntry, len(self.b)) + copy(p.b, self.b) + return p } func (self *ProgramMap) rehash() *ProgramMap { - c := (self.m + 1) << 1 - r := &ProgramMap{m: c - 1, b: make([]ProgramEntry, int(c))} - - /* rehash every entry */ - for i := uint32(0); i <= self.m; i++ { - if b := self.b[i]; b.vt != nil { - r.insert(b.vt, b.fn) - } - } - - /* rebuild successful */ - return r + c := (self.m + 1) << 1 + r := &ProgramMap{m: c - 1, b: make([]ProgramEntry, int(c))} + + /* rehash every entry */ + for i := uint32(0); i <= self.m; i++ { + if b := self.b[i]; b.vt != nil { + r.insert(b.vt, b.fn) + } + } + + /* rebuild successful */ + return r } func (self *ProgramMap) insert(vt *rt.GoType, fn interface{}) { - h := vt.Hash - p := h & self.m - - /* linear probing */ - for i := uint32(0); i <= self.m; i++ { - if b := &self.b[p]; b.vt != nil { - p += 1 - p &= self.m - } else { - b.vt = vt - b.fn = fn - atomic.AddUint64(&self.n, 1) - return - } - } - - /* should never happens */ - panic("no available slots") + h := vt.Hash + p := h & self.m + + /* linear probing */ + for i := uint32(0); i <= self.m; i++ { + if b := &self.b[p]; b.vt != nil { + p += 1 + p &= self.m + } else { + b.vt = vt + b.fn = fn + atomic.AddUint64(&self.n, 1) + return + } + } + + /* should never happens */ + panic("no available slots") } /** RCU Program Cache **/ type ProgramCache struct { - m sync.Mutex - p unsafe.Pointer + m sync.Mutex + p unsafe.Pointer } func CreateProgramCache() *ProgramCache { - return &ProgramCache { - m: sync.Mutex{}, - p: unsafe.Pointer(newProgramMap()), - } + return &ProgramCache{ + m: sync.Mutex{}, + p: unsafe.Pointer(newProgramMap()), + } } func (self *ProgramCache) Get(vt *rt.GoType) interface{} { - return (*ProgramMap)(atomic.LoadPointer(&self.p)).get(vt) + return (*ProgramMap)(atomic.LoadPointer(&self.p)).get(vt) } func (self *ProgramCache) Compute(vt *rt.GoType, compute func(*rt.GoType) (interface{}, error)) (interface{}, error) { - var err error - var val interface{} - - /* use defer to prevent inlining of this function */ - self.m.Lock() - defer self.m.Unlock() - - /* double check with write lock held */ - if val = self.Get(vt); val != nil { - return val, nil - } - - /* compute the value */ - if val, err = compute(vt); err != nil { - return nil, err - } - - /* update the RCU cache */ - atomic.StorePointer(&self.p, unsafe.Pointer((*ProgramMap)(atomic.LoadPointer(&self.p)).add(vt, val))) - return val, nil + var err error + var val interface{} + + /* use defer to prevent inlining of this function */ + self.m.Lock() + defer self.m.Unlock() + + /* double check with write lock held */ + if val = self.Get(vt); val != nil { + return val, nil + } + + /* compute the value */ + if val, err = compute(vt); err != nil { + return nil, err + } + + /* update the RCU cache */ + atomic.StorePointer(&self.p, unsafe.Pointer((*ProgramMap)(atomic.LoadPointer(&self.p)).add(vt, val))) + return val, nil } diff --git a/internal/utils/types.go b/internal/utils/types.go index d659dd7..8e36b2a 100644 --- a/internal/utils/types.go +++ b/internal/utils/types.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,13 +17,13 @@ package utils import ( - `reflect` + "reflect" ) var ( - byteType = reflect.TypeOf(byte(0)) + byteType = reflect.TypeOf(byte(0)) ) func IsByteType(vt reflect.Type) bool { - return vt == byteType + return vt == byteType } diff --git a/iov/buffer.go b/iov/buffer.go index 4482d3e..643d8ec 100644 --- a/iov/buffer.go +++ b/iov/buffer.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ package iov // BufferWriter implement zero-copy buffer writing. type BufferWriter interface { - // WriteDirect appends buf to the underlying buffer chain - // without copy. It splits the original buffer at remainingCap. - WriteDirect(buf []byte, remainingCap int) error + // WriteDirect appends buf to the underlying buffer chain + // without copy. It splits the original buffer at remainingCap. + WriteDirect(buf []byte, remainingCap int) error } diff --git a/licenses/LISENSE-go-spew b/licenses/LICENSE-go-spew similarity index 100% rename from licenses/LISENSE-go-spew rename to licenses/LICENSE-go-spew diff --git a/options.go b/options.go index 860d1d4..304c871 100644 --- a/options.go +++ b/options.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,13 +17,13 @@ package frugal import ( - `fmt` + "fmt" - `github.com/cloudwego/frugal/internal/opts` + "github.com/cloudwego/frugal/internal/opts" ) const ( - _MinILSize = 1024 + _MinILSize = 1024 ) // Option is the property setter function for opts.Options. @@ -39,11 +39,11 @@ type Option func(*opts.Options) // // The default value of this option is "2". func WithMaxInlineDepth(depth int) Option { - if depth < 0 { - panic(fmt.Sprintf("frugal: invalid inline depth: %d", depth)) - } else { - return func(o *opts.Options) { o.MaxInlineDepth = depth } - } + if depth < 0 { + panic(fmt.Sprintf("frugal: invalid inline depth: %d", depth)) + } else { + return func(o *opts.Options) { o.MaxInlineDepth = depth } + } } // WithMaxInlineILSize sets the maximum IL instruction count before not inlining. @@ -57,11 +57,11 @@ func WithMaxInlineDepth(depth int) Option { // // The default value of this option is "50000". func WithMaxInlineILSize(size int) Option { - if size != 0 && size < _MinILSize { - panic(fmt.Sprintf("frugal: invalid inline IL size: %d", size)) - } else { - return func(o *opts.Options) { o.MaxInlineILSize = size } - } + if size != 0 && size < _MinILSize { + panic(fmt.Sprintf("frugal: invalid inline IL size: %d", size)) + } else { + return func(o *opts.Options) { o.MaxInlineILSize = size } + } } // WithMaxPretouchDepth controls how deep the compiler goes to compile @@ -78,11 +78,11 @@ func WithMaxInlineILSize(size int) Option { // This option is only available when performing pretouch, otherwise it is // ignored and do not have any effect. func WithMaxPretouchDepth(depth int) Option { - if depth < 0 { - panic(fmt.Sprintf("frugal: invalid pretouch depth: %d", depth)) - } else { - return func(o *opts.Options) { o.MaxPretouchDepth = depth } - } + if depth < 0 { + panic(fmt.Sprintf("frugal: invalid pretouch depth: %d", depth)) + } else { + return func(o *opts.Options) { o.MaxPretouchDepth = depth } + } } // SetMaxInlineDepth sets the default maximum inlining depth for all types from @@ -95,8 +95,8 @@ func WithMaxPretouchDepth(depth int) Option { // // Returns the old opts.MaxInlineDepth value. func SetMaxInlineDepth(depth int) int { - depth, opts.MaxInlineDepth = opts.MaxInlineDepth, depth - return depth + depth, opts.MaxInlineDepth = opts.MaxInlineDepth, depth + return depth } // SetMaxInlineILSize sets the default maximum inlining IL instructions for all @@ -109,6 +109,6 @@ func SetMaxInlineDepth(depth int) int { // // Returns the old opts.MaxInlineILSize value. func SetMaxInlineILSize(size int) int { - size, opts.MaxInlineILSize = opts.MaxInlineILSize, size - return size + size, opts.MaxInlineILSize = opts.MaxInlineILSize, size + return size } diff --git a/pretouch.go b/pretouch.go index 6bc72fc..f48a092 100644 --- a/pretouch.go +++ b/pretouch.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,86 +17,86 @@ package frugal import ( - `reflect` - `sync` - - `github.com/cloudwego/frugal/internal/binary/decoder` - `github.com/cloudwego/frugal/internal/binary/encoder` - `github.com/cloudwego/frugal/internal/opts` - `github.com/cloudwego/frugal/internal/rt` - `github.com/oleiade/lane` + "reflect" + "sync" + + "github.com/cloudwego/frugal/internal/binary/decoder" + "github.com/cloudwego/frugal/internal/binary/encoder" + "github.com/cloudwego/frugal/internal/opts" + "github.com/cloudwego/frugal/internal/rt" + "github.com/oleiade/lane" ) type _Ty struct { - d int - ty *rt.GoType + d int + ty *rt.GoType } var ( - typool sync.Pool + typool sync.Pool ) func newty(ty *rt.GoType, d int) *_Ty { - if v := typool.Get(); v == nil { - return &_Ty { d, ty } - } else { - r := v.(*_Ty) - r.d, r.ty = d, ty - return r - } + if v := typool.Get(); v == nil { + return &_Ty{d, ty} + } else { + r := v.(*_Ty) + r.d, r.ty = d, ty + return r + } } // Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in // order to reduce the first-hit latency. func Pretouch(vt reflect.Type, options ...Option) error { - d := 0 - o := opts.GetDefaultOptions() - - /* apply all the options */ - for _, fn := range options { - fn(&o) - } - - /* unpack the type */ - v := make(map[*rt.GoType]bool) - t := rt.Dereference(rt.UnpackType(vt)) - - /* add the root type */ - q := lane.NewQueue() - q.Enqueue(newty(t, 1)) - - /* BFS the type tree */ - for !q.Empty() { - ty := q.Pop().(*_Ty) - tv, err := decoder.Pretouch(ty.ty, o) - - /* also pretouch the encoder */ - if err == nil { - err = encoder.Pretouch(ty.ty, o) - } - - /* mark the type as been visited */ - d, v[ty.ty] = ty.d, true - typool.Put(ty) - - /* check for errors */ - if err != nil { - return err - } - - /* check for cutoff conditions */ - if !o.CanPretouch(d) { - continue - } - - /* add all the not visited sub-types */ - for s := range tv { - if t = rt.UnpackType(s); !v[t] { - q.Enqueue(newty(t, d + 1)) - } - } - } - - /* completed with no errors */ - return nil + d := 0 + o := opts.GetDefaultOptions() + + /* apply all the options */ + for _, fn := range options { + fn(&o) + } + + /* unpack the type */ + v := make(map[*rt.GoType]bool) + t := rt.Dereference(rt.UnpackType(vt)) + + /* add the root type */ + q := lane.NewQueue() + q.Enqueue(newty(t, 1)) + + /* BFS the type tree */ + for !q.Empty() { + ty := q.Pop().(*_Ty) + tv, err := decoder.Pretouch(ty.ty, o) + + /* also pretouch the encoder */ + if err == nil { + err = encoder.Pretouch(ty.ty, o) + } + + /* mark the type as been visited */ + d, v[ty.ty] = ty.d, true + typool.Put(ty) + + /* check for errors */ + if err != nil { + return err + } + + /* check for cutoff conditions */ + if !o.CanPretouch(d) { + continue + } + + /* add all the not visited sub-types */ + for s := range tv { + if t = rt.UnpackType(s); !v[t] { + q.Enqueue(newty(t, d+1)) + } + } + } + + /* completed with no errors */ + return nil } diff --git a/tests/allsize_test.go b/tests/allsize_test.go index 7bc5fdd..ea435c7 100644 --- a/tests/allsize_test.go +++ b/tests/allsize_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/tests/baseline_test.go b/tests/baseline_test.go index 42779fc..efd2a21 100644 --- a/tests/baseline_test.go +++ b/tests/baseline_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/tests/debughook_amd64.go b/tests/debughook_amd64.go index ca44102..f67c64c 100644 --- a/tests/debughook_amd64.go +++ b/tests/debughook_amd64.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,11 +17,11 @@ package tests import ( - `os` - `syscall` - `unsafe` + "os" + "syscall" + "unsafe" - `github.com/cloudwego/frugal/internal/rt` + "github.com/cloudwego/frugal/internal/rt" ) //go:nosplit @@ -30,29 +30,29 @@ func rt_exit(int) //go:nosplit func rt_exit_hook(r int) { - if r != 0 { - println("Non-zero exit code:", r) - println("Now it's time to attach debugger, PID:", os.Getpid()) - for { - _ = true - } - } + if r != 0 { + println("Non-zero exit code:", r) + println("Now it's time to attach debugger, PID:", os.Getpid()) + for { + _ = true + } + } } func mprotectpage(addr unsafe.Pointer, prot uintptr) { - if _, _, err := syscall.RawSyscall(syscall.SYS_MPROTECT, uintptr(addr) &^ 4095, 4096, prot); err != 0 { - panic(err) - } + if _, _, err := syscall.RawSyscall(syscall.SYS_MPROTECT, uintptr(addr)&^4095, 4096, prot); err != 0 { + panic(err) + } } func init() { - if os.Getenv("FRUGAL_DEBUGGER_HOOK") == "yes" { - fp := rt.FuncAddr(rt_exit) - to := rt.FuncAddr(rt_exit_hook) - mprotectpage(fp, syscall.PROT_READ | syscall.PROT_WRITE) - *(*[2]byte)(fp) = [2]byte{0x48, 0xba} - *(*uintptr)(unsafe.Pointer(uintptr(fp) + 2)) = uintptr(to) - *(*[2]byte)(unsafe.Pointer(uintptr(fp) + 10)) = [2]byte{0xff, 0xe2} - mprotectpage(fp, syscall.PROT_READ | syscall.PROT_EXEC) - } + if os.Getenv("FRUGAL_DEBUGGER_HOOK") == "yes" { + fp := rt.FuncAddr(rt_exit) + to := rt.FuncAddr(rt_exit_hook) + mprotectpage(fp, syscall.PROT_READ|syscall.PROT_WRITE) + *(*[2]byte)(fp) = [2]byte{0x48, 0xba} + *(*uintptr)(unsafe.Pointer(uintptr(fp) + 2)) = uintptr(to) + *(*[2]byte)(unsafe.Pointer(uintptr(fp) + 10)) = [2]byte{0xff, 0xe2} + mprotectpage(fp, syscall.PROT_READ|syscall.PROT_EXEC) + } } diff --git a/tests/deep_nested_test.go b/tests/deep_nested_test.go index 07b7b20..9365d5c 100644 --- a/tests/deep_nested_test.go +++ b/tests/deep_nested_test.go @@ -1,3 +1,19 @@ +/* + * Copyright 2023 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package tests import ( diff --git a/tests/frugal_test.go b/tests/frugal_test.go index cc9ab51..379e2db 100644 --- a/tests/frugal_test.go +++ b/tests/frugal_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 ByteDance Inc. + * Copyright 2022 CloudWeGo Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License.