From 96456b6f3e89fad94808785363548941a955239a Mon Sep 17 00:00:00 2001 From: SnowOnion Date: Tue, 17 Sep 2024 23:44:39 +0800 Subject: [PATCH] feat: fuzzy search, first version (#1) * WIP: (may git squash!) feat: 1. init distance by weakening params / results 2. goimports + group (std + third party + current project) * fix: arrow direction of 'weaken results' (WR) * fix: 1. arrow direction of 'weaken results' (WR), 'weaken params' (WP); 2. Anonymize when building sigGraph * feat: 1. Distance->DistanceWithCache => avg latency (200x same distance query) -60%; 2. (WP) distance 2->3 => query func(string)int, Atoi rank 17->4. * feat: FloydWarshall; offline calc script * feat: 1. parallel Floyd-Warshall; 2. Floyd-Warshall output dump to / load from file. * tmp * v0.0.4 * upgrade hertz to avoid https://github.com/bytedance/sonic/issues/637 * floyd.json -> sigGraph.json * NewHooglyRanker functional options * HooglyRanker -> SigGraphRanker * remove deadcode; polish webpage and testcase * tailor tailor.py --- .gitignore | 5 + README.md | 8 +- collect/candidates.go | 564 +-------------------- collect/collect_test.go | 151 +----- collect/query.go | 23 +- draft.md | 55 ++ go.mod | 31 +- go.sum | 88 ++-- ranking/naive.go | 23 +- ranking/naive_test.go | 35 +- ranking/ranking.go | 5 +- ranking/sigGraph.go | 408 +++++++++++++++ ranking/sigGraph_test.go | 345 +++++++++++++ script/floydRead.go | 34 ++ script/floydWrite.go | 28 + script/parallel-floyd-gpt4o.go | 100 ++++ server/.gitignore | 1 + server/handler.go | 13 +- server/main.go | 14 +- server/model/reqresp.go | 8 +- server/res/tailor.py | 19 + server/res/views/search.html | 17 +- server/res/views/welcome_example.tmpl.html | 15 +- server/service/search_test.go | 63 ++- server/service/service.go | 15 +- u/basic.go | 35 ++ u/biz.go | 16 + u/candi.go | 177 +++++++ u/candi_test.go | 157 ++++++ u/types.go | 288 +++++++++++ u/types_test.go | 55 ++ u/u.go | 2 + u/util.go | 27 - 33 files changed, 1956 insertions(+), 869 deletions(-) create mode 100644 ranking/sigGraph.go create mode 100644 ranking/sigGraph_test.go create mode 100644 script/floydRead.go create mode 100644 script/floydWrite.go create mode 100644 script/parallel-floyd-gpt4o.go create mode 100644 server/.gitignore create mode 100755 server/res/tailor.py create mode 100644 u/basic.go create mode 100644 u/biz.go create mode 100644 u/candi.go create mode 100644 u/candi_test.go create mode 100644 u/types.go create mode 100644 u/types_test.go create mode 100644 u/u.go delete mode 100644 u/util.go diff --git a/.gitignore b/.gitignore index 843b79a..b340b8e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +######## Godoogle +sigGraph.gv +sigGraph.gv.svg +sigGraph.json + ######## GitHub generated for Go (2024-02-22) # If you prefer the allow list template instead of the deny list, see community template: # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore diff --git a/README.md b/README.md index c5650f3..6c41d08 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,14 @@ https://godoogle.sonion.xyz/ ### 🏡Deploy Your Own Godoogle ```shell +# per version +cd script/ +go run floydWrite.go + + +# per run cd server/ -go run *.go +go run . ``` Then visit [localhost:8888](http://localhost:8888). diff --git a/collect/candidates.go b/collect/candidates.go index 8561cd6..3383021 100644 --- a/collect/candidates.go +++ b/collect/candidates.go @@ -2,30 +2,40 @@ package collect import ( "fmt" - "github.com/SnowOnion/godoogle/ranking" - "github.com/SnowOnion/godoogle/u" - "github.com/samber/lo" "go/ast" "go/importer" "go/parser" "go/token" "go/types" - "golang.org/x/tools/go/packages" "os" + + "github.com/samber/lo" + "golang.org/x/tools/go/packages" + + "github.com/SnowOnion/godoogle/u" ) var ( FuncDatabase []u.T2 - NaiveRanker ranking.NaiveRanker ) func InitFuncDatabase() { var err error pkgIDs := []string{ - `golang.org/x/exp/slices`, + //`golang.org/x/exp/slices`, `github.com/samber/lo`, - //`sort`, `std`, + `github.com/dominikbraun/graph`, + + //`sort`, + // when siggraph has no depth limit: |V|=60509; |E|=351739 + // depthTTL=2: |V|=7950; |E|=13607 + // depthTTL=1: |V|=4465; |E|=4187 + + //`strconv`, + //`slices`, + //`strings`, + //`maps`, } FuncDatabase, err = ParseFuncSigsFromPackage(pkgIDs...) if err != nil { @@ -96,76 +106,6 @@ func ParseFuncSigs(src string) (sigs []u.T2, err error) { return sigs, nil } -// ParseFuncSigsFromPackage 解析 path 里 Go 程序所有函数的签名。 -func ParseFuncSigsFromDirNaive(path string) (sigs []u.T2, err error) { - // 创建一个新的 token 文件集，用于语法解析。 - fset := token.NewFileSet() - - // Parse the package's *.go files. - pkgs, err := parser.ParseDir(fset, path, nil, parser.AllErrors) // 不同级别注意一下；似乎是 .go ∪ filter - if err != nil { - return nil, err - } - - for _, pkg := range pkgs { - // Type-check the package. - // 创建类型信息配置并初始化一个新的类型检查器。 - conf := types.Config{Importer: importer.Default()} - info := &types.Info{ - Defs: make(map[*ast.Ident]types.Object), - Uses: make(map[*ast.Ident]types.Object), // 真有用还是幻觉？ - } - - // Create the package and type-check it. - files := lo.Values(pkg.Files) - - _, err := conf.Check(path, fset, files, info) // path 正确吗 // 这个返回 pkgs 用起来？ - if err != nil { - return nil, err - } - - for ind, file := range files { - fmt.Println("ind", ind, file) - - // 遍历所有的顶级声明。 - for _, decl := range file.Decls { - // 确保声明是函数声明。 - fn, ok := decl.(*ast.FuncDecl) - if !ok { - continue - } - - // 获取函数定义的对象。 - obj := info.Defs[fn.Name] - if obj == nil { - continue - } - - // 确保对象是函数对象。 - fnObj, ok := obj.(*types.Func) - if !ok { - continue - } - - // 打印函数的签名。 - typ := fnObj.Type() - sig := typ.(*types.Signature) - //sig.TypeParams() - //sig.Params() - //sig.Results() - //similarity 开整！ - - //fmt.Printf("Function %s: %s\n", fnObj.Name(), sig) - sigs = append(sigs, u.T2(lo.T2(sig, fnObj))) - //sig.Params().Len() - } - } - - } - - return sigs, nil -} - // ParseFuncSigsFromPackage 解析 ~~~ 里 Go 程序所有函数的签名，正确处理 import // patterns[i] can be: // `github.com/samber/lo` (already being go get) @@ -222,6 +162,7 @@ func ParseFuncSigsFromPackage(patterns ...string) (sigs []u.T2, err error) { // TODO not-exported receiver may have exported method, but seems not in pkg.go.dev …… // p -> q == !p || q if recv := sig.Recv(); fnObj.Exported() && (recv == nil || recv.Exported()) { + sig = u.Anonymize(sig) // TODO 不循环依赖了；去掉后续冗余的 Anonymize sigs = append(sigs, u.T2(lo.T2(sig, fnObj))) } @@ -234,31 +175,6 @@ func ParseFuncSigsFromPackage(patterns ...string) (sigs []u.T2, err error) { return sigs, nil } -//func LoadDirGPT(path string) { -// cfg := &packages.Config{ -// Mode: packages.LoadSyntax, // Load all syntax trees for the packages. -// Tests: false, -// } -// -// // Load the current package using go.mod file. -// pkgs, err := packages.Load(cfg, ".") -// if err != nil { -// fmt.Fprintf(os.Stderr, "failed to load packages: %v", err) -// os.Exit(1) -// } -// -// // Process each package and analyze it as needed. -// for _, pkg := range pkgs { -// fmt.Printf("Package: %s\n", pkg.PkgPath) -// -// // Iterate through the package's syntax trees. -// for _, file := range pkg.Syntax { -// // Perform syntax analysis or type checking on the file. -// // ... -// } -// } -//} - // patterns[i] can be: // `github.com/samber/lo` (already being go get) func LoadDirDoc(patterns ...string) ([]*packages.Package, error) { @@ -287,449 +203,7 @@ func LoadDirDoc(patterns ...string) ([]*packages.Package, error) { return pkgs, nil } -//func ParseGenDeclTypeSpecFuncSigsWithImport(src string) (sigs []*types.Signature, err error) { -// // 创建一个新的 token 文件集，用于语法解析。 -// fset := token.NewFileSet() -// -// // 解析 Go 源代码字符串。 -// file, err := parser.ParseFile(fset, "", src, parser.AllErrors) -// if err != nil { -// return nil, err -// } -// -// // 创建类型信息配置并初始化一个新的类型检查器。 -// config := packages.Config{Mode: packages.LoadAllSyntax} -// info := &types.Info{ -// Types: make(map[ast.Expr]types.TypeAndValue), -// Instances: make(map[*ast.Ident]types.Instance), // 感觉这个对 signature.tparams 比较重要…… -// Defs: make(map[*ast.Ident]types.Object), -// Uses: make(map[*ast.Ident]types.Object), -// Implicits: make(map[ast.Node]types.Object), -// Selections: make(map[*ast.SelectorExpr]*types.Selection), -// Scopes: make(map[ast.Node]*types.Scope), -// InitOrder: make([]*types.Initializer, 0), -// } -// -// // 对 AST 进行类型检查，填充 info。 -// _, err = config.Check("", fset, []*ast.File{file}, info) -// if err != nil { -// return nil, err -// } -// -// // 遍历所有的顶级声明。 -// for _, decl := range file.Decls { -// // -// genDecl, ok := decl.(*ast.GenDecl) -// if !ok { -// continue -// } -// -// // 遍历一般声明中的规格（Specs）。 -// for _, spec := range genDecl.Specs { -// // 确保规格是类型规格。 -// typeSpec, ok := spec.(*ast.TypeSpec) -// if !ok { -// continue -// } -// -// // 查找类型名称对应的对象。 -// obj := info.Defs[typeSpec.Name] -// if obj == nil { -// continue -// } -// -// //typeObj, ok := obj.(*types.Named) -// // 确保对象是类型对象。 -// -// typeObj, ok := obj.(*types.TypeName) -// if !ok { -// continue -// } -// -// //if typeParam, ok := typeObj.Type().Underlying().(*types.TypeParam); ok { -// // fmt.Println("typeParam", typeParam) -// // -// //} -// -// // 获取类型对象的类型并断言为 *types.Signature。 -// if signature, ok := typeObj.Type().Underlying().(*types.Signature); ok { -// //fmt.Printf("Function type %s: %s\n", typeObj.Name(), signature) -// -// // todo support method -// recvTypeParams0 := signature.RecvTypeParams() -// recvTypeParams := make([]*types.TypeParam, recvTypeParams0.Len()) -// for i := 0; i < recvTypeParams0.Len(); i++ { -// recvTypeParams[i] = recvTypeParams0.At(i) -// } -// -// typeParams := make([]*types.TypeParam, 0) -// nameToTypeParam := make(map[string]*types.TypeParam) -// // 填补 signature.TypeParams() 的空白 -// //fmt.Println("typeSpec.TypeParams", typeSpec.TypeParams) -// if typeSpec.TypeParams != nil { -// for _, tp := range typeSpec.TypeParams.List { -// //fmt.Println(tp) -// for _, tpn := range tp.Names { -// obj := info.Defs[tpn] -// if obj == nil { -// continue -// } -// -// if tn, ok := obj.Type().(*types.TypeParam); ok { -// //fmt.Println("tnnnnnnn", tn) -// //typeParams = append(typeParams, tn) //panic: type parameter bound more than once -// newTP := types.NewTypeParam(types.NewTypeName( -// tn.Obj().Pos(), tn.Obj().Pkg(), tn.Obj().Name(), tn.Obj().Type()), -// tn.Constraint()) -// typeParams = append(typeParams, newTP) -// nameToTypeParam[newTP.Obj().Name()] = newTP -// } -// -// } -// -// } -// } -// -// params, err := rebindVars(signature.Params(), nameToTypeParam, "params of "+signature.String()) -// if err != nil { -// return sigs, fmt.Errorf("rebinding params of %s: %w", signature, err) -// } -// results, err := rebindVars(signature.Results(), nameToTypeParam, "results of "+signature.String()) -// if err != nil { -// return sigs, fmt.Errorf("rebinding results of %s: %w", signature, err) -// } -// -// sigWithTypeParams := types.NewSignatureType( -// signature.Recv(), -// recvTypeParams, -// typeParams, //! -// types.NewTuple(params...), -// types.NewTuple(results...), -// signature.Variadic(), -// ) -// //panic: type parameter bound more than once -// //考虑让用户输入带函数名了。至少，泛型的情况。 // 我还得费劲加函数体啊，别了 -// -// sigs = append(sigs, sigWithTypeParams) -// } -// } -// -// } -// -// return sigs, nil -//} - -// Naive -func ParseGenDeclTypeSpecFuncSigs(src string) (sigs []*types.Signature, err error) { - // 创建一个新的 token 文件集，用于语法解析。 - fset := token.NewFileSet() - - // 解析 Go 源代码字符串。 - file, err := parser.ParseFile(fset, "", src, parser.AllErrors) - if err != nil { - return nil, err - } - - // 创建类型信息配置并初始化一个新的类型检查器。 - config := &types.Config{ - Importer: importer.Default(), - DisableUnusedImportCheck: true, - IgnoreFuncBodies: true, - - //types.DefaultImporter(), // gpt-4 幻觉 or 旧版？2024-03-06 01:49:43 - } - info := &types.Info{ - Types: make(map[ast.Expr]types.TypeAndValue), - Instances: make(map[*ast.Ident]types.Instance), // 感觉这个对 signature.tparams 比较重要…… - Defs: make(map[*ast.Ident]types.Object), - Uses: make(map[*ast.Ident]types.Object), - Implicits: make(map[ast.Node]types.Object), - Selections: make(map[*ast.SelectorExpr]*types.Selection), - Scopes: make(map[ast.Node]*types.Scope), - InitOrder: make([]*types.Initializer, 0), - } - - // 对 AST 进行类型检查，填充 info。 - _, err = config.Check("", fset, []*ast.File{file}, info) - if err != nil { - return nil, err - } - - // 遍历所有的顶级声明。 - for _, decl := range file.Decls { - // - genDecl, ok := decl.(*ast.GenDecl) - if !ok { - continue - } - - // 遍历一般声明中的规格（Specs）。 - for _, spec := range genDecl.Specs { - // 确保规格是类型规格。 - typeSpec, ok := spec.(*ast.TypeSpec) - if !ok { - continue - } - - // 查找类型名称对应的对象。 - obj := info.Defs[typeSpec.Name] - if obj == nil { - continue - } - - //typeObj, ok := obj.(*types.Named) - // 确保对象是类型对象。 - - typeObj, ok := obj.(*types.TypeName) - if !ok { - continue - } - - //if typeParam, ok := typeObj.Type().Underlying().(*types.TypeParam); ok { - // fmt.Println("typeParam", typeParam) - // - //} - - // 获取类型对象的类型并断言为 *types.Signature。 - if signature, ok := typeObj.Type().Underlying().(*types.Signature); ok { - //fmt.Printf("Function type %s: %s\n", typeObj.Name(), signature) - - // todo support method - recvTypeParams0 := signature.RecvTypeParams() - recvTypeParams := make([]*types.TypeParam, recvTypeParams0.Len()) - for i := 0; i < recvTypeParams0.Len(); i++ { - recvTypeParams[i] = recvTypeParams0.At(i) - } - - typeParams := make([]*types.TypeParam, 0) - nameToTypeParam := make(map[string]*types.TypeParam) - // 填补 signature.TypeParams() 的空白 - //fmt.Println("typeSpec.TypeParams", typeSpec.TypeParams) - if typeSpec.TypeParams != nil { - for _, tp := range typeSpec.TypeParams.List { - //fmt.Println(tp) - for _, tpn := range tp.Names { - obj := info.Defs[tpn] - if obj == nil { - continue - } - - if tn, ok := obj.Type().(*types.TypeParam); ok { - //fmt.Println("tnnnnnnn", tn) - //typeParams = append(typeParams, tn) //panic: type parameter bound more than once - newTP := types.NewTypeParam(types.NewTypeName( - tn.Obj().Pos(), tn.Obj().Pkg(), tn.Obj().Name(), tn.Obj().Type()), - tn.Constraint()) - typeParams = append(typeParams, newTP) - nameToTypeParam[newTP.Obj().Name()] = newTP - } - - } - - } - } - - params, err := rebindVars(signature.Params(), nameToTypeParam, "params of "+signature.String()) - if err != nil { - return sigs, fmt.Errorf("rebinding params of %s: %w", signature, err) - } - results, err := rebindVars(signature.Results(), nameToTypeParam, "results of "+signature.String()) - if err != nil { - return sigs, fmt.Errorf("rebinding results of %s: %w", signature, err) - } - - sigWithTypeParams := types.NewSignatureType( - signature.Recv(), - recvTypeParams, - typeParams, //! - types.NewTuple(params...), - types.NewTuple(results...), - signature.Variadic(), - ) - //panic: type parameter bound more than once - //考虑让用户输入带函数名了。至少，泛型的情况。 // 我还得费劲加函数体啊，别了 - - sigs = append(sigs, sigWithTypeParams) - } - } - - } - - return sigs, nil -} - -// vars can be signature.Params() or signature.Results() -func rebindVars(vars *types.Tuple, nameToTypeParam map[string]*types.TypeParam, debug any) (varsBound []*types.Var, err error) { - varsBound = make([]*types.Var, vars.Len()) - for i := 0; i < vars.Len(); i++ { - var_ := vars.At(i) - newParam, err := rebindVar(var_, nameToTypeParam, debug) - if err != nil { - return nil, fmt.Errorf("rebindVars %d: %w", i, err) - } - varsBound[i] = newParam - } - return varsBound, nil -} - -func rebindVar(var_ *types.Var, nameToTypeParam map[string]*types.TypeParam, debug any) (varBound *types.Var, err error) { - if var_ == nil { - return nil, nil - } - fdType, err := rebindType(var_.Type(), nameToTypeParam, debug) - if err != nil { - return nil, fmt.Errorf("rebindVar: %w", err) - } - return types.NewVar(var_.Pos(), var_.Pkg(), var_.Name(), fdType), nil -} - -// [typBound] will have the same type as [typ]. -// I tried to use [T types.Type](var_ T……), but failed. -// TODO maybe make it OO, hiding nameToTypeParam to field. -func rebindType(typ types.Type, nameToTypeParam map[string]*types.TypeParam, debug any) (typBound types.Type, err error) { - // refers to types.IdenticalIgnoreTags - switch typ.(type) { - case *types.TypeParam: - tp := typ.(*types.TypeParam) - newTP, bound := nameToTypeParam[tp.Obj().Name()] - if !bound { - // Type checking should not let this happen? - return nil, fmt.Errorf("[typ=%s] has a parameterized [type=%s] not declared in type typ list", typ, tp.Obj().Name()) - } - return newTP, nil - - case *types.Array: - ar := typ.(*types.Array) - // I need a Maybe Monad - el, err := rebindType(ar.Elem(), nameToTypeParam, debug) - if err != nil { - return nil, fmt.Errorf("rebindType Array: %w", err) - } - return types.NewArray(el, ar.Len()), nil - - case *types.Slice: - sl := typ.(*types.Slice) - el, err := rebindType(sl.Elem(), nameToTypeParam, debug) - if err != nil { - return nil, fmt.Errorf("rebindType Slice: %w", err) - } - return types.NewSlice(el), nil - - case *types.Struct: // TODO test - st := typ.(*types.Struct) - fields := make([]*types.Var, st.NumFields()) - tags := make([]string, st.NumFields()) - for i := 0; i < st.NumFields(); i++ { - fd, err := rebindVar(st.Field(i), nameToTypeParam, debug) - if err != nil { - return nil, fmt.Errorf("rebindType Struct: %w", err) - } - fields[i] = fd - tags[i] = st.Tag(i) // just copy - } - return types.NewStruct(fields, tags), nil - - case *types.Pointer: - pt := typ.(*types.Pointer) - el, err := rebindType(pt.Elem(), nameToTypeParam, debug) - if err != nil { - return nil, fmt.Errorf("rebindType Pointer: %w", err) - } - return types.NewPointer(el), nil - - case *types.Tuple: // “simplified struct” - tp := typ.(*types.Tuple) - vars := make([]*types.Var, tp.Len()) - for i := 0; i < tp.Len(); i++ { - var_, err := rebindVar(tp.At(i), nameToTypeParam, debug) - if err != nil { - return nil, fmt.Errorf("rebindType Tuple: %w", err) - } - vars[i] = var_ - } - return types.NewTuple(vars...), nil - - case *types.Signature: - // spec#FunctionType can not have type params, so I don’t need to handle nested type param scope. Lucky. - // But that’s exactly the reason to rebindVars. Unlucky. - - sig := typ.(*types.Signature) - params, err := rebindVars(sig.Params(), nameToTypeParam, "params of "+sig.String()+" <- "+debug.(string)) // mutual recursion - if err != nil { - return nil, fmt.Errorf("rebindType Signature Params: %w", err) - } - results, err := rebindVars(sig.Results(), nameToTypeParam, "results of "+sig.String()+" <- "+debug.(string)) - if err != nil { - return nil, fmt.Errorf("rebindType Signature Results: %w", err) - } - recv, err := rebindVar(sig.Recv(), nameToTypeParam, debug) - if err != nil { - return nil, fmt.Errorf("rebindType Signature Recv: %w", err) - } - - // trivial - recvTypeParams0 := sig.RecvTypeParams() - recvTypeParams := make([]*types.TypeParam, recvTypeParams0.Len()) - for i := 0; i < recvTypeParams0.Len(); i++ { - recvTypeParams[i] = recvTypeParams0.At(i) - } - typeParams0 := sig.TypeParams() - typeParams := make([]*types.TypeParam, typeParams0.Len()) - for i := 0; i < typeParams0.Len(); i++ { - typeParams[i] = typeParams0.At(i) - } - - return types.NewSignatureType(recv, recvTypeParams, typeParams, types.NewTuple(params...), types.NewTuple(results...), sig.Variadic()), nil - - case *types.Union: // TODO: is there? - un := typ.(*types.Union) - terms := make([]*types.Term, un.Len()) - for i := 0; i < un.Len(); i++ { - newType, err := rebindType(un.Term(i).Type(), nameToTypeParam, debug) - if err != nil { - return nil, fmt.Errorf("rebindType Union Recv: %w", err) - } - terms[i] = types.NewTerm(un.Term(i).Tilde(), newType) - } - return types.NewUnion(terms), nil - - case *types.Interface: - // TODO impl - //iface:=typ.(*types.Interface) - return typ, nil - - case *types.Map: - mp := typ.(*types.Map) - key, err := rebindType(mp.Key(), nameToTypeParam, debug) - if err != nil { - return nil, fmt.Errorf("rebindType Map Key: %w", err) - } - el, err := rebindType(mp.Elem(), nameToTypeParam, debug) - if err != nil { - return nil, fmt.Errorf("rebindType Map Elem: %w", err) - } - return types.NewMap(key, el), nil - - case *types.Chan: - ch := typ.(*types.Chan) - // I need a Maybe Monad - el, err := rebindType(ch.Elem(), nameToTypeParam, debug) - if err != nil { - return nil, fmt.Errorf("rebindType Chan: %w", err) - } - return types.NewChan(ch.Dir(), el), nil - - case *types.Named: // func(A[T]) - nm := typ.(*types.Named) - nm.TypeParams() // does it mean type args here? - // TODO impl - return typ, nil - - default: // nil, *types.Basic - return typ, nil - } -} - +// TODO use or del // ParseFuncSigsFromPackage 解析 ~~~ 里 Go 程序所有函数的签名，正确处理 import // patterns[i] can be: // `github.com/samber/lo` (already being go get) diff --git a/collect/collect_test.go b/collect/collect_test.go index 6bb9227..3d32ce5 100644 --- a/collect/collect_test.go +++ b/collect/collect_test.go @@ -2,10 +2,11 @@ package collect import ( "fmt" - "github.com/stretchr/testify/assert" "go/token" "go/types" "testing" + + "github.com/stretchr/testify/assert" ) func Test1(t *testing.T) { @@ -67,154 +68,6 @@ func Test3(t *testing.T) { //} } -func TestDummy(t *testing.T) { - // TODO more, to cover rebind* - inps := []string{ - "[T comparable] func([]T)T", - "func()", - "func(string)", - "func(int32, int) int", - "func(string,...interface{})", - "func (format string, a ...any) (n int, err error)", - "[T any] func(T)", - "[T comparable] func(T)", - //"[a,b any] func (col []a, iter func(it a) b) (r1 []b)", // hsMap - //"[a,b any] func (col []a, iter func(it a) b) []b", // hsMap - "[a,b any] func([]a, func(a) b) []b", // hsMap - //"[a,b any] func(col []a, iter func(it a, idx int) b) (r1 []b)", // lo.Map - //"[a,b any] func(col []a, iter func(it a, idx int) b) []b", // lo.Map - "[a,b any] func([]a, func(a, int) b) []b", // lo.Map, - "[S ~[]E, E constraints.Ordered] func(x S)", // https://pkg.go.dev/golang.org/x/exp/slices#Sort - "[E constraints.Ordered] func(x []E)", - } - - t1Any := types.NewTypeParam( - types.NewTypeName(token.NoPos, nil, "T1", nil /*这里？TODO*/), - types.Universe.Lookup("any").Type()) - t1Comparable := types.NewTypeParam( - types.NewTypeName(token.NoPos, nil, "T1", nil), - types.Universe.Lookup("comparable").Type()) - t1Comparable2 := types.NewTypeParam( - types.NewTypeName(token.NoPos, nil, "T1", nil), - types.Universe.Lookup("comparable").Type()) - t1Any2 := types.NewTypeParam( - types.NewTypeName(token.NoPos, nil, "T1", nil /*这里？TODO*/), - types.Universe.Lookup("any").Type()) - t2Any2 := types.NewTypeParam( - types.NewTypeName(token.NoPos, nil, "T1", nil /*这里？TODO*/), - types.Universe.Lookup("any").Type()) - t1Any3 := types.NewTypeParam( - types.NewTypeName(token.NoPos, nil, "T1", nil /*这里？TODO*/), - types.Universe.Lookup("any").Type()) - t2Any3 := types.NewTypeParam( - types.NewTypeName(token.NoPos, nil, "T1", nil /*这里？TODO*/), - types.Universe.Lookup("any").Type()) - - outputs := []*types.Signature{ - types.NewSignatureType(nil, nil, - []*types.TypeParam{t1Comparable2}, - types.NewTuple(types.NewVar(token.NoPos, nil, "xxxx", types.NewSlice(t1Comparable2))), - types.NewTuple(types.NewVar(token.NoPos, nil, "xxxx", t1Comparable2)), - false, - ), - types.NewSignatureType(nil, nil, nil, - types.NewTuple(), - types.NewTuple(), - false), - types.NewSignatureType(nil, nil, nil, - types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.String])), - types.NewTuple(), - false), - types.NewSignatureType(nil, nil, nil, - types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.Int32]), - types.NewVar(token.NoPos, nil, "", types.Typ[types.Int])), - types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.Int])), - false), - types.NewSignatureType(nil, nil, nil, - types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.String]), - types.NewVar(token.NoPos, nil, "", types.NewSlice(types.NewInterfaceType(nil, nil)) /*any/interface{}*/)), - types.NewTuple(), - true /*params[-1] needs to be a slice*/), - types.NewSignatureType(nil, nil, nil, - types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.String]), - types.NewVar(token.NoPos, nil, "", types.NewSlice(types.NewInterfaceType(nil, nil)) /*any/interface{}*/)), - types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.Int]), - types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("error").Type())), - true /*params[-1] needs to be a slice*/), - types.NewSignatureType(nil, nil, - []*types.TypeParam{t1Any}, - types.NewTuple(types.NewVar(token.NoPos, nil, "xxxx", t1Any)), - nil, false, - ), - types.NewSignatureType(nil, nil, - []*types.TypeParam{t1Comparable}, - types.NewTuple(types.NewVar(token.NoPos, nil, "xxxx", t1Comparable)), - nil, false, - ), - - // hsMap - types.NewSignatureType(nil, nil, - []*types.TypeParam{t1Any2, t2Any2}, - types.NewTuple( - types.NewVar(token.NoPos, nil, "", - types.NewSlice(t1Any2), - ), - types.NewVar(token.NoPos, nil, "", - types.NewSignatureType(nil, nil, nil, - types.NewTuple( - types.NewVar(token.NoPos, nil, "", t1Any2), - ), - types.NewTuple(types.NewVar(token.NoPos, nil, "", t2Any2)), - false), - ), - ), - types.NewTuple( - types.NewVar(token.NoPos, nil, "", - types.NewSlice(t2Any2), - ), - ), - false), - - // lo.Map - types.NewSignatureType(nil, nil, - []*types.TypeParam{t1Any3, t2Any3}, - types.NewTuple( - types.NewVar(token.NoPos, nil, "", - types.NewSlice(t1Any3), - ), - types.NewVar(token.NoPos, nil, "", - types.NewSignatureType(nil, nil, nil, - types.NewTuple( - types.NewVar(token.NoPos, nil, "", t1Any3), - types.NewVar(token.NoPos, nil, "", types.Typ[types.Int]), // lo > hs - ), - types.NewTuple(types.NewVar(token.NoPos, nil, "", t2Any3)), - false), - ), - ), - types.NewTuple( - types.NewVar(token.NoPos, nil, "", - types.NewSlice(t2Any3), - ), - ), - false), - nil, - nil, - } - - for i, inp := range inps { - out := outputs[i] - sig, err := Dummy(inp) - t.Log(i, err) - t.Log(i, inp) - t.Log(i, sig) - t.Log(i, out) - assert.Nil(t, err, "case~~~~ %d", i) - assert.True(t, types.IdenticalIgnoreTags(out, sig), "case~~~~ %d", i) - } - -} - // ! Error: Expected nil, but got: types.Error{Fset:(*token.FileSet)(0xc00016b280), Pos:41, Msg:"missing return", Soft:false, go116code:102, go116start:41, go116end:41} // 补上 named return 和 {return} 呢…… -> 可以！但要注意定义和引用相同的 TypeParam 对象~~~ func TestDummyF(t *testing.T) { diff --git a/collect/query.go b/collect/query.go index 8212b7b..dda1db5 100644 --- a/collect/query.go +++ b/collect/query.go @@ -5,30 +5,11 @@ import ( "go/types" ) -// Dummy rawQuery e.g. -// "func(rune) bool" -// "[T, R any, K comparable] func (collection []T, iteratee func(item T, index int) R) []R" -func Dummy(rawQuery string) (*types.Signature, error) { - augmentedQuery := `package dummy -import ( - //"golang.org/x/exp/constraints" - "sort" - "time" -) -type dummy ` + rawQuery - sigs, err := ParseGenDeclTypeSpecFuncSigs(augmentedQuery) - if err != nil { - return nil, err - } - if len(sigs) == 0 { - return nil, errors.New("no type signature in augmentedQuery") - } - return sigs[0], nil -} - // DummyF rawQuery e.g. // "func f(rune) bool" // " func fff[T, R any, K comparable](collection []T, f func(item T, index int) R) []R" +// +// Deprecated. Use Dummy. func DummyF(rawQuery string) (*types.Signature, error) { augmentedQuery := `package dummy ` + rawQuery + `{return}` diff --git a/draft.md b/draft.md index e69de29..457fd29 100644 --- a/draft.md +++ b/draft.md @@ -0,0 +1,55 @@ + +``` +# Heuristics +(Mixing Go and Haskell notations...) + +从类型 f 指向类型 g，则用 g 的实例可以实现 f 的实例。（考虑 柯里-霍华德对应。） +例： +// func(string) (int, error) -> func(string) int +// 一种实现方式： +func Atoi(s string) (int, error); +func MustAtoi(s string) int { + i, _ := Atoi(s) + return i +} + +## Distances: +0. + a -> b <~~> a -> b // identical (ID) +1. + (x1,x2) -> (y1,y2) <~~> (x2,x1) -> (y1,y2) // permute params (PP) + (x1,x2) -> (y1,y2) <~~> (x1,x2) -> (y2,y1) // permute results (PR) +2. + (x1,x2) -> (y1,y2) ~~> (x1,x2) -> y2 // weaken results (WR) +3. // Motivation: query `func(string)int` => get too many `func() int` before the expected `strconv.Atoi :: func(string) (int, error)` + (x1,x2) -> (y1,y2) <~~ x2 -> (y1,y2) // weaken params (WP) + +``` + +``` +- subtyping + +- currying? convenient or trouble? + +- a -> b ~~ a -> c // possible first step +- a -> b ~~ c -> b // possible last step +``` + +``` +- Fuzzy context: “Type name, type only name” ~~> Guess what user means (module, package, version) + - type (v.) + - according to popularity? + +- Beyond expectation: + - `func([]string) []int` ~~> `func[a,b any]([]a) []b` + - `func([]string) []int` ~~> `func[a,b any]([]a, func(a) b) []b` + `func(string) int` + +... +``` + +## Benchmark + +```shell +cd ranking +go test -v . -test.bench ^BenchmarkDistance -test.run ^BenchmarkDistance -benchtime=200x -benchmem +``` \ No newline at end of file diff --git a/go.mod b/go.mod index a28ec89..b154e3a 100644 --- a/go.mod +++ b/go.mod @@ -1,26 +1,32 @@ module github.com/SnowOnion/godoogle -go 1.21.4 +go 1.21 require ( - github.com/cloudwego/hertz v0.8.1 + github.com/cloudwego/hertz v0.9.3 + github.com/dominikbraun/graph v0.23.0 github.com/hertz-contrib/logger/logrus v1.0.1 + github.com/hertz-contrib/pprof v0.1.2 github.com/hertz-contrib/requestid v1.1.0 - github.com/samber/lo v1.39.0 + github.com/samber/lo v1.47.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 - golang.org/x/tools v0.19.0 + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d ) require ( github.com/bytedance/go-tagexpr/v2 v2.9.2 // indirect - github.com/bytedance/gopkg v0.0.0-20220413063733-65bf48ffb3a7 // indirect - github.com/bytedance/sonic v1.8.1 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/cloudwego/netpoll v0.5.0 // indirect + github.com/bytedance/gopkg v0.1.0 // indirect + github.com/bytedance/sonic v1.12.0 // indirect + github.com/bytedance/sonic/loader v0.2.0 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/cloudwego/netpoll v0.6.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/felixge/fgprof v0.9.3 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/golang/protobuf v1.5.0 // indirect + github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect github.com/google/uuid v1.3.0 // indirect github.com/henrylee2cn/ameda v1.4.10 // indirect github.com/henrylee2cn/goutil v0.0.0-20210127050712-89660552f6f8 // indirect @@ -32,9 +38,12 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect - golang.org/x/exp v0.0.0-20240314144324-c7f7c6466f7f // indirect - golang.org/x/mod v0.16.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.16.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/dominikbraun/graph v0.23.0 => github.com/SnowOnion/graph v0.0.0-20240707012141-297ba1ff3c08 diff --git a/go.sum b/go.sum index 43075b8..73304dc 100644 --- a/go.sum +++ b/go.sum @@ -1,32 +1,48 @@ +github.com/SnowOnion/graph v0.0.0-20240707012141-297ba1ff3c08 h1:MfwymH7yusMDHKNj4fw6d06czuH5+ehpRMNvjlOSd1w= +github.com/SnowOnion/graph v0.0.0-20240707012141-297ba1ff3c08/go.mod h1:U8MccQuv/s2QmEeBCy3AZZkPsvhuoEhkm4QY4pIAcYA= github.com/bytedance/go-tagexpr/v2 v2.9.2 h1:QySJaAIQgOEDQBLS3x9BxOWrnhqu5sQ+f6HaZIxD39I= github.com/bytedance/go-tagexpr/v2 v2.9.2/go.mod h1:5qsx05dYOiUXOUgnQ7w3Oz8BYs2qtM/bJokdLb79wRM= -github.com/bytedance/gopkg v0.0.0-20220413063733-65bf48ffb3a7 h1:PtwsQyQJGxf8iaPptPNaduEIu9BnrNms+pcRdHAxZaM= github.com/bytedance/gopkg v0.0.0-20220413063733-65bf48ffb3a7/go.mod h1:2ZlV9BaUH4+NXIBF0aMdKKAnHTzqH+iMU4KUjAbL23Q= -github.com/bytedance/mockey v1.2.1 h1:g84ngI88hz1DR4wZTL3yOuqlEcq67MretBfQUdXwrmw= -github.com/bytedance/mockey v1.2.1/go.mod h1:+Jm/fzWZAuhEDrPXVjDf/jLM2BlLXJkwk94zf2JZ3X4= +github.com/bytedance/gopkg v0.0.0-20240507064146-197ded923ae3/go.mod h1:FtQG3YbQG9L/91pbKSw787yBQPutC+457AvDW77fgUQ= +github.com/bytedance/gopkg v0.1.0 h1:aAxB7mm1qms4Wz4sp8e1AtKDOeFLtdqvGiUe7aonRJs= +github.com/bytedance/gopkg v0.1.0/go.mod h1:FtQG3YbQG9L/91pbKSw787yBQPutC+457AvDW77fgUQ= +github.com/bytedance/mockey v1.2.12 h1:aeszOmGw8CPX8CRx1DZ/Glzb1yXvhjDh6jdFBNZjsU4= +github.com/bytedance/mockey v1.2.12/go.mod h1:3ZA4MQasmqC87Tw0w7Ygdy7eHIc2xgpZ8Pona5rsYIk= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.8.1 h1:NqAHCaGaTzro0xMmnTCLUyRlbEP6r8MCA1cJUrH3Pu4= -github.com/bytedance/sonic v1.8.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.12.0 h1:YGPgxF9xzaCNvd/ZKdQ28yRovhfMFZQjuk6fKBzZ3ls= +github.com/bytedance/sonic v1.12.0/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= +github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/hertz v0.4.2/go.mod h1:K1U0RlU07CDeBINfHNbafH/3j9uSgIW8otbjUys3OPY= -github.com/cloudwego/hertz v0.8.1 h1:3Upzd9o5yNPz6rLx70J5xpo5emosKNkmwW00WgQhf/0= -github.com/cloudwego/hertz v0.8.1/go.mod h1:WliNtVbwihWHHgAaIQEbVXl0O3aWj0ks1eoPrcEAnjs= +github.com/cloudwego/hertz v0.9.3 h1:uajvLn6LjEPjUqN/ewUZtWoRQWa2es2XTELdqDlOYMw= +github.com/cloudwego/hertz v0.9.3/go.mod h1:gGVUfJU/BOkJv/ZTzrw7FS7uy7171JeYIZvAyV3wS3o= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cloudwego/netpoll v0.3.1/go.mod h1:1T2WVuQ+MQw6h6DpE45MohSvDTKdy2DlzCx2KsnPI4E= -github.com/cloudwego/netpoll v0.5.0 h1:oRrOp58cPCvK2QbMozZNDESvrxQaEHW2dCimmwH1lcU= -github.com/cloudwego/netpoll v0.5.0/go.mod h1:xVefXptcyheopwNDZjDPcfU6kIjZXZ4nY550k1yH9eQ= +github.com/cloudwego/netpoll v0.6.2 h1:+KdILv5ATJU+222wNNXpHapYaBeRvvL8qhJyhcxRxrQ= +github.com/cloudwego/netpoll v0.6.2/go.mod h1:kaqvfZ70qd4T2WtIIpCOi5Cxyob8viEpzLhCrTrz3HM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= +github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -38,20 +54,22 @@ github.com/henrylee2cn/goutil v0.0.0-20210127050712-89660552f6f8 h1:yE9ULgp02BhY github.com/henrylee2cn/goutil v0.0.0-20210127050712-89660552f6f8/go.mod h1:Nhe/DM3671a5udlv2AdV2ni/MZzgfv2qrPL5nIi3EGQ= github.com/hertz-contrib/logger/logrus v1.0.1 h1:1iFu/L92QlFSDXUn77WJL32dk/5HBzAUziG1OqcNMeE= github.com/hertz-contrib/logger/logrus v1.0.1/go.mod h1:SqDYLwVq5hTItYqimgZQbFCYPOIGNvBTq0Ip2OQwMcY= +github.com/hertz-contrib/pprof v0.1.2 h1:eC4jpg8ToSi+9YEOPIr3jki8e/ix3aFPtphCgJ36T6Q= +github.com/hertz-contrib/pprof v0.1.2/go.mod h1:OKXw5wCUcy1OVwgQLsoCS9JzfcdjoofP+7Uk4c7P9Po= github.com/hertz-contrib/requestid v1.1.0 h1:+y1cuNlNX2KUoEC1SnBJ6M55/TlMTx3M9yxkqi0oTkk= github.com/hertz-contrib/requestid v1.1.0/go.mod h1:+l5CbZl//cSUoos421fnDFKQ6YYlVHcYc3Ri7AS8DUA= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/nyaruka/phonenumbers v1.0.55 h1:bj0nTO88Y68KeUQ/n3Lo2KgK7lM1hF7L9NFuwcCl3yg= github.com/nyaruka/phonenumbers v1.0.55/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= -github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= +github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -79,30 +97,29 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= -golang.org/x/exp v0.0.0-20240314144324-c7f7c6466f7f h1:3CW0unweImhOzd5FmYuRsD4Y4oQFKZIjAnKbjV4WIrw= -golang.org/x/exp v0.0.0-20240314144324-c7f7c6466f7f/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220110181412-a018aaa089fe/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= @@ -113,4 +130,5 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/ranking/naive.go b/ranking/naive.go index c776a41..4ae7247 100644 --- a/ranking/naive.go +++ b/ranking/naive.go @@ -1,31 +1,15 @@ package ranking import ( - "github.com/SnowOnion/godoogle/u" "go/types" "sort" + + "github.com/SnowOnion/godoogle/u" ) type NaiveRanker struct{} -/* -Heuristics -(Mixing Go and Haskell notations...) -Distances: -0. a -> b ~~ a -> b // identical -+1. a -> b -> c ~~ b -> a -> c // flip params - +1. (a,b,c) -> d ~~ (c,a,b) -> d // flip params twice -+1. a -> (b,c) ~~ a -> (c,b) // flip results - +1. a -> (b,c,d) ~~ a -> (c,d,b) // flip results twice - -+2. a -> b ~~ a -> c // possible first step -+2. a -> b ~~ c -> b // possible last step ---- -- subtyping - - -*/ - +// Rank by “discrete metric”(离散度量): signatures identical to query are ranked first; others are ranked arbitrarily. func (r NaiveRanker) Rank(query *types.Signature, candidates []u.T2) []u.T2 { if query == nil { panic("Rank query == nil") @@ -36,7 +20,6 @@ func (r NaiveRanker) Rank(query *types.Signature, candidates []u.T2) []u.T2 { less := func(i, j int) bool { candi := result[i].A eq := types.IdenticalIgnoreTags(candi, query) - //fmt.Printf("%s==%s: %t\n", candi, query, eq) return eq } sort.Slice(result, less) diff --git a/ranking/naive_test.go b/ranking/naive_test.go index 3432949..5090206 100644 --- a/ranking/naive_test.go +++ b/ranking/naive_test.go @@ -2,17 +2,14 @@ package ranking import ( "fmt" + "testing" + + "github.com/samber/lo" + "github.com/SnowOnion/godoogle/collect" "github.com/SnowOnion/godoogle/u" - "github.com/samber/lo" - "go/types" - "testing" ) -type a func(int, int) int -type b func(int, int) bool -type c[T comparable] func(T, T) bool - func TestNaive1(t *testing.T) { src := ` package main @@ -52,7 +49,7 @@ func Eq[T comparable](a, b T) bool { for _, inp := range inps { //fmt.Println(types.IdenticalIgnoreTags(q.A, q.A), q) fmt.Println() - fmt.Println(ranker.Rank(lo.T2(collect.Dummy(inp)).A, sigs)) + fmt.Println(ranker.Rank(lo.T2(u.Dummy(inp)).A, sigs)) } //fmt.Println(lo.Map(ranker.Rank(sigs[0].A, sigs), fst)) @@ -72,7 +69,7 @@ func TestNaive2(t *testing.T) { t.Error(err) } - inps := []string{ + suite := []lo.Tuple2[string, string]{ //`func (int,int) int`, //`func (int,int) bool`, //`[T comparable] func (T,T) bool`, @@ -82,7 +79,8 @@ func TestNaive2(t *testing.T) { //`[E constraints.Ordered] func(x []E)`, //`[E constraints.Ordered] func([]E)`, //`[E constraints.Ordered] func([]E) []E`, - `[a, b any] func (collection []a, iteratee func(item a, index int) b) []b`, // lo.Map + lo.T2(`[a, b any] func (collection []a, iteratee func(item a, index int) b) []b`, `github.com/samber/lo.Map`), + lo.T2(`[b ,a any] func (collection []a, iteratee func(item a, index int) b) []b`, `github.com/samber/lo.Map`), // TODO //`[T, R any] func (collection []T, iteratee func(item T, index int) R) []R`, //`[T, R any, K comparable] func (collection []T, iteratee func(item T, index int) R) []R`, } @@ -95,16 +93,22 @@ func TestNaive2(t *testing.T) { // fmt.Println(ind, sigDecl) //} - for _, inp := range inps { - inpSig := lo.T2(collect.Dummy(inp)).A + for _, inOut := range suite { + inpSig := lo.T2(u.Dummy(inOut.A)).A //fmt.Println(types.IdenticalIgnoreTags(q.A, q.A), q) fmt.Println("~~~Search result of", inpSig) //fmt.Println(ranker.Rank(inpSig, sigs)) - for ind, sigDecl := range ranker.Rank(inpSig, sigs) { - fmt.Printf("%4d %s#%s\n", ind, sigDecl.B.Name(), sigDecl) + result := ranker.Rank(inpSig, sigs) + for ind, sigDecl := range result[:min(10, len(result))] { + fmt.Printf("%4d %s - %s\n", ind, sigDecl.B.Name(), sigDecl) } fmt.Println("~~~~~~~~~~~~") + if inOut.B != "" { + if result[0].B.FullName() != inOut.B { + t.Errorf("Top 1 is %s; expected %s", result[0].B.FullName(), inOut.B) + } + } //fmt.Println("Rank~~~~", inpSig) //fmt.Println("Candi~~~", sigs[152].A) @@ -116,6 +120,3 @@ func TestNaive2(t *testing.T) { //fmt.Println(lo.Map(ranker.Rank(sigs[2].A, sigs), fst)) } - -// func fst[A, B any](t lo.Tuple2[A, B], _ int) A { return t.A } -func fst(t u.T2, _ int) *types.Signature { return t.A } diff --git a/ranking/ranking.go b/ranking/ranking.go index a8a24cf..9a00a6e 100644 --- a/ranking/ranking.go +++ b/ranking/ranking.go @@ -1,8 +1,9 @@ package ranking import ( - "github.com/SnowOnion/godoogle/u" "go/types" + + "github.com/SnowOnion/godoogle/u" ) type Ranker interface { @@ -17,3 +18,5 @@ type IdentityRanker struct{} func (r IdentityRanker) Rank(query *types.Signature, candidates []u.T2) []u.T2 { return candidates } + +var DefaultRanker Ranker = &SigGraphRanker{} diff --git a/ranking/sigGraph.go b/ranking/sigGraph.go new file mode 100644 index 0000000..b5ba520 --- /dev/null +++ b/ranking/sigGraph.go @@ -0,0 +1,408 @@ +package ranking + +import ( + "encoding/json" + "go/types" + "math" + "os" + "runtime" + "sort" + "strconv" + "sync" + + "github.com/cloudwego/hertz/pkg/common/hlog" + "github.com/dominikbraun/graph" + "github.com/dominikbraun/graph/draw" + "github.com/samber/lo" + + "github.com/SnowOnion/godoogle/u" +) + +type SigGraphRanker struct { + sigIndex map[SigStr][]u.T2 + hash func(sig *types.Signature) string + sigGraph graph.Graph[SigStr, *types.Signature] + distCache map[lo.Tuple2[SigStr, SigStr]]int // TODO just use map[SigStr]map[SigStr]int? + + loadFromFile bool +} + +type SigStr = string // types.Signature#String() + +func NewSigGraphRanker(candidates []u.T2, options ...func(*SigGraphRanker)) *SigGraphRanker { + hash := func(sig *types.Signature) string { + return sig.String() + } + r := &SigGraphRanker{ + sigIndex: make(map[SigStr][]u.T2), + hash: hash, + sigGraph: graph.New(hash, graph.Directed(), graph.Acyclic(), graph.Weighted()), + distCache: make(map[lo.Tuple2[SigStr, SigStr]]int), + loadFromFile: false, + } + for _, o := range options { + o(r) + } + r.InitCandidates(candidates) + + return r +} + +func LoadFromFile(b bool) func(*SigGraphRanker) { + return func(r *SigGraphRanker) { + r.loadFromFile = b + } +} + +func (r *SigGraphRanker) InitCandidates(candidates []u.T2) { + for _, t := range candidates { + //hlog.Debug(i, " ", t.String()) + sigStr := t.A.String() // todo anonymize? + _, ok := r.sigIndex[sigStr] + if ok { + r.sigIndex[sigStr] = append(r.sigIndex[sigStr], t) + continue + } + r.sigIndex[sigStr] = []u.T2{t} + + // TODO 暴露配置项 + r.InitDFS(u.Anonymize(t.A), 3) + } + hlog.Info("|candidates|=", len(candidates), "; |sigIndex|=", len(r.sigIndex)) + o, _ := r.sigGraph.Order() + s, _ := r.sigGraph.Size() + hlog.Info("Graph order and size: |V|=", o, "; |E|=", s) + file2, _ := os.Create("./sigGraph.gv") + _ = draw.DOT(r.sigGraph, file2) // then: dot -Tsvg -O sigGraph.gv && open sigGraph.gv.svg -a firefox + + if r.loadFromFile { + r.InitFloydWarshallFromFile() + } else { + workers := max(runtime.NumCPU()-2, 1) + hlog.Info("workers: ", workers) + r.InitFloydWarshall(workers) // usually too slow. + } +} + +func (r *SigGraphRanker) InitDFS(sig *types.Signature, depthTTL int) { + if _, err := r.sigGraph.Vertex(r.hash(sig)); err == nil { + return + } + sig = u.Anonymize(sig) // if not Anonymize: things like `func(s string) string` go in... // TODO 提效：一棵树只需 Anonymize 一次 + _ = r.sigGraph.AddVertex(sig) + + if depthTTL <= 0 { + return + } + + // pause, sleep + //for _, mut := range permuteParams(sig) { + // r.InitDFS(mut) + // _ = addEdge(r.sigGraph, r.hash(mut), r.hash(sig), 1) + //} + //for _, mut := range permuteResults(sig) { + // r.InitDFS(mut) + // _ = addEdge(r.sigGraph, r.hash(mut), r.hash(sig), 1) + //} + for _, mut := range weakenParams(sig) { + r.InitDFS(mut, depthTTL-1) + _ = addEdge(r.sigGraph, r.hash(sig), r.hash(mut), 3) + } + for _, mut := range weakenResults(sig) { + r.InitDFS(mut, depthTTL-1) + _ = addEdge(r.sigGraph, r.hash(mut), r.hash(sig), 2) + } +} + +// InitFloydWarshall refresh distCache by applying Floyd-Warshall algorithm to sigGraph. +func (r *SigGraphRanker) InitFloydWarshall(numWorkers int) { + r.distCache = make(map[lo.Tuple2[SigStr, SigStr]]int) + // It would be stupid to lock the whole distCache... Wanna use map[SigStr]map[SigStr]int now. TODO + + g := r.sigGraph + adj, err := g.AdjacencyMap() + if err != nil { + panic("InitFloydWarshall .AdjacencyMap(): " + err.Error()) + } + + vertices, err := g.Vertices() + if err != nil { + panic("InitFloydWarshall .Vertices(): " + err.Error()) + } + vertexIndex := make(map[SigStr]int) + for i, v := range vertices { + vertexIndex[v] = i + } + + const inf = math.MaxInt + // 空间大点大点吧，访问快 + 本算法里不用锁 + dist := make([][]int, len(vertices)) + for i := 0; i < len(dist); i++ { + dist[i] = make([]int, len(vertices)) + for j := 0; j < len(dist[i]); j++ { + if i != j { + dist[i][j] = inf + } + } + } + + for u, es := range adj { + i := vertexIndex[u] + for v, e := range es { + j := vertexIndex[v] + dist[i][j] = e.Properties.Weight + } + } + + var wg sync.WaitGroup + chunks := len(vertices) / numWorkers + for ki, k := range vertices { + hlog.Infof("%d/%d k=%s", ki+1, len(vertices), k) + wg.Add(numWorkers) + for worker := 0; worker < numWorkers; worker++ { + go func(worker int) { + defer wg.Done() + start := worker * chunks + end := lo.Ternary(worker == numWorkers-1, len(vertices), start+chunks) + + for i := start; i < end; i++ { + for j := range vertices { + if dIK, dKJ := dist[i][ki], dist[ki][j]; dIK != inf && dKJ != inf { + if dist[i][j] > dIK+dKJ { + dist[i][j] = dIK + dKJ + } + } + + } + } + }(worker) + } + wg.Wait() + } + for i := 0; i < len(dist); i++ { + for j := 0; j < len(dist[0]); j++ { + if dist[i][j] != inf { + r.distCache[lo.T2(vertices[i], vertices[j])] = dist[i][j] + } + } + } +} + +func (r *SigGraphRanker) InitFloydWarshallFromFile() { + // pre-requisite: the res/ is in the working directory. Shell: cd. GoLand: Edit Configurations. + // SEO: panic: open res/sigGraph.json: no such file or directory + // TODO elegant + path := "res/sigGraph.json" // generated by script/floydWrite.go + + hlog.Info("os.Args:") + lo.ForEach(os.Args, func(arg string, argI int) { + hlog.Info(argI, arg) + }) + if len(os.Args) >= 2 && os.Args[1] != "" && + os.Args[1] != "-test.v" /*for unit test in Goland*/ { + path = os.Args[1] + } + + bytes, err := os.ReadFile(path) + if err != nil { + panic(err) + } + dist := make(map[SigStr]map[SigStr]int) + err = json.Unmarshal(bytes, &dist) + if err != nil { + panic(err) + } + + for u, es := range dist { + for v, d := range es { + r.distCache[lo.T2(u, v)] = d + } + } +} + +func (r *SigGraphRanker) MarshalDistCache() []byte { + // map map to map + m := make(map[SigStr]map[SigStr]int) + for uv, d := range r.distCache { + if _, ok := m[uv.A]; !ok { + m[uv.A] = make(map[SigStr]int) + } + m[uv.A][uv.B] = d + } + + j, err := json.Marshal(m) + if err != nil { + panic("MarshalDistCache err=" + err.Error()) + } + return j + +} + +func (r *SigGraphRanker) UnmarshalDistCache(j []byte) { + +} + +func (r *SigGraphRanker) Distance(src, tar *types.Signature) int { + return dist(r.sigGraph, r.hash(src), r.hash(tar)) +} + +func (r *SigGraphRanker) DistanceWithCache(src, tar *types.Signature) int { + key := lo.T2(r.hash(src), r.hash(tar)) + if d, ok := r.distCache[key]; ok { + return d + } + r.distCache[key] = dist(r.sigGraph, r.hash(src), r.hash(tar)) + return r.distCache[key] +} + +func (r *SigGraphRanker) DistanceWithFloydWarshall(src, tar *types.Signature) int { + key := lo.T2(r.hash(src), r.hash(tar)) + if d, ok := r.distCache[key]; ok { + return d + } + return math.MaxInt +} + +//func permuteParams(sig *types.Signature) []*types.Signature { +// return nil +//} + +// (x1,x2,x3) -> Y ~~> +// [(x2,x3) -> Y, (x1,x3) -> Y, (x2,x3) -> Y] +func weakenParams(sig *types.Signature) []*types.Signature { + rst := make([]*types.Signature, sig.Params().Len()) + for i := 0; i < sig.Params().Len(); i++ { + newParamsSlice := u.TupleToSliceOfVarExcept(sig.Params(), i) + newParams := types.NewTuple(newParamsSlice...) + + newVariadic := sig.Variadic() + if i == sig.Params().Len()-1 { + newVariadic = false // otherwise: panic: got int, want variadic parameter with unnamed slice type or string as core type + } + + rst[i] = types.NewSignatureType(sig.Recv(), + u.CopyTypeParams(u.TypeParamListToSliceOfTypeParam(sig.RecvTypeParams())), + u.CopyTypeParams(u.TypeParamListToSliceOfTypeParam(sig.TypeParams())), // if not copy -> panic: type parameter bound more than once + newParams, + sig.Results(), + newVariadic) + } + return rst +} + +// X -> (y1,y2,y3) ~~> +// [X -> (y2,y3), X -> (y1,y3), X -> (y2,y3)] +func weakenResults(sig *types.Signature) []*types.Signature { + rst := make([]*types.Signature, sig.Results().Len()) + for i := 0; i < sig.Results().Len(); i++ { + newResultSlice := u.TupleToSliceOfVarExcept(sig.Results(), i) + newResult := types.NewTuple(newResultSlice...) + rst[i] = types.NewSignatureType(sig.Recv(), + u.CopyTypeParams(u.TypeParamListToSliceOfTypeParam(sig.RecvTypeParams())), + u.CopyTypeParams(u.TypeParamListToSliceOfTypeParam(sig.TypeParams())), // if not copy -> panic: type parameter bound more than once + sig.Params(), + newResult, + sig.Variadic()) + } + return rst +} + +func addEdge[K comparable, T any](g graph.Graph[K, T], src, tar K, weight int) error { + return g.AddEdge(src, tar, graph.EdgeWeight(weight), graph.EdgeAttribute("label", strconv.Itoa(weight))) +} + +func addRevE[K comparable, T any](g graph.Graph[K, T], tar, src K, weight int) error { + return addEdge(g, src, tar, weight) +} + +// if src==tar, 0; +// if not reachable, math.MaxInt; +// else, sum(weight) of the shortest path. +func dist[K comparable, T any](g graph.Graph[K, T], src, tar K) int { + d := math.MaxInt + path, err := graph.ShortestPath(g, src, tar) + + if err == nil { + d = 0 + for i := range path[:len(path)-1] { + edge, err2 := g.Edge(path[i], path[i+1]) + if err2 != nil { + // should not happen TODO + continue + } else { + d += edge.Properties.Weight + } + } + } + return d +} + +// Rank by distance +// TODO remove candidates param +func (r *SigGraphRanker) Rank(query *types.Signature, candidates []u.T2) []u.T2 { + if query == nil { + panic("Rank query == nil") + } + query = u.Anonymize(query) + + result := make([]u.T2, len(candidates)) + copy(result, candidates) + for i := 0; i < len(result); i++ { + result[i].A = u.Anonymize(result[i].A) + } + //less := func(i, j int) bool { + // return distance(query, result[i].A) < distance(query, result[j].A) + //} + + //// debug + //for i, candidate := range result { + // hlog.Debugf("Distance!%d %d %s", i, r.Distance(query, candidate.A), candidate.A) + //} + + less := func(i, j int) bool { + return r.DistanceWithFloydWarshall(query, result[i].A) < r.DistanceWithFloydWarshall(query, result[j].A) + } + sort.Slice(result, less) + return result +} + +// distance src -> dst +// distance(s,r) may != distance(r,s) +// TODO memoize +// TODO buggy: bind more than once. +func distance(src, dst *types.Signature) int64 { + if eq := types.IdenticalIgnoreTags(src, dst); eq { + return 0 + } + + //permute type params + srcSortTparams := signatureWithNewTypeParamList(src, sortTypeParamList(src.TypeParams())) + dstSortTparams := signatureWithNewTypeParamList(dst, sortTypeParamList(dst.TypeParams())) + if eq := types.IdenticalIgnoreTags(srcSortTparams, dstSortTparams); eq { + return 1 + } + + // permute params + //src.Params() + + // others. + return 114514 +} + +func sortTypeParamList(inp *types.TypeParamList) []*types.TypeParam { + out := u.TypeParamListToSliceOfTypeParam(inp) + sort.SliceStable(out, func(i, j int) bool { + return out[i].Obj().Name() < out[j].Obj().Name() + }) + return out +} + +// very immutable! very purely functional +func signatureWithNewTypeParamList(inp *types.Signature, tparams []*types.TypeParam) *types.Signature { + return types.NewSignatureType(inp.Recv(), + u.CopyTypeParams(u.TypeParamListToSliceOfTypeParam(inp.RecvTypeParams())), // TODO no copy but no panic? Oh I have not bound (rebind) them for the first time. + u.CopyTypeParams(tparams), // if not copy -> panic: type parameter bound more than once + inp.Params(), + inp.Results(), + inp.Variadic()) +} diff --git a/ranking/sigGraph_test.go b/ranking/sigGraph_test.go new file mode 100644 index 0000000..b5610f1 --- /dev/null +++ b/ranking/sigGraph_test.go @@ -0,0 +1,345 @@ +package ranking + +import ( + "go/token" + "go/types" + "math" + "os" + "testing" + + //"github.com/SnowOnion/graph" + //"github.com/SnowOnion/graph/draw" + + "github.com/dominikbraun/graph" + "github.com/dominikbraun/graph/draw" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + + "github.com/SnowOnion/godoogle/collect" + "github.com/SnowOnion/godoogle/u" +) + +func TestTrySortTypeParams(t *testing.T) { + inps := []string{ + //"[T comparable] func([]T)T", + //"func()", + //"func(string)", + //"func(int32, int) int", + //"func(string,...interface{})", + //"func (format string, a ...any) (n int, err error)", + //"[T any] func(T)", + //"[T comparable] func(T)", + ////"[a,b any] func (col []a, iter func(it a) b) (r1 []b)", // hsMap + ////"[a,b any] func (col []a, iter func(it a) b) []b", // hsMap + //"[a,b any] func([]a, func(a) b) []b", // hsMap + ////"[a,b any] func(col []a, iter func(it a, idx int) b) (r1 []b)", // lo.Map + ////"[a,b any] func(col []a, iter func(it a, idx int) b) []b", // lo.Map + "[b,a any] func([]a, func(a, int) b) []b", // lo.Map, flip type params + "[a,b any] func([]a, func(a, int) b) []b", // lo.Map, + + } + + for i, inp := range inps { + sig, err := u.Dummy(inp) + t.Log(i, err) + t.Log(i, inp) + t.Log(i, sig) + + t.Log(i, sig.TypeParams()) + //t.Log(i, out) + //assert.Nil(t, err, "case~~~~ %d", i) + //assert.True(t, types.IdenticalIgnoreTags(out, sig), "case~~~~ %d", i) + } +} + +func TestTryDominikbraunGraph(t *testing.T) { + /* + g := graph.New(graph.IntHash, graph.Directed(), graph.Acyclic(), graph.Weighted()) + + _ = g.AddVertex(1) + _ = g.AddVertex(2) + _ = g.AddVertex(3) + _ = g.AddVertex(4) + + _ = g.AddEdge(1, 2, graph.EdgeWeight(42)) + _ = g.AddEdge(1, 3) + _ = g.AddEdge(2, 3) + _ = g.AddEdge(2, 4) + _ = g.AddEdge(3, 4) + + file, _ := os.Create("./simple.gv") + _ = draw.DOT(g, file) + //_ = draw.DOT(g, file, draw.GraphAttribute("label", "my-graph")) + // Then: + // dot -Tsvg -Kneato -O simple.gv && open simple.gv.svg -a firefox + + */ + + hash := func(sig *types.Signature) string { return sig.String() } + g2 := graph.New(hash, graph.Directed(), graph.Acyclic(), graph.Weighted()) + + s := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("string").Type()) + i := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("int").Type()) + e := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("error").Type()) + // v for void + sie := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(i, e), false) + sei := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(e, i), false) + vie := types.NewSignatureType(nil, nil, nil, types.NewTuple(), types.NewTuple(i, e), false) + vei := types.NewSignatureType(nil, nil, nil, types.NewTuple(), types.NewTuple(e, i), false) + si := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(i), false) + se := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(e), false) + vi := types.NewSignatureType(nil, nil, nil, types.NewTuple(), types.NewTuple(i), false) + ve := types.NewSignatureType(nil, nil, nil, types.NewTuple(), types.NewTuple(e), false) + sv := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(), false) + vv := types.NewSignatureType(nil, nil, nil, types.NewTuple(), types.NewTuple(), false) + /* + func(string) (int, error) + func(string) (error, int) + func() (int, error) + func() (error, int) + func(string) int + func(string) error + func() int + func() error + func(string) + func() + */ + + for _, t := range []*types.Signature{sie, sei, vie, vei, si, se, vi, ve, sv, vv} { + _ = g2.AddVertex(t) + //fmt.Println(t.String()) + } + + // 为啥 AddEdge 不接受 T 而是 K，要我自己调用 hash 啊，乌鱼子 // TODO PR func AddEdgeT + // 大致是 BFS 地添加…… 但有的边是反的 + // TODO 哎，Go 可以 in-place 修改。那么参数可以既是输入又是输出。…… + _ = addEdge(g2, hash(sie), hash(sei), 1) // (PR) + _ = addEdge(g2, hash(sie), hash(vie), 3) // (WP) + _ = addRevE(g2, hash(sie), hash(se), 2) // (WR) + _ = addRevE(g2, hash(sie), hash(si), 2) // (WR) + _ = addEdge(g2, hash(sei), hash(sie), 1) // (PR) + _ = addEdge(g2, hash(sei), hash(vei), 3) // (WP) + _ = addRevE(g2, hash(sei), hash(si), 2) // (WR) + _ = addRevE(g2, hash(sei), hash(se), 2) // (WR) + _ = addEdge(g2, hash(vie), hash(vei), 1) // (PR) + _ = addRevE(g2, hash(vie), hash(ve), 2) // (WR) + _ = addRevE(g2, hash(vie), hash(vi), 2) // (WR) + _ = addEdge(g2, hash(se), hash(ve), 3) // (WP) + _ = addRevE(g2, hash(se), hash(sv), 2) // (WR) + _ = addEdge(g2, hash(si), hash(vi), 3) // (WP) + _ = addRevE(g2, hash(si), hash(sv), 2) // (WR) + _ = addEdge(g2, hash(vei), hash(vie), 1) // (PR) + _ = addRevE(g2, hash(vei), hash(vi), 2) // (WR) + _ = addRevE(g2, hash(vei), hash(ve), 2) // (WR) + _ = addRevE(g2, hash(vi), hash(vv), 2) // (WR) + _ = addRevE(g2, hash(ve), hash(vv), 2) // (WR) + _ = addEdge(g2, hash(sv), hash(vv), 3) // (WP) + + file2, _ := os.Create("./sigGraph.gv") + _ = draw.DOT(g2, file2) + t.Log("dot -Tsvg -O sigGraph.gv && open sigGraph.gv.svg -a firefox") + + // why not returning the sum(weight) together with the path? TODO + path, err := graph.ShortestPath(g2, hash(vv), hash(sie)) + //path, err := graph.ShortestPath(g2, hash(sie), hash(vv)) + + t.Log(err, dist(g2, hash(vv), hash(sie)), path) + assert.Equal(t, 4, dist(g2, hash(sv), hash(sie))) + assert.Equal(t, 0, dist(g2, hash(vv), hash(vv))) + assert.Equal(t, math.MaxInt, dist(g2, hash(sie), hash(vv))) + assert.Equal(t, 1, dist(g2, hash(sei), hash(sie))) + assert.Equal(t, 1, dist(g2, hash(sie), hash(sei))) +} + +func TestWeakenResults(t *testing.T) { + s := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("string").Type()) + i := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("int").Type()) + e := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("error").Type()) + + sie := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(i, e), false) + si := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(i), false) + se := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(e), false) + + mutants := weakenResults(sie) + assert.Equal(t, 2, len(mutants)) + for _, m := range mutants { + t.Log(m.String()) + } + assert.True(t, types.IdenticalIgnoreTags(se, mutants[0])) + assert.True(t, types.IdenticalIgnoreTags(si, mutants[1])) +} + +func TestWeakenParams(t *testing.T) { + s := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("string").Type()) + i := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("int").Type()) + e := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("error").Type()) + + sie := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(i, e), false) + vie := types.NewSignatureType(nil, nil, nil, types.NewTuple(), types.NewTuple(i, e), false) + + mutants := weakenParams(sie) + assert.Equal(t, 1, len(mutants)) + for _, m := range mutants { + t.Log(m.String()) + } + assert.True(t, types.IdenticalIgnoreTags(vie, mutants[0])) +} + +// Test only the preparation, not the ranking +func TestNewSigGraphRanker(t *testing.T) { + s := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("string").Type()) + i := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("int").Type()) + e := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("error").Type()) + + sie := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(i, e), false) + + expectedVertices := u.SliceToSet([]SigStr{ + "func() (int, error)", + "func() error", + "func()", + "func() int", + "func(string) error", + "func(string)", + "func(string) int", + "func(string) (int, error)", + }...) + type v2d1 struct { + s SigStr + t SigStr + d int + } + expectedEdges := u.SliceToSet([]v2d1{ + {s: `func() int`, t: `func() (int, error)`, d: 2}, + {s: `func(string) (int, error)`, t: `func() (int, error)`, d: 3}, + {s: `func(string) error`, t: `func() error`, d: 3}, + {s: `func(string) error`, t: `func(string) (int, error)`, d: 2}, + {s: `func(string)`, t: `func()`, d: 3}, + {s: `func(string)`, t: `func(string) error`, d: 2}, + {s: `func(string)`, t: `func(string) int`, d: 2}, + {s: `func(string) int`, t: `func() int`, d: 3}, + {s: `func(string) int`, t: `func(string) (int, error)`, d: 2}, + {s: `func()`, t: `func() error`, d: 2}, + {s: `func()`, t: `func() int`, d: 2}, + {s: `func() error`, t: `func() (int, error)`, d: 2}, + }...) + + expectedPaths := u.SliceToSet([]v2d1{ + {s: `func(string) error`, t: `func(string) error`, d: 0}, + {s: `func(string)`, t: `func(string)`, d: 0}, + {s: `func(string) int`, t: `func(string) int`, d: 0}, + {s: `func(string) (int, error)`, t: `func(string) (int, error)`, d: 0}, + {s: `func() (int, error)`, t: `func() (int, error)`, d: 0}, + {s: `func() error`, t: `func() error`, d: 0}, + {s: `func()`, t: `func()`, d: 0}, + {s: `func() int`, t: `func() int`, d: 0}, + {s: `func() error`, t: `func() (int, error)`, d: 2}, + {s: `func(string) error`, t: `func(string) (int, error)`, d: 2}, + {s: `func(string)`, t: `func(string) int`, d: 2}, + {s: `func(string)`, t: `func() (int, error)`, d: 7}, + {s: `func(string)`, t: `func() error`, d: 5}, + {s: `func() int`, t: `func() (int, error)`, d: 2}, + {s: `func(string) error`, t: `func() error`, d: 3}, + {s: `func(string)`, t: `func(string) error`, d: 2}, + {s: `func(string) int`, t: `func() int`, d: 3}, + {s: `func(string)`, t: `func() int`, d: 5}, + {s: `func(string)`, t: `func()`, d: 3}, + {s: `func(string) (int, error)`, t: `func() (int, error)`, d: 3}, + {s: `func()`, t: `func() error`, d: 2}, + {s: `func()`, t: `func() int`, d: 2}, + {s: `func(string) int`, t: `func(string) (int, error)`, d: 2}, + {s: `func(string)`, t: `func(string) (int, error)`, d: 4}, + {s: `func(string) int`, t: `func() (int, error)`, d: 5}, + {s: `func(string) error`, t: `func() (int, error)`, d: 5}, + {s: `func()`, t: `func() (int, error)`, d: 4}, + }...) + + candi := []u.T2{ + {sie, types.NewFunc(token.NoPos, nil, "", sie)}, + } + r := NewSigGraphRanker(candi, LoadFromFile(false)) + t.Log(r.sigGraph.Order()) + t.Log(r.sigGraph.Size()) + file2, _ := os.Create("./sigGraph.gv") + _ = draw.DOT(r.sigGraph, file2) // then: dot -Tsvg -O sigGraph.gv && open sigGraph.gv.svg -a firefox + + V, _ := r.sigGraph.Order() + E, _ := r.sigGraph.Size() + assert.Equal(t, 8, V) + assert.Equal(t, 12, E) + + vertices, _ := r.sigGraph.Vertices() + assert.True(t, u.SliceToSet(vertices...).Equals(expectedVertices)) + //for _, v := range vertices { + // fmt.Printf("{s:`%s`,t:`%s`,d:%d},\n", v, v, 0) + //} + + edges, _ := r.sigGraph.Edges() + //for _, e := range edges { + // fmt.Printf("{s:`%s`,t:`%s`,d:%d},\n", e.Source, e.Target, e.Properties.Weight) + //} + edgeSet := u.SliceToSet(lo.Map(edges, func(e graph.Edge[SigStr], _ int) v2d1 { + return v2d1{s: e.Source, t: e.Target, d: e.Properties.Weight} + })...) + assert.True(t, edgeSet.Equals(expectedEdges)) + + //r.InitFloydWarshall(16) + //for uv, d := range r.distCache { + // //t.Log(d, uv.A, "->", uv.B) + // //fmt.Printf("{s:`%s`,t:`%s`,d:%d},\n", uv.A, uv.B, d) + //} + pathSet := u.SliceToSet(lo.MapToSlice(r.distCache, func(uv lo.Tuple2[SigStr, SigStr], d int) v2d1 { + return v2d1{s: uv.A, t: uv.B, d: d} + })...) + assert.True(t, pathSet.Equals(expectedPaths)) +} + +func BenchmarkDistance(b *testing.B) { + collect.InitFuncDatabase() + ranker := NewSigGraphRanker(collect.FuncDatabase, LoadFromFile(false)) + + s := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("string").Type()) + i := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("int").Type()) + e := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("error").Type()) + + sie := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(i, e), false) + si := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(i), false) + + for i := 0; i < b.N; i++ { + ranker.Distance(si, sie) + //b.Log() + } +} + +func BenchmarkDistanceWithCache(b *testing.B) { + collect.InitFuncDatabase() + ranker := NewSigGraphRanker(collect.FuncDatabase, LoadFromFile(false)) + + s := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("string").Type()) + i := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("int").Type()) + e := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("error").Type()) + + sie := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(i, e), false) + si := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(i), false) + + for i := 0; i < b.N; i++ { + ranker.DistanceWithCache(si, sie) + //b.Log() + } +} + +func BenchmarkDistanceWithFloydWarshall(b *testing.B) { + collect.InitFuncDatabase() + ranker := NewSigGraphRanker(collect.FuncDatabase, LoadFromFile(true)) + + s := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("string").Type()) + i := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("int").Type()) + e := types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("error").Type()) + + sie := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(i, e), false) + si := types.NewSignatureType(nil, nil, nil, types.NewTuple(s), types.NewTuple(i), false) + + for i := 0; i < b.N; i++ { + ranker.DistanceWithFloydWarshall(si, sie) + //b.Log() + } +} diff --git a/script/floydRead.go b/script/floydRead.go new file mode 100644 index 0000000..954a029 --- /dev/null +++ b/script/floydRead.go @@ -0,0 +1,34 @@ +package main + +import ( + "encoding/json" + "fmt" + _ "net/http/pprof" + "os" + + "github.com/SnowOnion/godoogle/ranking" +) + +func main() { + bytes, err := os.ReadFile("script/floyd-std-ttl-3-worker-10.json") + if err != nil { + panic(err) + } + dist := make(map[ranking.SigStr]map[ranking.SigStr]int) + err = json.Unmarshal(bytes, &dist) + if err != nil { + panic(err) + } + fmt.Println(len(dist)) + for _, sig := range []string{ + `func(string) (int, error)`, + `func(string) int`, + } { + fmt.Println() + fmt.Println(len(dist[sig]), "~~FROM~~", sig) + for k, v := range dist[sig] { + fmt.Println(k, v) + } + } + +} diff --git a/script/floydWrite.go b/script/floydWrite.go new file mode 100644 index 0000000..38a4674 --- /dev/null +++ b/script/floydWrite.go @@ -0,0 +1,28 @@ +package main + +import ( + "fmt" + "net/http" + _ "net/http/pprof" + "os" + "runtime" + + "github.com/SnowOnion/godoogle/collect" + "github.com/SnowOnion/godoogle/ranking" +) + +func main() { + // pprof + go func() { + fmt.Println(http.ListenAndServe("localhost:6061", nil)) + }() + + workers := max(runtime.NumCPU()-2, 1) + fmt.Println("workers:", workers) + + collect.InitFuncDatabase() + ranker := ranking.NewSigGraphRanker(collect.FuncDatabase, ranking.LoadFromFile(false)) + ranker.InitFloydWarshall(workers) + + os.WriteFile("sigGraph.json", ranker.MarshalDistCache(), 0666) +} diff --git a/script/parallel-floyd-gpt4o.go b/script/parallel-floyd-gpt4o.go new file mode 100644 index 0000000..b669a2c --- /dev/null +++ b/script/parallel-floyd-gpt4o.go @@ -0,0 +1,100 @@ +package main + +import ( + "fmt" + "math" + "slices" + "sync" +) + +// INF represents infinity +const INF = math.MaxInt + +// Initialize the graph with distances +func initializeGraph(V int) [][]int { + dist := make([][]int, V) + for i := 0; i < V; i++ { + dist[i] = make([]int, V) + for j := 0; j < V; j++ { + if i == j { + dist[i][j] = 0 + } else { + dist[i][j] = INF + } + } + } + return dist +} + +// ParallelFloydWarshall implements the Floyd-Warshall algorithm in parallel +func ParallelFloydWarshall(graph [][]int, V, numWorkers int) [][]int { + dist := make([][]int, V) + for i := range graph { + dist[i] = make([]int, V) + copy(dist[i], graph[i]) + } + + var wg sync.WaitGroup + chunks := V / numWorkers + + for k := 0; k < V; k++ { + wg.Add(numWorkers) + for worker := 0; worker < numWorkers; worker++ { + go func(worker int) { + defer wg.Done() + start := worker * chunks + end := start + chunks + if worker == numWorkers-1 { + end = V + } + for i := start; i < end; i++ { + for j := 0; j < V; j++ { + if dist[i][k] != INF && dist[k][j] != INF && dist[i][j] > dist[i][k]+dist[k][j] { + dist[i][j] = dist[i][k] + dist[k][j] + } + } + } + }(worker) + } + wg.Wait() + } + + return dist +} + +func main() { + V := 4 + numWorkers := 4 + + graph := initializeGraph(V) + graph[0][1] = 3 + graph[0][2] = 1 + graph[1][2] = 7 + graph[1][3] = 5 + graph[2][3] = 2 + + fmt.Println("Initial graph:") + for _, row := range graph { + fmt.Println(row) + } + + dist := ParallelFloydWarshall(graph, V, numWorkers) + + fmt.Println("Distance matrix after executing Floyd-Warshall:") + for _, row := range dist { + fmt.Println(row) + } + + // Below is written by me, not AI ( ﾟ∀。) + + dist1 := ParallelFloydWarshall(graph, V, 1) + fmt.Println("Distance matrix after executing Floyd-Warshall (serial):") + for _, row := range dist1 { + fmt.Println(row) + } + + eq := slices.EqualFunc(dist, dist1, func(row1 []int, row2 []int) bool { + return slices.Equal(row1, row2) + }) + fmt.Println(eq) +} diff --git a/server/.gitignore b/server/.gitignore new file mode 100644 index 0000000..13cd1fa --- /dev/null +++ b/server/.gitignore @@ -0,0 +1 @@ +server \ No newline at end of file diff --git a/server/handler.go b/server/handler.go index f1bdc1a..b5fb0f5 100644 --- a/server/handler.go +++ b/server/handler.go @@ -6,6 +6,7 @@ import ( "github.com/cloudwego/hertz/pkg/app" "github.com/cloudwego/hertz/pkg/common/hlog" "github.com/cloudwego/hertz/pkg/protocol/consts" + "github.com/hertz-contrib/requestid" "github.com/samber/lo" "github.com/SnowOnion/godoogle/server/model" @@ -38,8 +39,9 @@ func SearchH(ctx context.Context, c *app.RequestContext) { hlog.CtxErrorf(ctx, "service.Search err=%s", err) c.HTML(consts.StatusInternalServerError, "search.html", map[string]any{ - "q": req.Query, - "error": "Sorry, something is wrong with server. It’s not your fault.", + "q": req.Query, + "error": `Sorry, something is wrong with server. It’s not your fault. +Request ID: ` + requestid.Get(c), }) return } @@ -54,21 +56,22 @@ func SearchH(ctx context.Context, c *app.RequestContext) { func SearchJ(ctx context.Context, c *app.RequestContext) { hlog.CtxInfof(ctx, "Search invoked~") + requestID := requestid.Get(c) req := model.SearchReq{} err := c.BindAndValidate(&req) if err != nil { - c.JSON(consts.StatusBadRequest, model.Resp{Code: 400000, Message: "Bad Request"}) + c.JSON(consts.StatusBadRequest, model.Resp{Code: 400000, Message: "Bad Request", RequestID: requestID}) return } result, err := service.Search(nil, req) if err != nil { hlog.CtxErrorf(ctx, "service.Search err=%s", err) - c.JSON(consts.StatusInternalServerError, model.Resp{Code: 500000, Message: "Server Error"}) + c.JSON(consts.StatusInternalServerError, model.Resp{Code: 500000, Message: "Server Error", RequestID: requestID}) return } - c.JSON(consts.StatusOK, model.Resp{Data: result}) + c.JSON(consts.StatusOK, model.Resp{Data: result, RequestID: requestID}) return } diff --git a/server/main.go b/server/main.go index a37927e..7adcdcc 100644 --- a/server/main.go +++ b/server/main.go @@ -1,12 +1,15 @@ package main import ( - "github.com/SnowOnion/godoogle/collect" "github.com/cloudwego/hertz/pkg/app/server" "github.com/cloudwego/hertz/pkg/common/hlog" hertzlogrus "github.com/hertz-contrib/logger/logrus" + "github.com/hertz-contrib/pprof" "github.com/hertz-contrib/requestid" "github.com/sirupsen/logrus" + + "github.com/SnowOnion/godoogle/collect" + "github.com/SnowOnion/godoogle/ranking" ) func main() { @@ -20,6 +23,8 @@ func main() { h := server.Default(server.WithHostPorts("[::]:8888")) h.Use(requestid.New()) + pprof.Register(h) + h.LoadHTMLGlob("res/views/*") h.Static("/", "./res/assets") h.GET("/", Home) @@ -27,8 +32,13 @@ func main() { h.GET("/search", SearchH) // todo elegantly initialize; OO - hlog.Info("Start InitFuncDatabase...") + hlog.Info("Start initializing FuncDatabase and ranker...") collect.InitFuncDatabase() + ranking.DefaultRanker = ranking.NewSigGraphRanker( + collect.FuncDatabase, // = =、TODO be elegant! + ranking.LoadFromFile(true), + ) + hlog.Info("End initializing FuncDatabase and ranker!") hlog.Info("Start serving...") h.Spin() diff --git a/server/model/reqresp.go b/server/model/reqresp.go index 786c743..ba7fdb9 100644 --- a/server/model/reqresp.go +++ b/server/model/reqresp.go @@ -1,9 +1,10 @@ package model type Resp struct { - Code int `json:"code"` - Message string `json:"message"` - Data any `json:"data"` + Code int `json:"code"` + Message string `json:"message"` + Data any `json:"data"` + RequestID string `json:"request_id"` } type SearchReq struct { @@ -23,4 +24,5 @@ type ResultItem struct { Pkg string `json:"pkg"` URL string `json:"url"` Signature string `json:"signature"` + //Distance int `json:"distance"` // from query to this result // u.T2 扩展性不如 struct，不太好传出来…… TODO } diff --git a/server/res/tailor.py b/server/res/tailor.py new file mode 100755 index 0000000..8678cb8 --- /dev/null +++ b/server/res/tailor.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 -i + +""" +func[a any](bool, a, a) a +func[a any](bool, a, a) a +""" + +import json +from pprint import pp + +with open("floyd.json", "r") as fi: + j=json.load(fi) +ter_sig="func[T any](bool, T, T) T" +ternary=j[ter_sig] + +j_only_ternary={ter_sig:j[ter_sig]} + +with open("ter.json","w") as fo: + json.dump(j_only_ternary,fo) diff --git a/server/res/views/search.html b/server/res/views/search.html index 6965155..b70bb34 100644 --- a/server/res/views/search.html +++ b/server/res/views/search.html @@ -9,15 +9,15 @@ Godoogle - -

Welcome to Godoogle

+ +

Godoogle

- +
- {{ if .error }}

{{.error}}

@@ -41,9 +41,12 @@

Welcome to Godoogle

diff --git a/server/res/views/welcome_example.tmpl.html b/server/res/views/welcome_example.tmpl.html index 8ddf507..92f3df0 100644 --- a/server/res/views/welcome_example.tmpl.html +++ b/server/res/views/welcome_example.tmpl.html @@ -1,20 +1,27 @@ {{ define "welcome_example" }}

Godoogle is a Go API search engine, which (by now) allows you to search by function type signature, including generic functions.

+ href="https://go.dev/doc/tutorial/generics" target="_blank">generic functions.

Example Searches:

diff --git a/server/service/search_test.go b/server/service/search_test.go index 5066a5f..befdfa1 100644 --- a/server/service/search_test.go +++ b/server/service/search_test.go @@ -2,32 +2,65 @@ package service import ( "context" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/SnowOnion/godoogle/collect" + "github.com/SnowOnion/godoogle/ranking" "github.com/SnowOnion/godoogle/server/model" - "github.com/stretchr/testify/assert" - "testing" ) +// TestSearch test whether all expected answers appear in top-10 results. func TestSearch(t *testing.T) { collect.InitFuncDatabase() + ranking.DefaultRanker = ranking.NewSigGraphRanker(collect.FuncDatabase, ranking.LoadFromFile(true)) // = =、TODO be elegant! - //q := `[a,b any] func([]a, func(a, int) b) []b` - //q := `func(sort.Interface)` - //q := `func(n int, cmp func(int) int) (i int, found bool)` - //q := `[T any] func(bool, T, T) T` - //q := `[T any] func(bool, func() T, func() T) T` - - qs := []string{ - `[T any] func (f func() T) <-chan T`, - `[T any] func(collection []T, size int) [][]T`, + qna := map[string][]string{ + `func(sort.Interface)`: {`sort.Sort`, `sort.Stable`}, + `func(n int, cmp func(int) int) (i int, found bool)`: {`sort.Find`}, + `[T any] func(bool, T, T) T`: {`github.com/samber/lo.Ternary`}, + `[T any] func(bool, func() T, func() T) T`: {`github.com/samber/lo.TernaryF`}, + `[a,b any] func([]a, func(a, int) b) []b`: {`github.com/samber/lo.Map`}, + `[a any, b any] func([]a, func(a, int) b) []b`: {`github.com/samber/lo.Map`}, // how do you turn this on? + `func (string) int`: {`unicode/utf8.RuneCountInString`, `strconv.Atoi`}, + `func (string) (int, error)`: {`strconv.Atoi`}, + `[T any] func (f func() T) <-chan T`: {`github.com/samber/lo.Async`, `github.com/samber/lo.Async1`}, + } + // Collect known bad case here. + qnaFail := map[string][]string{ + `[T any] func(collection []T, size int) [][]T`: {`github.com/samber/lo.Chunk`}, // lo.Chunk changed! func[T any, Slice ~[]T](collection Slice, size int) []Slice + `[b,a any] func([]a, func(a, int) b) []b`: {`github.com/samber/lo.Map`}, // consider sortTypeParamList. 函統网, UniSig. } + ctx := context.Background() - for _, q := range qs { + for q, as := range qna { resp, err := Search(ctx, model.SearchReq{Query: q}) assert.Nil(t, err) - t.Log(len(resp.Result)) - for _, r := range resp.Result[:5] { - t.Log(r) + + top10 := resp.Result[:min(10, len(resp.Result))] + //for _, r := range top10 { + // t.Log(r) + //} + for _, a := range as { + assert.True(t, slices.ContainsFunc(top10, func(r model.ResultItem) bool { + return r.FullName == a + })) + } + } + for q, as := range qnaFail { + resp, err := Search(ctx, model.SearchReq{Query: q}) + assert.Nil(t, err) + + top10 := resp.Result[:min(10, len(resp.Result))] + //for _, r := range top10 { + // t.Log(r) + //} + for _, a := range as { + assert.False(t, slices.ContainsFunc(top10, func(r model.ResultItem) bool { + return r.FullName == a + })) } } diff --git a/server/service/service.go b/server/service/service.go index 42ec65f..d8d46ff 100644 --- a/server/service/service.go +++ b/server/service/service.go @@ -3,23 +3,26 @@ package service import ( "context" "fmt" + + "github.com/cloudwego/hertz/pkg/common/hlog" + "github.com/samber/lo" + "github.com/SnowOnion/godoogle/collect" + "github.com/SnowOnion/godoogle/ranking" "github.com/SnowOnion/godoogle/server/model" "github.com/SnowOnion/godoogle/u" - "github.com/cloudwego/hertz/pkg/common/hlog" - "github.com/samber/lo" ) func Search(ctx context.Context, req model.SearchReq) (model.SearchResp, error) { hlog.CtxInfof(ctx, "Search %s", req) // todo security concern - inpSig, err := lo.T2(collect.Dummy(req.Query)).Unpack() + inpSig, err := lo.T2(u.Dummy(req.Query)).Unpack() if err != nil { hlog.CtxErrorf(ctx, "Dummy err=%s", err) - // TODO on error, print request_id to user return model.SearchResp{}, fmt.Errorf("error parsing query") } - result := lo.Map(collect.NaiveRanker.Rank(inpSig, collect.FuncDatabase), + ranked := ranking.DefaultRanker.Rank(inpSig, collect.FuncDatabase) + result := lo.Map(ranked, func(sigDecl u.T2, ind int) model.ResultItem { name := sigDecl.B.Name() pkg := sigDecl.B.Pkg().Path() @@ -35,7 +38,7 @@ func Search(ctx context.Context, req model.SearchReq) (model.SearchResp, error) FullName: sigDecl.B.FullName(), Pkg: pkg, URL: url, - Signature: sigDecl.A.String(), + Signature: sigDecl.B.Signature().String(), // show signature before param anonymizing and type param renaming } }) diff --git a/u/basic.go b/u/basic.go new file mode 100644 index 0000000..ade248b --- /dev/null +++ b/u/basic.go @@ -0,0 +1,35 @@ +package u + +import "maps" + +// KsVs returns []K, []V in corresponding order. +func KsVs[K comparable, V any](m map[K]V) ([]K, []V) { + ks := make([]K, len(m)) + vs := make([]V, len(m)) + i := 0 + for k, v := range m { + ks[i] = k + vs[i] = v + } + return ks, vs +} + +type Set[T comparable] map[T]struct{} + +func SliceToSet[T comparable](xs ...T) Set[T] { + s := make(Set[T]) + e := struct{}{} + for _, x := range xs { + s[x] = e + } + return s +} + +func (s Set[T]) Contains(x T) bool { + _, ok := s[x] + return ok +} + +func (s Set[T]) Equals(t Set[T]) bool { + return maps.Equal(s, t) +} diff --git a/u/biz.go b/u/biz.go new file mode 100644 index 0000000..52d5780 --- /dev/null +++ b/u/biz.go @@ -0,0 +1,16 @@ +package u + +import ( + "go/types" + + "github.com/samber/lo" +) + +type T2 lo.Tuple2[*types.Signature, *types.Func] + +//type T2T lo.Tuple2[*types.Signature, *ast.GenDecl] + +// for better debugging +func (t T2) String() string { + return t.B.FullName() + " :: " + t.A.String() +} diff --git a/u/candi.go b/u/candi.go new file mode 100644 index 0000000..df166fa --- /dev/null +++ b/u/candi.go @@ -0,0 +1,177 @@ +package u + +import ( + "errors" + "fmt" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" +) + +// Dummy rawQuery e.g. +// "func(rune) bool" +// "[T, R any, K comparable] func (collection []T, iteratee func(item T, index int) R) []R" +func Dummy(rawQuery string) (*types.Signature, error) { + augmentedQuery := `package dummy +import ( + //"golang.org/x/exp/constraints" + "sort" + "time" +) +type dummy ` + rawQuery + sigs, err := ParseGenDeclTypeSpecFuncSigs(augmentedQuery) + if err != nil { + return nil, err + } + if len(sigs) == 0 { + return nil, errors.New("no type signature in augmentedQuery") + } + return sigs[0], nil +} + +// Naive +func ParseGenDeclTypeSpecFuncSigs(src string) (sigs []*types.Signature, err error) { + // 创建一个新的 token 文件集，用于语法解析。 + fset := token.NewFileSet() + + // 解析 Go 源代码字符串。 + file, err := parser.ParseFile(fset, "", src, parser.AllErrors) + if err != nil { + return nil, err + } + + // 创建类型信息配置并初始化一个新的类型检查器。 + config := &types.Config{ + Importer: importer.Default(), + DisableUnusedImportCheck: true, + IgnoreFuncBodies: true, + + //types.DefaultImporter(), // gpt-4 幻觉 or 旧版？2024-03-06 01:49:43 + } + info := &types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Instances: make(map[*ast.Ident]types.Instance), // 感觉这个对 signature.tparams 比较重要…… + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + Implicits: make(map[ast.Node]types.Object), + Selections: make(map[*ast.SelectorExpr]*types.Selection), + Scopes: make(map[ast.Node]*types.Scope), + InitOrder: make([]*types.Initializer, 0), + } + + // 对 AST 进行类型检查，填充 info。 + _, err = config.Check("", fset, []*ast.File{file}, info) + if err != nil { + return nil, err + } + + // 遍历所有的顶级声明。 + for _, decl := range file.Decls { + // + genDecl, ok := decl.(*ast.GenDecl) + if !ok { + continue + } + + // 遍历一般声明中的规格（Specs）。 + for _, spec := range genDecl.Specs { + // 确保规格是类型规格。 + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + + // 查找类型名称对应的对象。 + obj := info.Defs[typeSpec.Name] + if obj == nil { + continue + } + + //typeObj, ok := obj.(*types.Named) + // 确保对象是类型对象。 + + typeObj, ok := obj.(*types.TypeName) + if !ok { + continue + } + + //if typeParam, ok := typeObj.Type().Underlying().(*types.TypeParam); ok { + // fmt.Println("typeParam", typeParam) + // + //} + + // 获取类型对象的类型并断言为 *types.Signature。 + if signature, ok := typeObj.Type().Underlying().(*types.Signature); ok { + //fmt.Printf("Function type %s: %s\n", typeObj.Name(), signature) + + // todo support method + recvTypeParams0 := signature.RecvTypeParams() + recvTypeParams := make([]*types.TypeParam, recvTypeParams0.Len()) + for i := 0; i < recvTypeParams0.Len(); i++ { + recvTypeParams[i] = recvTypeParams0.At(i) + } + + typeParams := make([]*types.TypeParam, 0) + nameToTypeParam := make(map[string]*types.TypeParam) // oldName -> TypeParamWithNewName + // 填补 signature.TypeParams() 的空白 + // 大概也可以提取到「for _, decl := range file.Decls」外面，复用 u.Anonymize + + //fmt.Println("typeSpec.TypeParams", typeSpec.TypeParams) + if typeSpec.TypeParams != nil { + typeParamID := 0 // 类型参数换名 1 + for _, tp := range typeSpec.TypeParams.List { + //fmt.Println(tp) + for _, tpn := range tp.Names { + obj := info.Defs[tpn] + if obj == nil { + continue + } + + if tn, ok := obj.Type().(*types.TypeParam); ok { + //fmt.Println("tnnnnnnn", tn) + //typeParams = append(typeParams, tn) //panic: type parameter bound more than once + newTP := types.NewTypeParam(types.NewTypeName( + tn.Obj().Pos(), tn.Obj().Pkg() /*tn.Obj().Name()*/, fmt.Sprintf("_T%d", typeParamID), tn.Obj().Type()), + tn.Constraint()) + typeParams = append(typeParams, newTP) + //nameToTypeParam[newTP.Obj().Name()] = newTP + nameToTypeParam[tn.Obj().Name()] = newTP + + typeParamID++ + } + + } + + } + } + + params, err := RebindVars(signature.Params(), nameToTypeParam, "params of "+signature.String()) + if err != nil { + return sigs, fmt.Errorf("rebinding params of %s: %w", signature, err) + } + results, err := RebindVars(signature.Results(), nameToTypeParam, "results of "+signature.String()) + if err != nil { + return sigs, fmt.Errorf("rebinding results of %s: %w", signature, err) + } + + sigWithTypeParams := types.NewSignatureType( + signature.Recv(), + recvTypeParams, + typeParams, //! + types.NewTuple(params...), + types.NewTuple(results...), + signature.Variadic(), + ) + //panic: type parameter bound more than once + //考虑让用户输入带函数名了。至少，泛型的情况。 // 我还得费劲加函数体啊，别了 + + sigs = append(sigs, sigWithTypeParams) + } + } + + } + + return sigs, nil +} diff --git a/u/candi_test.go b/u/candi_test.go new file mode 100644 index 0000000..f64efd9 --- /dev/null +++ b/u/candi_test.go @@ -0,0 +1,157 @@ +package u + +import ( + "go/token" + "go/types" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDummy(t *testing.T) { + // TODO more, to cover rebind* + inps := []string{ + "[T comparable] func([]T)T", + "func()", + "func(string)", + "func(int32, int) int", + "func(string,...interface{})", + "func (format string, a ...any) (n int, err error)", + "[T any] func(T)", + "[T comparable] func(T)", + //"[a,b any] func (col []a, iter func(it a) b) (r1 []b)", // hsMap + //"[a,b any] func (col []a, iter func(it a) b) []b", // hsMap + "[a,b any] func([]a, func(a) b) []b", // hsMap + //"[a,b any] func(col []a, iter func(it a, idx int) b) (r1 []b)", // lo.Map + //"[a,b any] func(col []a, iter func(it a, idx int) b) []b", // lo.Map + "[a,b any] func([]a, func(a, int) b) []b", // lo.Map, + "[S ~[]E, E constraints.Ordered] func(x S)", // https://pkg.go.dev/golang.org/x/exp/slices#Sort + "[E constraints.Ordered] func(x []E)", + } + + t1Any := types.NewTypeParam( + types.NewTypeName(token.NoPos, nil, "T1", nil /*这里？TODO*/), + types.Universe.Lookup("any").Type()) + t1Comparable := types.NewTypeParam( + types.NewTypeName(token.NoPos, nil, "T1", nil), + types.Universe.Lookup("comparable").Type()) + t1Comparable2 := types.NewTypeParam( + types.NewTypeName(token.NoPos, nil, "T1", nil), + types.Universe.Lookup("comparable").Type()) + t1Any2 := types.NewTypeParam( + types.NewTypeName(token.NoPos, nil, "T1", nil /*这里？TODO*/), + types.Universe.Lookup("any").Type()) + t2Any2 := types.NewTypeParam( + types.NewTypeName(token.NoPos, nil, "T1", nil /*这里？TODO*/), + types.Universe.Lookup("any").Type()) + t1Any3 := types.NewTypeParam( + types.NewTypeName(token.NoPos, nil, "T1", nil /*这里？TODO*/), + types.Universe.Lookup("any").Type()) + t2Any3 := types.NewTypeParam( + types.NewTypeName(token.NoPos, nil, "T1", nil /*这里？TODO*/), + types.Universe.Lookup("any").Type()) + + outputs := []*types.Signature{ + types.NewSignatureType(nil, nil, + []*types.TypeParam{t1Comparable2}, + types.NewTuple(types.NewVar(token.NoPos, nil, "xxxx", types.NewSlice(t1Comparable2))), + types.NewTuple(types.NewVar(token.NoPos, nil, "xxxx", t1Comparable2)), + false, + ), + types.NewSignatureType(nil, nil, nil, + types.NewTuple(), + types.NewTuple(), + false), + types.NewSignatureType(nil, nil, nil, + types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.String])), + types.NewTuple(), + false), + types.NewSignatureType(nil, nil, nil, + types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.Int32]), + types.NewVar(token.NoPos, nil, "", types.Typ[types.Int])), + types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.Int])), + false), + types.NewSignatureType(nil, nil, nil, + types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.String]), + types.NewVar(token.NoPos, nil, "", types.NewSlice(types.NewInterfaceType(nil, nil)) /*any/interface{}*/)), + types.NewTuple(), + true /*params[-1] needs to be a slice*/), + types.NewSignatureType(nil, nil, nil, + types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.String]), + types.NewVar(token.NoPos, nil, "", types.NewSlice(types.NewInterfaceType(nil, nil)) /*any/interface{}*/)), + types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.Int]), + types.NewVar(token.NoPos, nil, "", types.Universe.Lookup("error").Type())), + true /*params[-1] needs to be a slice*/), + types.NewSignatureType(nil, nil, + []*types.TypeParam{t1Any}, + types.NewTuple(types.NewVar(token.NoPos, nil, "xxxx", t1Any)), + nil, false, + ), + types.NewSignatureType(nil, nil, + []*types.TypeParam{t1Comparable}, + types.NewTuple(types.NewVar(token.NoPos, nil, "xxxx", t1Comparable)), + nil, false, + ), + + // hsMap + types.NewSignatureType(nil, nil, + []*types.TypeParam{t1Any2, t2Any2}, + types.NewTuple( + types.NewVar(token.NoPos, nil, "", + types.NewSlice(t1Any2), + ), + types.NewVar(token.NoPos, nil, "", + types.NewSignatureType(nil, nil, nil, + types.NewTuple( + types.NewVar(token.NoPos, nil, "", t1Any2), + ), + types.NewTuple(types.NewVar(token.NoPos, nil, "", t2Any2)), + false), + ), + ), + types.NewTuple( + types.NewVar(token.NoPos, nil, "", + types.NewSlice(t2Any2), + ), + ), + false), + + // lo.Map + types.NewSignatureType(nil, nil, + []*types.TypeParam{t1Any3, t2Any3}, + types.NewTuple( + types.NewVar(token.NoPos, nil, "", + types.NewSlice(t1Any3), + ), + types.NewVar(token.NoPos, nil, "", + types.NewSignatureType(nil, nil, nil, + types.NewTuple( + types.NewVar(token.NoPos, nil, "", t1Any3), + types.NewVar(token.NoPos, nil, "", types.Typ[types.Int]), // lo > hs + ), + types.NewTuple(types.NewVar(token.NoPos, nil, "", t2Any3)), + false), + ), + ), + types.NewTuple( + types.NewVar(token.NoPos, nil, "", + types.NewSlice(t2Any3), + ), + ), + false), + nil, + nil, + } + + for i, inp := range inps { + out := outputs[i] + sig, err := Dummy(inp) + t.Log(i, err) + t.Log(i, inp) + t.Log(i, sig) + t.Log(i, out) + assert.Nil(t, err, "case~~~~ %d", i) + assert.True(t, types.IdenticalIgnoreTags(out, sig), "case~~~~ %d", i) + } + +} diff --git a/u/types.go b/u/types.go new file mode 100644 index 0000000..f7471bc --- /dev/null +++ b/u/types.go @@ -0,0 +1,288 @@ +package u + +import ( + "fmt" + "go/types" +) + +func CopyTypeParams(src []*types.TypeParam) []*types.TypeParam { + cp := make([]*types.TypeParam, len(src)) + for i := 0; i < len(src); i++ { + tp := src[i] + cp[i] = types.NewTypeParam(tp.Obj(), tp.Constraint()) + } + return cp +} + +// RebindVars also anonymize vars. +// vars can be signature.Params() or signature.Results() +func RebindVars(vars *types.Tuple, nameToTypeParam map[string]*types.TypeParam, debug any) (varsBound []*types.Var, err error) { + varsBound = make([]*types.Var, vars.Len()) + for i := 0; i < vars.Len(); i++ { + var_ := vars.At(i) + newParam, err := rebindVar(var_, nameToTypeParam, debug) + if err != nil { + return nil, fmt.Errorf("RebindVars %d: %w", i, err) + } + varsBound[i] = newParam + } + return varsBound, nil +} + +// rebindVar also anonymize var. +func rebindVar(var_ *types.Var, nameToTypeParam map[string]*types.TypeParam, debug any) (varBound *types.Var, err error) { + if var_ == nil { + return nil, nil + } + fdType, err := rebindType(var_.Type(), nameToTypeParam, debug) + if err != nil { + return nil, fmt.Errorf("rebindVar: %w", err) + } + // Anonymize early here! + return types.NewVar(var_.Pos(), var_.Pkg(), "" /*var_.Name()*/, fdType), nil +} + +// [typBound] will have the same type as [typ]. +// I tried to use [T types.Type](var_ T……), but failed. +// TODO maybe make it OO, hiding nameToTypeParam to field. +func rebindType(typ types.Type, nameToTypeParam map[string]*types.TypeParam, debug any) (typBound types.Type, err error) { + // refers to types.IdenticalIgnoreTags + switch typ.(type) { + case *types.TypeParam: + tp := typ.(*types.TypeParam) + // nameToTypeParam is finally used here. + newTP, bound := nameToTypeParam[tp.Obj().Name()] + if !bound { + // Type checking should not let this happen? + return nil, fmt.Errorf("[typ=%s] has a parameterized [type=%s] not declared in type typ list", typ, tp.Obj().Name()) + } + return newTP, nil + + case *types.Array: + ar := typ.(*types.Array) + // I need a Maybe Monad + el, err := rebindType(ar.Elem(), nameToTypeParam, debug) + if err != nil { + return nil, fmt.Errorf("rebindType Array: %w", err) + } + return types.NewArray(el, ar.Len()), nil + + case *types.Slice: + sl := typ.(*types.Slice) + el, err := rebindType(sl.Elem(), nameToTypeParam, debug) + if err != nil { + return nil, fmt.Errorf("rebindType Slice: %w", err) + } + return types.NewSlice(el), nil + + case *types.Struct: // TODO test + return typ, nil + + //// 先不管这个 case……（后果：struct + 使用类型参数，会不正常 TODO） + //// internal/fuzz ReadCorpus + //// func(dir string, types []reflect.Type) ([]struct{Parent string; Path string; Data []byte; Values []any; Generation int; IsSeed bool}, error) + //// panic: multiple fields with the same name + // + //st := typ.(*types.Struct) + //fields := make([]*types.Var, st.NumFields()) + //tags := make([]string, st.NumFields()) + //for i := 0; i < st.NumFields(); i++ { + // fd, err := rebindVar(st.Field(i), nameToTypeParam, debug) + // if err != nil { + // return nil, fmt.Errorf("rebindType Struct: %w", err) + // } + // fields[i] = fd + // tags[i] = st.Tag(i) // just copy + //} + //return types.NewStruct(fields, tags), nil + + case *types.Pointer: + pt := typ.(*types.Pointer) + el, err := rebindType(pt.Elem(), nameToTypeParam, debug) + if err != nil { + return nil, fmt.Errorf("rebindType Pointer: %w", err) + } + return types.NewPointer(el), nil + + case *types.Tuple: // “simplified struct” + tp := typ.(*types.Tuple) + vars := make([]*types.Var, tp.Len()) + for i := 0; i < tp.Len(); i++ { + var_, err := rebindVar(tp.At(i), nameToTypeParam, debug) + if err != nil { + return nil, fmt.Errorf("rebindType Tuple: %w", err) + } + vars[i] = var_ + } + return types.NewTuple(vars...), nil + + case *types.Signature: + // spec#FunctionType can not have type params, so I don’t need to handle nested type param scope. Lucky. + // But that’s exactly the reason to RebindVars. Unlucky. + + sig := typ.(*types.Signature) + params, err := RebindVars(sig.Params(), nameToTypeParam, "params of "+sig.String()+" <- "+debug.(string)) // mutual recursion + if err != nil { + return nil, fmt.Errorf("rebindType Signature Params: %w", err) + } + results, err := RebindVars(sig.Results(), nameToTypeParam, "results of "+sig.String()+" <- "+debug.(string)) + if err != nil { + return nil, fmt.Errorf("rebindType Signature Results: %w", err) + } + recv, err := rebindVar(sig.Recv(), nameToTypeParam, debug) + if err != nil { + return nil, fmt.Errorf("rebindType Signature Recv: %w", err) + } + + // trivial + // todo use u.CopyTypeParams? + recvTypeParams0 := sig.RecvTypeParams() + recvTypeParams := make([]*types.TypeParam, recvTypeParams0.Len()) + for i := 0; i < recvTypeParams0.Len(); i++ { + recvTypeParams[i] = recvTypeParams0.At(i) + } + typeParams0 := sig.TypeParams() + typeParams := make([]*types.TypeParam, typeParams0.Len()) + for i := 0; i < typeParams0.Len(); i++ { + typeParams[i] = typeParams0.At(i) + } + + return types.NewSignatureType(recv, recvTypeParams, typeParams, types.NewTuple(params...), types.NewTuple(results...), sig.Variadic()), nil + + case *types.Union: // TODO: is there? + un := typ.(*types.Union) + terms := make([]*types.Term, un.Len()) + for i := 0; i < un.Len(); i++ { + newType, err := rebindType(un.Term(i).Type(), nameToTypeParam, debug) + if err != nil { + return nil, fmt.Errorf("rebindType Union Recv: %w", err) + } + terms[i] = types.NewTerm(un.Term(i).Tilde(), newType) + } + return types.NewUnion(terms), nil + + case *types.Interface: + // TODO impl + //iface:=typ.(*types.Interface) + return typ, nil + + case *types.Map: + mp := typ.(*types.Map) + key, err := rebindType(mp.Key(), nameToTypeParam, debug) + if err != nil { + return nil, fmt.Errorf("rebindType Map Key: %w", err) + } + el, err := rebindType(mp.Elem(), nameToTypeParam, debug) + if err != nil { + return nil, fmt.Errorf("rebindType Map Elem: %w", err) + } + return types.NewMap(key, el), nil + + case *types.Chan: + ch := typ.(*types.Chan) + // I need a Maybe Monad + el, err := rebindType(ch.Elem(), nameToTypeParam, debug) + if err != nil { + return nil, fmt.Errorf("rebindType Chan: %w", err) + } + return types.NewChan(ch.Dir(), el), nil + + case *types.Named: // func(A[T]) + nm := typ.(*types.Named) + nm.TypeParams() // does it mean type args here? + // TODO impl + return typ, nil + + default: // nil, *types.Basic + return typ, nil + } +} + +// Anonymize 先写一层吧……累了。显而易见的 badcase: lo.Map 模糊搜索不到 TODO recursive +// TODO 融进 RebindVars? +// 2024-09-13 09:42:31 先在 Anonymize 加入「类型参数换名 2」的成分。参考 collect.ParseGenDeclTypeSpecFuncSigs +// RebindVars Anonymize 类型参数换名 三者大概会融合起来： +// 大概把 Anonymize 去掉变量名的操作写进 RebindVars，然后把 Anonymize 的核心实现改为调用 RebindVars。 +func Anonymize(sig *types.Signature) *types.Signature { + //type S[T any] struct {} + //func (S[T])f[F any](){} // Method cannot have type parameters（指 [F any]） + + //// 先只管 .TypeParams()；TODO .RecvTypeParams() + typeParams := make([]*types.TypeParam, sig.TypeParams().Len()) + nameToTypeParam := make(map[string]*types.TypeParam) + for i := 0; i < sig.TypeParams().Len(); i++ { + tp := sig.TypeParams().At(i) + newTP := types.NewTypeParam(types.NewTypeName( + tp.Obj().Pos(), tp.Obj().Pkg(), fmt.Sprintf("_T%d", i), tp.Obj().Type()), + tp.Constraint()) + typeParams[i] = newTP + nameToTypeParam[tp.Obj().Name()] = newTP + } + + params, err := RebindVars(sig.Params(), nameToTypeParam, "params of "+sig.String()) + if err != nil { + // TODO + panic(fmt.Errorf("rebinding params of %s: %w", sig, err)) + } + results, err := RebindVars(sig.Results(), nameToTypeParam, "results of "+sig.String()) + if err != nil { + // TODO + panic(fmt.Errorf("rebinding results of %s: %w", sig, err)) + } + recv, err := rebindVar(sig.Recv(), nameToTypeParam /*TODO 用 sig.RecvTypeParams() 对应物*/, "recv of "+sig.String()) + + anonSig := types.NewSignatureType( + recv, + CopyTypeParams(TypeParamListToSliceOfTypeParam(sig.RecvTypeParams())), + typeParams, // with new name! + types.NewTuple(params...), + types.NewTuple(results...), + sig.Variadic(), + ) + return anonSig + + //return types.NewSignatureType( + // anonymizeVar(sig.Recv()), + // CopyTypeParams(TypeParamListToSliceOfTypeParam(sig.RecvTypeParams())), + // CopyTypeParams(TypeParamListToSliceOfTypeParam(sig.TypeParams())), + // types.NewTuple(lo.Map(TupleToSliceOfVar(sig.Params()), anonymizeVarI)...), + // types.NewTuple(lo.Map(TupleToSliceOfVar(sig.Results()), anonymizeVarI)...), + // sig.Variadic(), + //) +} +func anonymizeVarI(v *types.Var, _ int) *types.Var { + return anonymizeVar(v) +} +func anonymizeVar(v *types.Var) *types.Var { + if v == nil { + return v + } + return types.NewVar(v.Pos(), nil, "", v.Type()) +} + +func TypeParamListToSliceOfTypeParam(inp *types.TypeParamList) []*types.TypeParam { + out := make([]*types.TypeParam, inp.Len()) + for i := 0; i < inp.Len(); i++ { + out[i] = inp.At(i) + } + return out +} + +func TupleToSliceOfVar(inp *types.Tuple) []*types.Var { + out := make([]*types.Var, inp.Len()) + for i := 0; i < inp.Len(); i++ { + out[i] = inp.At(i) + } + return out +} + +// todo see also https://pkg.go.dev/golang.org/x/exp/slices#Delete +func TupleToSliceOfVarExcept(inp *types.Tuple, except int) []*types.Var { + out := make([]*types.Var, 0, inp.Len()-1) + for i := 0; i < inp.Len(); i++ { + if i != except { + out = append(out, inp.At(i)) + } + } + return out +} diff --git a/u/types_test.go b/u/types_test.go new file mode 100644 index 0000000..a79738a --- /dev/null +++ b/u/types_test.go @@ -0,0 +1,55 @@ +package u + +import ( + "go/token" + "go/types" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAnonymize(t *testing.T) { + ////Dummy contains Anonymize + //sig, err := Dummy(`[a,b any, c comparable, d any] func(bool, a, a) (y a)`) + //if err != nil { + // t.Error(err) + //} + //t.Log(sig) + + // copied from TestDummy + t1Any3 := types.NewTypeParam( + types.NewTypeName(token.NoPos, nil, "a", nil /*这里？TODO*/), + types.Universe.Lookup("any").Type()) + t2Any3 := types.NewTypeParam( + types.NewTypeName(token.NoPos, nil, "b", nil /*这里？TODO*/), + types.Universe.Lookup("any").Type()) + // lo.Map + loMap := types.NewSignatureType(nil, nil, + []*types.TypeParam{t1Any3, t2Any3}, + types.NewTuple( + types.NewVar(token.NoPos, nil, "xs", + types.NewSlice(t1Any3), + ), + types.NewVar(token.NoPos, nil, "f", + types.NewSignatureType(nil, nil, nil, + types.NewTuple( + types.NewVar(token.NoPos, nil, "", t1Any3), + types.NewVar(token.NoPos, nil, "", types.Typ[types.Int]), // lo > hs + ), + types.NewTuple(types.NewVar(token.NoPos, nil, "", t2Any3)), + false), + ), + ), + types.NewTuple( + types.NewVar(token.NoPos, nil, "ys", + types.NewSlice(t2Any3), + ), + ), + false) + t.Log(loMap) + + sigA := Anonymize(loMap) + t.Log(sigA) + + assert.Equal(t, `func[_T0, _T1 any]([]_T0, func(_T0, int) _T1) []_T1`, sigA.String()) +} diff --git a/u/u.go b/u/u.go new file mode 100644 index 0000000..0a012a3 --- /dev/null +++ b/u/u.go @@ -0,0 +1,2 @@ +// Package u contains... utils. ( ﾟ∀。) +package u diff --git a/u/util.go b/u/util.go deleted file mode 100644 index 7355256..0000000 --- a/u/util.go +++ /dev/null @@ -1,27 +0,0 @@ -package u - -import ( - "github.com/samber/lo" - "go/types" -) - -type T2 lo.Tuple2[*types.Signature, *types.Func] - -//type T2T lo.Tuple2[*types.Signature, *ast.GenDecl] - -// for better debugging -func (t T2) String() string { - return t.B.FullName() + ": " + t.A.String() -} - -// KsVs returns []K, []V in corresponding order. -func KsVs[K comparable, V any](m map[K]V) ([]K, []V) { - ks := make([]K, len(m)) - vs := make([]V, len(m)) - i := 0 - for k, v := range m { - ks[i] = k - vs[i] = v - } - return ks, vs -}