diff --git a/go.mod b/go.mod index 19673d4e..5b848882 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/julienschmidt/httprouter v1.3.0 github.com/mitchellh/mapstructure v1.5.0 github.com/powerman/rpc-codec v1.2.2 + github.com/protocolbuffers/txtpbfmt v0.0.0-20240416193709-1e18ef0a7fdc github.com/qri-io/jsonpointer v0.1.1 github.com/stretchr/testify v1.9.0 github.com/wk8/go-ordered-map/v2 v2.1.8 @@ -103,6 +104,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/sys/mountinfo v0.7.1 // indirect github.com/moby/sys/user v0.1.0 // indirect diff --git a/go.sum b/go.sum index 283a7997..06853bd6 100644 --- a/go.sum +++ b/go.sum @@ -547,6 +547,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubescape/go-git-url v0.0.30 h1:PIbg86ae0ftee/p/Tu/6CA1ju6VoJ51G3sQWNHOm6wg= github.com/kubescape/go-git-url v0.0.30/go.mod h1:3ddc1HEflms1vMhD9owt/3FBES070UaYTUarcjx8jDk= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -566,6 +568,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= @@ -630,6 +634,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/protocolbuffers/txtpbfmt v0.0.0-20240416193709-1e18ef0a7fdc h1:DRZwH75/E4a2SOr7+gKZ99OEhmjzBzAhgyTnzo1TepY= +github.com/protocolbuffers/txtpbfmt v0.0.0-20240416193709-1e18ef0a7fdc/go.mod h1:jgxiZysxFPM+iWKwQwPR+y+Jvo54ARd4EisXxKYpB5c= github.com/qri-io/jsonpointer v0.1.1 h1:prVZBZLL6TW5vsSB9fFHFAMBLI4b0ri5vribQlTJiBA= github.com/qri-io/jsonpointer v0.1.1/go.mod h1:DnJPaYgiKu56EuDp8TU5wFLdZIcAnb/uH9v37ZaMV64= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= diff --git a/pkg/ast/ast.go b/pkg/ast/ast.go new file mode 100644 index 00000000..fc1f4fec --- /dev/null +++ b/pkg/ast/ast.go @@ -0,0 +1,26 @@ +package ast + +// TODO: add more nodes from https://github.com/kcl-lang/kcl/blob/main/kclvm/ast/src/ast.rs + +// Pos denotes the struct tuple (filename, line, column, end_line, end_column). +type Pos struct { + Filename string `json:"filename"` + Line uint64 `json:"line"` + Column uint64 `json:"column"` + EndLine uint64 `json:"end_line"` + EndColumn uint64 `json:"end_column"` +} + +// Node is the file, line, and column number information that all AST nodes need to contain. +type Node interface { + Pos() Pos + Index() string +} + +// AstIndex represents a unique identifier for AST nodes. +type AstIndex string + +// Comment node. +type Comment struct { + Text string +} diff --git a/pkg/tools/gen/genkcl_proto.go b/pkg/tools/gen/genkcl_proto.go index 6a6ee18b..4f62e0c2 100644 --- a/pkg/tools/gen/genkcl_proto.go +++ b/pkg/tools/gen/genkcl_proto.go @@ -160,7 +160,9 @@ func (k *kclGenerator) genKclFromProtoDef(builder *bufio.Writer, definitions *pr builder.WriteString(lineBreak) // TODO: Import node on multi proto files case *proto.Import: - logger.GetLogger().Warningf("unsupported import statement for %v", def.Filename) + if def.Filename != pbTypAnyPkgPath { + logger.GetLogger().Warningf("unsupported import statement for %v", def.Filename) + } } } if len(oneOfSchemas) > 0 { diff --git a/pkg/tools/gen/genkcl_test.go b/pkg/tools/gen/genkcl_test.go index 87dd14e7..42c62cd0 100644 --- a/pkg/tools/gen/genkcl_test.go +++ b/pkg/tools/gen/genkcl_test.go @@ -272,6 +272,31 @@ func TestGenKclFromProto(t *testing.T) { }) } +func TestGenKclFromTextProto(t *testing.T) { + generator := TextProtoGenerator{} + got, err := generator.GenFromSchemaFile("./testdata/textproto/data.textproto", nil, "./testdata/textproto/schema.k", nil, "Config") + assert2.Nil(t, err) + expected := &config{ + Name: "Config", + Data: []data{ + {Key: "a", Value: 1}, + {Key: "b", Value: 2.0}, + {Key: "c", Value: true}, + {Key: "d", Value: "value"}, + {Key: "empty1", Value: []interface{}(nil)}, + {Key: "empty2", Value: []interface{}(nil)}, + {Key: "int1", Value: []interface{}{1, 2}}, + {Key: "int2", Value: []interface{}{1, 2}}, + {Key: "int3", Value: []interface{}{1, 2}}, + {Key: "string1", Value: []interface{}{"a", "b"}}, + {Key: "float1", Value: []interface{}{100.0, 1.0, 0.0}}, + {Key: "map", Value: []data{{Key: "foo", Value: 2}}}, + {Key: "map", Value: []data{{Key: "bar", Value: 3}}}, + }, + } + assert2.Equal(t, expected, got) +} + type TestData = data func TestGenKclFromJsonAndImports(t *testing.T) { diff --git a/pkg/tools/gen/genkcl_textproto.go b/pkg/tools/gen/genkcl_textproto.go new file mode 100644 index 00000000..614d6914 --- /dev/null +++ b/pkg/tools/gen/genkcl_textproto.go @@ -0,0 +1,224 @@ +package gen + +import ( + "errors" + "fmt" + "runtime" + "strconv" + "strings" + + "kcl-lang.io/kcl-go/pkg/ast" + "kcl-lang.io/kcl-go/pkg/kcl" + "kcl-lang.io/kcl-go/pkg/loader" + + pbast "github.com/protocolbuffers/txtpbfmt/ast" + "github.com/protocolbuffers/txtpbfmt/parser" + "github.com/protocolbuffers/txtpbfmt/unquote" +) + +var ( + ErrNoSchemaFound = errors.New("no expected schema found") +) + +type TextProtoGenerator struct { + file string +} + +// Parse parses the given textproto bytes and converts them to KCL configs. +// Note fields in the textproto that have no corresponding field in schema +// are ignored. +func (d *TextProtoGenerator) Gen(filename string, src any, schema *kcl.KclType) (*config, error) { + source, err := loader.ReadSource(filename, src) + if runtime.GOOS == "windows" { + source = []byte(strings.Replace(string(source), "\r\n", "\n", -1)) + } + if err != nil { + return nil, err + } + cfg := parser.Config{} + d.file = filename + nodes, err := parser.ParseWithConfig(source, cfg) + if err != nil { + return nil, err + } + return d.genProperties(schema, nodes) +} + +// ParseFromSchemaFile parses the given textproto bytes and converts them +// to KCL configs with the schema file. Note fields in the textproto that +// have no corresponding field in schema are ignored. +func (d *TextProtoGenerator) GenFromSchemaFile(filename string, src any, schemaFile string, schemaSrc any, schemaName string) (*config, error) { + types, err := kcl.GetSchemaType(schemaFile, schemaSrc, schemaName) + if err != nil { + return nil, err + } + if len(types) == 0 { + return nil, ErrNoSchemaFound + } + return d.Gen(filename, src, types[0]) +} + +func (d *TextProtoGenerator) genProperties(ty *kcl.KclType, nodes []*pbast.Node) (*config, error) { + var values []data + for _, n := range nodes { + var comments []*ast.Comment + if n.Values == nil && n.Children == nil { + if comments = addComments(n.PreComments...); comments != nil { + continue + } + } + if ty == nil || ty.Properties == nil { + continue + } + ty, ok := ty.Properties[n.Name] + // Ignore unknown attributes that not defined in the schema + if !ok { + continue + } + value, err := d.genValue(ty, n) + if err != nil { + return nil, err + } + values = append(values, data{ + Key: n.Name, + Value: value, + Comments: comments, + }) + } + return &config{ + Name: ty.SchemaName, + Data: values, + }, nil +} + +func (d *TextProtoGenerator) genValue(ty *kcl.KclType, n *pbast.Node) (any, error) { + if n == nil { + return nil, nil + } + tyStr := typAny + if ty != nil { + tyStr = ty.Type + } + switch tyStr { + case typSchema: + if k := len(n.Values); k > 0 { + return nil, d.errorf(n, "not allowed for the message type; found %d", k) + } + return d.genProperties(ty, n.Children) + case typDict: + if k := len(n.Values); k > 0 { + return nil, d.errorf(n, "not allowed for the message type; found %d", k) + } + var values []data + var key string + var value any + var comments []*ast.Comment + for _, c := range n.Children { + if len(c.Values) != 1 { + return nil, d.errorf(n, "expected 1 value, found %d", len(c.Values)) + } + switch c.Name { + case "key": + s, err := d.genValue(ty.Key, c) + if err != nil { + return nil, err + } + key = s.(string) + case "value": + s, err := d.genValue(ty.Item, c) + if err != nil { + return nil, err + } + value = s + comments = addComments(n.ClosingBraceComment) + default: + return nil, d.errorf(c, "unsupported key name %q in map", c.Name) + } + } + if key != "" { + values = append(values, data{ + Key: key, + Value: value, + Comments: comments, + }) + } + return values, nil + case typList: + var values []any + for _, v := range n.Values { + if comments := addComments(n.PreComments...); comments != nil { + continue + } + y := *n + y.Values = []*pbast.Value{v} + genV, err := d.genValue(ty.Item, &y) + if err != nil { + return nil, err + } + values = append(values, genV) + } + return values, nil + case typInt: + if len(n.Values) != 1 { + return nil, d.errorf(n, "expected 1 value, found %d", len(n.Values)) + } + s := n.Values[0].Value + v, err := strconv.Atoi(s) + if err != nil { + return nil, d.errorf(n, "invalid number %s", s) + } + return v, nil + case typFloat: + if len(n.Values) != 1 { + return nil, d.errorf(n, "expected 1 value, found %d", len(n.Values)) + } + s := n.Values[0].Value + switch s { + case "inf", "nan": + return nil, d.errorf(n, "unexpected float value %s", s) + } + v, err := strconv.ParseFloat(s, 32) + if err != nil { + return nil, d.errorf(n, "invalid number %s", s) + } + return v, nil + case typBool: + if len(n.Values) != 1 { + return nil, d.errorf(n, "expected 1 value, found %d", len(n.Values)) + } + s := n.Values[0].Value + switch s { + case "true": + return true, nil + default: + return false, nil + } + case typStr, typAny, typUnion: + s, _, err := unquote.Unquote(n) + if err != nil { + return nil, d.errorf(n, "invalid value to string %s", err.Error()) + } + return s, nil + default: + return nil, fmt.Errorf("unsupported type '%v'", ty.Type) + } +} + +func (d *TextProtoGenerator) errorf(n *pbast.Node, format string, a ...any) error { + return errors.New(d.locationFormat(n) + ": " + fmt.Sprintf(format, a...)) +} + +func (d *TextProtoGenerator) locationFormat(n *pbast.Node) string { + return fmt.Sprintf("%s:%d:%d", d.file, n.Start.Line, n.Start.Column) +} + +func addComments(lines ...string) []*ast.Comment { + var comments []*ast.Comment + for _, c := range lines { + if !strings.HasPrefix(c, "#") { + continue + } + comments = append(comments, &ast.Comment{Text: c}) + } + return comments +} diff --git a/pkg/tools/gen/genpb.go b/pkg/tools/gen/genpb.go index 2f2224ac..6f8bc2e6 100644 --- a/pkg/tools/gen/genpb.go +++ b/pkg/tools/gen/genpb.go @@ -14,7 +14,8 @@ import ( ) const ( - pbTypAny = "google.protobuf.Any" + pbTypAny = "google.protobuf.Any" + pbTypAnyPkgPath = "google/protobuf/any.proto" ) type Options struct { @@ -49,13 +50,13 @@ func (p *pbGenerator) GenProto(filename string, src interface{}) (string, error) } if p.opt.GoPackage == "" { - p.opt.GoPackage = p.getOptopn_go_package(string(code)) + p.opt.GoPackage = p.getOptionGoPackage(string(code)) } if p.opt.PbPackage == "" { - p.opt.PbPackage = p.getOptopn_pb_package(string(code)) + p.opt.PbPackage = p.getOptionPbPackage(string(code)) } - typs, err := kcl.GetSchemaType(filename, string(code), "") + types, err := kcl.GetSchemaType(filename, string(code), "") if err != nil { return "", err } @@ -74,7 +75,7 @@ func (p *pbGenerator) GenProto(filename string, src interface{}) (string, error) fmt.Fprintf(&buf, "option go_package = \"%s\";\n", p.opt.GoPackage) - var messageBody = p.genProto_messages(typs...) + var messageBody = p.genProtoMessages(types...) if p.needImportAny { fmt.Fprintln(&buf) @@ -86,12 +87,12 @@ func (p *pbGenerator) GenProto(filename string, src interface{}) (string, error) return buf.String(), nil } -func (p *pbGenerator) genProto_messages(types ...*pb.KclType) string { +func (p *pbGenerator) genProtoMessages(types ...*pb.KclType) string { var buf bytes.Buffer for _, typ := range types { switch typ.Type { case typSchema: - p.genProto_schema(&buf, typ) + p.genProtoSchema(&buf, typ) default: fmt.Fprintf(&buf, "ERR: unknown '%v', json = %v\n", typ.Type, jsonString(typ)) } @@ -99,7 +100,7 @@ func (p *pbGenerator) genProto_messages(types ...*pb.KclType) string { return buf.String() } -func (p *pbGenerator) genProto_schema(w io.Writer, typ *pb.KclType) { +func (p *pbGenerator) genProtoSchema(w io.Writer, typ *pb.KclType) { assert(typ.Type == typSchema) fmt.Fprintln(w) @@ -147,7 +148,7 @@ func (p *pbGenerator) genProto_schema(w io.Writer, typ *pb.KclType) { } // #kclvm/genpb: option go_package = kcl_gen/_/hello_k -func (p *pbGenerator) getOptopn_go_package(code string) string { +func (p *pbGenerator) getOptionGoPackage(code string) string { if !strings.Contains(code, `#kclvm/genpb:`) { return "" } @@ -161,7 +162,7 @@ func (p *pbGenerator) getOptopn_go_package(code string) string { } // #kclvm/genpb: option pb_package = kcl_gen._.hello_k -func (p *pbGenerator) getOptopn_pb_package(code string) string { +func (p *pbGenerator) getOptionPbPackage(code string) string { if !strings.Contains(code, `#kclvm/genpb:`) { return "" } diff --git a/pkg/tools/gen/testdata/textproto/data.textproto b/pkg/tools/gen/testdata/textproto/data.textproto new file mode 100644 index 00000000..45b2a6c4 --- /dev/null +++ b/pkg/tools/gen/testdata/textproto/data.textproto @@ -0,0 +1,29 @@ +a: 1 +b: 2.0 +c: true +d: "value" +empty1: [] +empty2: [ +] + +int1: [1, 2] +int2: [1 2] +int3: [ + 1 + 2 +] + +string1: [ + "a", + "b" +] + +float1: [ 1e+2 1. 0] +map: { + key: "foo" + value: 2 +} +map: { + key: "bar" + value: 3 +} diff --git a/pkg/tools/gen/testdata/textproto/schema.k b/pkg/tools/gen/testdata/textproto/schema.k new file mode 100644 index 00000000..76e3887f --- /dev/null +++ b/pkg/tools/gen/testdata/textproto/schema.k @@ -0,0 +1,14 @@ +schema Config: + a: int + b: float + c: bool + d: str + e: any + empty1: [any] + empty2: [] + int1: [int] + int2: [int] + int3: [int] + string1: [str] + float1: [float] + $map: {str:int} diff --git a/pkg/tools/gen/types.go b/pkg/tools/gen/types.go index fb2f7776..dc11a721 100644 --- a/pkg/tools/gen/types.go +++ b/pkg/tools/gen/types.go @@ -8,6 +8,8 @@ import ( "strconv" "strings" + "kcl-lang.io/kcl-go/pkg/ast" + pb "kcl-lang.io/kcl-go/pkg/spec/gpyrpc" ) @@ -132,7 +134,7 @@ type kclFile struct { Data []data // [k =] [T]v configurations, k and T is optional. Config []config - // ExtraCode denotes + // ExtraCode denotes the any kcl code that we want to append the end of file. ExtraCode string } @@ -199,15 +201,17 @@ type indexSignature struct { // data is a kcl data definition. type data struct { - Key string - Value interface{} + Key string + Value interface{} + Comments []*ast.Comment } type config struct { - Var string - Name string - IsUnion bool - Data []data + Var string + Name string + IsUnion bool + Data []data + Comments []*ast.Comment } type typeInterface interface {