diff --git a/bin/xlsxcfg/root_cmd.go b/bin/xlsxcfg/root_cmd.go index a7522c6..76d4bc9 100644 --- a/bin/xlsxcfg/root_cmd.go +++ b/bin/xlsxcfg/root_cmd.go @@ -46,9 +46,9 @@ func run(cmd *cobra.Command, args []string) { if err != nil { log.Fatalln("load proto files failed:", err) } - sheetsData, err := xlsxcfg.LoadXlsxFiles(ctx, xlsxcfg.NewParameter(cfg, typeProvider), args...) + sheetsData, err := xlsxcfg.LoadXlsxFiles(ctx, xlsxcfg.NewConfig(cfg, typeProvider), args...) if err != nil { - log.Fatalln("parse xls files failed:", err) + log.Fatalln("parse xlsx files failed:", err) } msgFactory := dynamic.NewMessageFactoryWithDefaults() @@ -77,7 +77,7 @@ func run(cmd *cobra.Command, args []string) { } } -func writeFile(cfg *xlsxcfg.Config, sheet string, msg protoiface.MessageV1, jm *jsonpb.Marshaler) { +func writeFile(cfg *xlsxcfg.ConfigFile, sheet string, msg protoiface.MessageV1, jm *jsonpb.Marshaler) { if cfg.Output.WriteBytes { // output proto bytes file buf, err := proto.Marshal(msg) diff --git a/config.go b/config.go index 21590d9..2558af2 100644 --- a/config.go +++ b/config.go @@ -8,7 +8,7 @@ import ( "gopkg.in/yaml.v3" ) -type Config struct { +type ConfigFile struct { Proto struct { Files []string `yaml:"files"` ImportPath []string `yaml:"import_path"` @@ -29,8 +29,8 @@ type Config struct { } `yaml:"output"` } -func ConfigFromFile(f string) (*Config, error) { - c := &Config{} +func ConfigFromFile(f string) (*ConfigFile, error) { + c := &ConfigFile{} if bs, err := os.ReadFile(f); err != nil { return nil, err } else if err = yaml.Unmarshal(bs, c); err != nil { @@ -39,19 +39,19 @@ func ConfigFromFile(f string) (*Config, error) { return c, nil } -type Parameter struct { - *Config +type Config struct { + *ConfigFile tp TypeProvidor } -func NewParameter(cfg *Config, tp TypeProvidor) *Parameter { - return &Parameter{ - Config: cfg, - tp: tp, +func NewConfig(cfg *ConfigFile, tp TypeProvidor) *Config { + return &Config{ + ConfigFile: cfg, + tp: tp, } } -func (p *Parameter) IsStrField(typeName, fieldPath string) bool { +func (p *Config) IsStrField(typeName, fieldPath string) bool { md := p.tp.MessageByName(typeName) if md == nil { log.Println("message " + typeName + " cannot find in proto messages") @@ -60,8 +60,8 @@ func (p *Parameter) IsStrField(typeName, fieldPath string) bool { return IsStrField(md, strings.Split(fieldPath, ".")...) } -func (p *Parameter) IsComment(i int, row []string) bool { - for _, l := range p.Config.Sheet.CommentRows { +func (p *Config) IsComment(i int, row []string) bool { + for _, l := range p.ConfigFile.Sheet.CommentRows { if l == i+1 { return true } @@ -69,10 +69,10 @@ func (p *Parameter) IsComment(i int, row []string) bool { return false } -func (p *Parameter) IsMeta(i int, row []string) bool { - return i+1 == p.Config.Sheet.MetaRow +func (p *Config) IsMeta(i int, row []string) bool { + return i+1 == p.ConfigFile.Sheet.MetaRow } -func (p *Parameter) IsData(i int, row []string) bool { +func (p *Config) IsData(i int, row []string) bool { return i+1 >= p.Sheet.DataRowStart } diff --git a/xlsx_row_parser.go b/xlsx_row_parser.go index 3c9aada..d188559 100644 --- a/xlsx_row_parser.go +++ b/xlsx_row_parser.go @@ -7,27 +7,36 @@ import ( ) type rowParser struct { - param *Parameter + param *Config isStr func(fieldPath string) bool errStack []error hasMeta bool res map[string]any - fn []func(v string) + set []func(v string) fnGet map[string]func() any fnSet map[string]func(v any) } +func newRowParser(typeName string, param *Config) *rowParser { + return &rowParser{ + param: param, + isStr: func(fieldPath string) bool { + return param.IsStrField(typeName, fieldPath) + }, + } +} + // Meta feed metadata row to the parser. -func (t *rowParser) Meta(ctx context.Context, row []string) { - t.fnGet = make(map[string]func() any, len(row)) - t.fnSet = make(map[string]func(v any), len(row)) - t.fn = make([]func(v string), 0, len(row)) - t.res = make(map[string]any) - t.fnGet[""] = func() any { return t.res } // top level getter +func (p *rowParser) Meta(ctx context.Context, row []string) { + p.fnGet = make(map[string]func() any, len(row)) + p.fnSet = make(map[string]func(v any), len(row)) + p.set = make([]func(v string), 0, len(row)) + p.res = make(map[string]any) + p.fnGet[""] = func() any { return p.res } // top level getter for _, meta := range row { if meta == "" { - t.fn = append(t.fn, func(_ string) {}) + p.set = append(p.set, func(_ string) {}) continue } tr := newTokenReader(meta) @@ -38,49 +47,55 @@ func (t *rowParser) Meta(ctx context.Context, row []string) { typePath := tr.TypePath() // log.Printf("head: %s -- ident: %s -- prevIdent: %s -- fullIdent: %s -- typePath: %s\n", hd, ident, prevIdent, fullIdent, typePath) if ai := tr.ListIndex(); ai >= 0 { - t.metaList(ident, prevIdent, fullIdent, typePath, ai-1) - if !tr.HasNext() { - // end at list item - t.fn = append(t.fn, func(v string) { - // log.Printf("arr set [%s] to [%s]\n", fullIdent, v) - t.fnSet[fullIdent](t.convertVal(typePath, fullIdent, v)) - }) - } + p.metaList(ident, prevIdent, fullIdent, typePath, ai-1, !tr.HasNext()) } else if tr.HasNext() { - t.metaStruct(ident, prevIdent, fullIdent, typePath) + p.metaStruct(ident, prevIdent, fullIdent, typePath) } else { - t.metaField(ident, prevIdent, fullIdent, typePath) + p.metaField(ident, prevIdent, fullIdent, typePath) } } } - t.hasMeta = true + p.hasMeta = true } // Parse parse one data row, MUST called after feed metadata. -func (t *rowParser) Parse(ctx context.Context, row []string) (map[string]any, error) { - if !t.hasMeta { +func (p *rowParser) Parse(ctx context.Context, row []string) (map[string]any, error) { + if !p.hasMeta { return nil, fmt.Errorf("row parser no metadata") } - t.res = make(map[string]any) + p.res = make(map[string]any) + ni := len(row) - 1 + for ; ni >= 0; ni-- { + if row[ni] != "" { + break + } + } + if ni < 0 { + return nil, nil + } + row = row[:ni+1] for i, cell := range row { - if i < len(t.fn) { - t.fn[i](cell) + if i < len(p.set) { + p.set[i](cell) } } - if len(t.errStack) > 0 { - return nil, t.errStack[0] + if len(p.errStack) > 0 { + return nil, p.errStack[0] } - return t.res, nil + return p.res, nil } -func (t *rowParser) convertVal(typePath, fullIdent, v string) any { +func (p *rowParser) convertVal(typePath, fullIdent, v string) any { var res any - if t.isStr(typePath) { + if p.isStr(typePath) { res = v } else { + if v == "" { + v = "0" + } n, e := strconv.ParseInt(v, 10, 64) if e != nil { - t.errStack = append(t.errStack, fmt.Errorf("parse col[%s] as number failed: %v", fullIdent, e)) + p.errStack = append(p.errStack, fmt.Errorf("parse col[%s] as number failed: %v", fullIdent, e)) } else { res = n } @@ -88,63 +103,76 @@ func (t *rowParser) convertVal(typePath, fullIdent, v string) any { return res } -func (t *rowParser) metaList(ident, prevIdent, fullIdent, typePath string, ai int) { - fullIdentList := ident - if prevIdent != "" { - fullIdentList = prevIdent + "." + ident - } - getList := t.fnGet[fullIdentList] - if getList == nil { - getList = func() any { - arr := t.fnGet[prevIdent]().(map[string]any)[ident] - if arr == nil || len(arr.([]any)) <= ai { - if arr != nil { - oa := arr.([]any) - na := make([]any, ai+1) - copy(na, oa) - arr = na - } else { - arr = make([]any, ai+1) - } - t.fnGet[prevIdent]().(map[string]any)[ident] = arr +func (p *rowParser) metaList(ident, prevIdent, fullIdent, typePath string, idx int, isEnd bool) { + p.fnGet[fullIdent] = func() any { + list := p.fnGet[prevIdent]().(map[string]any)[ident] + if list == nil || len(list.([]any)) <= idx { + if list != nil { + oldList := list.([]any) + newList := make([]any, idx+1) + copy(newList, oldList) + list = newList + } else { + list = make([]any, idx+1) } - return arr + p.fnGet[prevIdent]().(map[string]any)[ident] = list } - t.fnGet[fullIdentList] = getList + return list.([]any)[idx] } - t.fnGet[fullIdent] = func() any { - arr := getList() - return arr.([]any)[ai] + p.fnSet[fullIdent] = func(v any) { + list := p.fnGet[prevIdent]().(map[string]any)[ident] + if list == nil || len(list.([]any)) <= idx { + if list != nil { + oldList := list.([]any) + newList := make([]any, idx+1) + copy(newList, oldList) + list = newList + } else { + list = make([]any, idx+1) + } + p.fnGet[prevIdent]().(map[string]any)[ident] = list + } + list.([]any)[idx] = v } - t.fnSet[fullIdent] = func(v any) { - arr := getList() - arr.([]any)[ai] = v + if !isEnd { + return } + // end at list item + p.set = append(p.set, func(v string) { + // log.Printf("arr set [%s] to [%s]\n", fullIdent, v) + if v == "" { + return + } + p.fnSet[fullIdent](p.convertVal(typePath, fullIdent, v)) + }) } -func (t *rowParser) metaStruct(ident, prevIdent, fullIdent, typePath string) { - t.fnGet[fullIdent] = func() any { - return t.fnGet[prevIdent]().(map[string]any)[ident] +func (p *rowParser) metaStruct(ident, prevIdent, fullIdent, typePath string) { + p.fnGet[fullIdent] = func() any { + return p.fnGet[prevIdent]().(map[string]any)[ident] } - t.fnSet[fullIdent] = func(v any) { - prev := t.fnGet[prevIdent]() + p.fnSet[fullIdent] = func(v any) { + prev := p.fnGet[prevIdent]() if prev == nil { prev = map[string]any{} - t.fnSet[prevIdent](prev) + p.fnSet[prevIdent](prev) } prev.(map[string]any)[ident] = v } } -func (t *rowParser) metaField(ident, prevIdent, fullIdent, typePath string) { +func (p *rowParser) metaField(ident, prevIdent, fullIdent, typePath string) { // end at a field - t.fn = append(t.fn, func(v string) { + p.set = append(p.set, func(v string) { // log.Printf("set [%s] to [%s]\n", fullIdent, v) - prev := t.fnGet[prevIdent]() + if v == "" { + return + } + prev := p.fnGet[prevIdent]() if prev == nil { prev = map[string]any{} - t.fnSet[prevIdent](prev) + p.fnSet[prevIdent](prev) } - prev.(map[string]any)[ident] = t.convertVal(typePath, fullIdent, v) + prev.(map[string]any)[ident] = p.convertVal(typePath, fullIdent, v) }) } diff --git a/xlsx_sheet_parser.go b/xlsx_sheet_parser.go index b915b9c..26fbff8 100644 --- a/xlsx_sheet_parser.go +++ b/xlsx_sheet_parser.go @@ -3,61 +3,59 @@ package xlsxcfg import "context" type sheetParser struct { - name string - maxRow int - cols [][]string - param *Parameter + typeName string + maxRow int + cols [][]string + param *Config } -func newSheetParser(p *Parameter) *sheetParser { +func newSheetParser(p *Config) *sheetParser { return &sheetParser{param: p} } -func (sl *sheetParser) SetName(name string) { - sl.name = name +func (p *sheetParser) SetName(name string) { + p.typeName = name } -func (sl *sheetParser) Feed(cells []string) { - sl.cols = append(sl.cols, cells) - if sl.maxRow < len(cells) { - sl.maxRow = len(cells) +func (p *sheetParser) Feed(cells []string) { + p.cols = append(p.cols, cells) + if p.maxRow < len(cells) { + p.maxRow = len(cells) } } // Parse one xls sheet -func (sl *sheetParser) Parse(ctx context.Context) ([]any, error) { - colCount := len(sl.cols) - rp := &rowParser{ - param: sl.param, - isStr: func(fieldPath string) bool { - return sl.param.IsStrField(sl.name, fieldPath) - }, - } - res := make([]any, 0, sl.maxRow) - dataRows := make([][]string, 0, sl.maxRow) - for i := 0; i < sl.maxRow; i++ { +func (p *sheetParser) Parse(ctx context.Context) ([]any, error) { + colCount := len(p.cols) + rowParser := newRowParser(p.typeName, p.param) + result := make([]any, 0, p.maxRow) + dataRows := make([][]string, 0, p.maxRow) + for i := 0; i < p.maxRow; i++ { row := make([]string, 0, colCount) - for _, col := range sl.cols { + for _, col := range p.cols { if len(col) > i { row = append(row, col[i]) } else { row = append(row, "") } } - if sl.param.IsMeta(i, row) { - rp.Meta(ctx, row) - } else if sl.param.IsComment(i, row) { + if p.param.IsMeta(i, row) { + rowParser.Meta(ctx, row) + } else if p.param.IsComment(i, row) { // comment row, skip - } else if sl.param.IsData(i, row) { + } else if p.param.IsData(i, row) { dataRows = append(dataRows, row) } } for _, row := range dataRows { - rowData, err := rp.Parse(ctx, row) + rowData, err := rowParser.Parse(ctx, row) if err != nil { return nil, err } - res = append(res, rowData) + if rowData == nil { + continue + } + result = append(result, rowData) } - return res, nil + return result, nil } diff --git a/xlsx_src.go b/xlsx_src.go index d6feda4..c30fb21 100644 --- a/xlsx_src.go +++ b/xlsx_src.go @@ -7,7 +7,7 @@ import ( "github.com/xuri/excelize/v2" ) -func LoadXlsxFiles(ctx context.Context, param *Parameter, files ...string) (data map[string][]any, err error) { +func LoadXlsxFiles(ctx context.Context, param *Config, files ...string) (data map[string][]any, err error) { data = map[string][]any{} var configData map[string][]any for _, xlsFile := range files { @@ -25,7 +25,7 @@ func LoadXlsxFiles(ctx context.Context, param *Parameter, files ...string) (data return } -func loadXlsxFile(ctx context.Context, filePath string, param *Parameter) (data map[string][]any, err error) { +func loadXlsxFile(ctx context.Context, filePath string, param *Config) (data map[string][]any, err error) { f, err := excelize.OpenFile(filePath) if err != nil { return