From e1bdfda2696376d175ff4a78f32eb474090e30a9 Mon Sep 17 00:00:00 2001 From: tkuchiki Date: Mon, 2 Oct 2023 00:26:14 +0900 Subject: [PATCH] Move from `alp count` to `alp (json|ltsv|regexp) count` --- cmd/alp/cmd/command.go | 19 ++++- cmd/alp/cmd/count.go | 82 ++---------------- cmd/alp/cmd/count_test.go | 23 ----- cmd/alp/cmd/flags.go | 142 +++++++++++++++++++++++-------- cmd/alp/cmd/json.go | 33 ++++++++ cmd/alp/cmd/json_test.go | 25 ++++++ cmd/alp/cmd/ltsv.go | 33 ++++++++ cmd/alp/cmd/ltsv_test.go | 25 ++++++ cmd/alp/cmd/regexp.go | 36 ++++++++ cmd/alp/cmd/regexp_test.go | 26 ++++++ counter/counter.go | 13 +-- counter/printer.go | 159 +++++++++++------------------------ example/logs/ltsv_access.log | 28 +++--- internal/testutil/cmd.go | 26 ++++-- 14 files changed, 398 insertions(+), 272 deletions(-) delete mode 100644 cmd/alp/cmd/count_test.go diff --git a/cmd/alp/cmd/command.go b/cmd/alp/cmd/command.go index 82e9716..0228b4d 100644 --- a/cmd/alp/cmd/command.go +++ b/cmd/alp/cmd/command.go @@ -20,6 +20,8 @@ type Command struct { jsonDiffCmd *cobra.Command // alp json topN jsonTopNCmd *cobra.Command + // alp json count + jsonCountCmd *cobra.Command // alp ltsv ltsvCmd *cobra.Command @@ -27,6 +29,8 @@ type Command struct { ltsvDiffCmd *cobra.Command // alp ltsv topN ltsvTopNCmd *cobra.Command + // alp ltsv count + ltsvCountCmd *cobra.Command // alp regexp regexpCmd *cobra.Command @@ -34,6 +38,8 @@ type Command struct { regexpDiffCmd *cobra.Command // alp regexp topN regexpTopNCmd *cobra.Command + // alp regexp count + regexpCountCmd *cobra.Command // alp pcap pcapCmd *cobra.Command @@ -62,6 +68,9 @@ func NewCommand(version string) *Command { // alp ltsv topN command.ltsvTopNCmd = newLTSVTopNCmd(command.flags) command.ltsvCmd.AddCommand(command.ltsvTopNCmd) + // alp ltsv count + command.ltsvCountCmd = newLTSVCountCmd(command.flags) + command.ltsvCmd.AddCommand(command.ltsvCountCmd) // alp json command.jsonCmd = newJSONCmd(command.flags) @@ -72,6 +81,9 @@ func NewCommand(version string) *Command { // alp json topN command.jsonTopNCmd = newJsonTopNCmd(command.flags) command.jsonCmd.AddCommand(command.jsonTopNCmd) + // alp json count + command.jsonCountCmd = newJsonCountCmd(command.flags) + command.jsonCmd.AddCommand(command.jsonCountCmd) // alp regexp command.regexpCmd = newRegexpCmd(command.flags) @@ -82,6 +94,9 @@ func NewCommand(version string) *Command { // alp regexp topN command.regexpTopNCmd = newRegexpTopNCmd(command.flags) command.regexpCmd.AddCommand(command.regexpTopNCmd) + // alp regexp count + command.regexpCountCmd = newRegexpCountCmd(command.flags) + command.regexpCmd.AddCommand(command.regexpCountCmd) // alp pcap command.pcapCmd = newPcapCmd(command.flags) @@ -97,10 +112,6 @@ func NewCommand(version string) *Command { command.diffCmd = newDiffCmd(command.flags) command.rootCmd.AddCommand(command.diffCmd) - // alp count - command.countCmd = newCountCmd(command.flags) - command.rootCmd.AddCommand(command.countCmd) - return command } diff --git a/cmd/alp/cmd/count.go b/cmd/alp/cmd/count.go index a98d4da..9a9188c 100644 --- a/cmd/alp/cmd/count.go +++ b/cmd/alp/cmd/count.go @@ -1,89 +1,21 @@ package cmd import ( - "os" - - "github.com/tkuchiki/alp/options" - "github.com/spf13/cobra" "github.com/tkuchiki/alp/counter" + "github.com/tkuchiki/alp/options" "github.com/tkuchiki/alp/parsers" ) -func newCountCmd(flags *flags) *cobra.Command { - var countCmd = &cobra.Command{ +func newCountSubCmd() *cobra.Command { + return &cobra.Command{ Use: "count", Short: "Count by log entries", Long: `Count by log entries`, - RunE: func(cmd *cobra.Command, args []string) error { - opts, err := flags.createCountOptions(cmd) - if err != nil { - return err - } - - // TODO: start - // Remove these after implementing `alp (json|ltsv|regex) count`. - pattern, err := cmd.PersistentFlags().GetString("pattern") - if err != nil { - return err - } - - format, err := cmd.PersistentFlags().GetString("format") - if err != nil { - return err - } - - opts = options.SetOptions(opts, - options.Pattern(pattern), - options.Format(format), - ) - // TODO: end - - cnter := counter.NewCounter(os.Stdout, os.Stderr, opts.Reverse) - - f, err := cnter.Open(opts.File) - if err != nil { - return err - } - defer f.Close() - - var parser parsers.Parser - switch format { - case "json": - jsonKeys := parsers.NewJSONKeys(opts.JSON.UriKey, opts.JSON.MethodKey, opts.JSON.TimeKey, - opts.JSON.ResponseTimeKey, opts.JSON.RequestTimeKey, opts.JSON.BodyBytesKey, opts.JSON.StatusKey) - parser = parsers.NewJSONParser(f, jsonKeys, opts.QueryString, opts.QueryStringIgnoreValues) - case "ltsv": - label := parsers.NewLTSVLabel(opts.LTSV.UriLabel, opts.LTSV.MethodLabel, opts.LTSV.TimeLabel, - opts.LTSV.ApptimeLabel, opts.LTSV.ReqtimeLabel, opts.LTSV.SizeLabel, opts.LTSV.StatusLabel, - ) - parser = parsers.NewLTSVParser(f, label, opts.QueryString, opts.QueryStringIgnoreValues) - case "regexp": - names := parsers.NewSubexpNames(opts.Regexp.UriSubexp, opts.Regexp.MethodSubexp, opts.Regexp.TimeSubexp, - opts.Regexp.ResponseTimeSubexp, opts.Regexp.RequestTimeSubexp, opts.Regexp.BodyBytesSubexp, opts.Regexp.StatusSubexp) - parser, err = parsers.NewRegexpParser(f, opts.Regexp.Pattern, names, opts.QueryString, opts.QueryStringIgnoreValues) - if err != nil { - return err - } - } - - cnter.SetParser(parser) - - err = cnter.CountAndPrint(opts.Count.Keys) - - return err - }, } +} - flags.defineCountOptions(countCmd) - - // TODO: Remove these after implementing `alp (json|ltsv|regex) count`. - countCmd.PersistentFlags().StringP("pattern", "", options.DefaultPatternOption, "Regular expressions pattern matching the log") - countCmd.PersistentFlags().StringP("format", "", "json", "Log format (json,ltsv,regexp)") - - countCmd.Flags().SortFlags = false - countCmd.PersistentFlags().SortFlags = false - countCmd.InheritedFlags().SortFlags = false - - return countCmd +func runCount(counter *counter.Counter, parser parsers.Parser, opts *options.Options) error { + counter.SetParser(parser) + return counter.CountAndPrint(opts.Count.Keys) } diff --git a/cmd/alp/cmd/count_test.go b/cmd/alp/cmd/count_test.go deleted file mode 100644 index 74058cb..0000000 --- a/cmd/alp/cmd/count_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package cmd - -import ( - "testing" -) - -func TestCountCmd(t *testing.T) { - file := "../../../example/logs/json_access.log" - args := []string{"count", - "--file", file, - "--format", "json", - "--reverse", - "--keys", "ua", - } - - command := NewCommand("test") - command.setArgs(args) - - err := command.Execute() - if err != nil { - t.Fatal(err) - } -} diff --git a/cmd/alp/cmd/flags.go b/cmd/alp/cmd/flags.go index 16628fd..9cab9a8 100644 --- a/cmd/alp/cmd/flags.go +++ b/cmd/alp/cmd/flags.go @@ -332,12 +332,6 @@ func (f *flags) definePcapOptions(cmd *cobra.Command) { f.definePcapPcapServerPort(cmd) } -func (f *flags) defineCountOptions(cmd *cobra.Command) { - f.defineFile(cmd) - f.defineReverse(cmd) - f.defineCountKeys(cmd) -} - func (f *flags) defineDiffOptions(cmd *cobra.Command) { f.defineFormat(cmd) f.defineSort(cmd) @@ -416,6 +410,47 @@ func (f *flags) defineTopNSubCommandOptions(cmd *cobra.Command) { f.definePage(cmd) } +func (f *flags) defineCountSubCommandOptions(cmd *cobra.Command) { + // overwrite and hidden => remove flag + cmd.LocalFlags().String(flagDump, "", "") + cmd.LocalFlags().MarkHidden(flagDump) + cmd.LocalFlags().String(flagLoad, "", "") + cmd.LocalFlags().MarkHidden(flagLoad) + cmd.LocalFlags().String(flagSort, "", "") + cmd.LocalFlags().MarkHidden(flagSort) + cmd.LocalFlags().String(flagShowFooters, "", "") + cmd.LocalFlags().MarkHidden(flagShowFooters) + cmd.LocalFlags().String(flagLimit, "", "") + cmd.LocalFlags().MarkHidden(flagLimit) + cmd.LocalFlags().String(flagOutput, "", "") + cmd.LocalFlags().MarkHidden(flagOutput) + cmd.LocalFlags().String(flagQueryString, "", "") + cmd.LocalFlags().MarkHidden(flagQueryString) + cmd.LocalFlags().String(flagQueryStringIgnoreValues, "", "") + cmd.LocalFlags().MarkHidden(flagQueryStringIgnoreValues) + cmd.LocalFlags().String(flagLocation, "", "") + cmd.LocalFlags().MarkHidden(flagLocation) + cmd.LocalFlags().String(flagDecodeUri, "", "") + cmd.LocalFlags().MarkHidden(flagDecodeUri) + cmd.LocalFlags().String(flagMatchingGroups, "", "") + cmd.LocalFlags().MarkHidden(flagMatchingGroups) + cmd.LocalFlags().String(flagFilters, "", "") + cmd.LocalFlags().MarkHidden(flagFilters) + cmd.LocalFlags().String(flagPositionFile, "", "") + cmd.LocalFlags().MarkHidden(flagPositionFile) + cmd.LocalFlags().String(flagNoSavePositionFile, "", "") + cmd.LocalFlags().MarkHidden(flagNoSavePositionFile) + cmd.LocalFlags().String(flagPercentiles, "", "") + cmd.LocalFlags().MarkHidden(flagPercentiles) + + f.defineFile(cmd) + f.defineReverse(cmd) + f.defineFormat(cmd) + f.defineNoHeaders(cmd) + f.definePage(cmd) + f.defineCountKeys(cmd) +} + func (f *flags) bindFlags(cmd *cobra.Command) { viper.BindPFlag("file", cmd.PersistentFlags().Lookup(flagFile)) viper.BindPFlag("dump", cmd.PersistentFlags().Lookup(flagDump)) @@ -878,17 +913,6 @@ func (f *flags) setPcapOptions(cmd *cobra.Command, opts *options.Options) (*opti ), nil } -func (f *flags) setCountOptions(cmd *cobra.Command, opts *options.Options) (*options.Options, error) { - keys, err := cmd.PersistentFlags().GetString(flagCountKeys) - if err != nil { - return nil, err - } - - return options.SetOptions(opts, - options.CountKeys(helpers.SplitCSV(keys)), - ), nil -} - func (f *flags) setDiffOptions(cmd *cobra.Command, opts *options.Options) (*options.Options, error) { _flags := []string{ flagFormat, @@ -966,6 +990,27 @@ func (f *flags) setTopNSubCommandOptions(cmd *cobra.Command, opts *options.Optio return f.setOptions(cmd, opts, _flags) } +func (f *flags) setCountSubCommandOptions(cmd *cobra.Command, opts *options.Options) (*options.Options, error) { + keys, err := cmd.PersistentFlags().GetString(flagCountKeys) + if err != nil { + return nil, err + } + + opts = options.SetOptions(opts, + options.CountKeys(helpers.SplitCSV(keys)), + ) + + _flags := []string{ + flagFile, + flagFormat, + flagReverse, + flagNoHeaders, + flagPage, + } + + return f.setOptions(cmd, opts, _flags) +} + // alp json func (f *flags) createJSONOptions(cmd *cobra.Command) (*options.Options, error) { if f.config != "" { @@ -1011,6 +1056,21 @@ func (f *flags) createJSONTopNOptions(cmd *cobra.Command) (*options.Options, err return f.setJSONOptions(cmd, opts) } +// alp json count +func (f *flags) createJSONCountOptions(cmd *cobra.Command) (*options.Options, error) { + if f.config != "" { + f.bindFlags(cmd) + return f.createOptionsFromConfig(cmd) + } + + opts, err := f.setCountSubCommandOptions(cmd, options.NewOptions()) + if err != nil { + return nil, err + } + + return f.setJSONOptions(cmd, opts) +} + // alp ltsv func (f *flags) createLTSVOptions(cmd *cobra.Command) (*options.Options, error) { if f.config != "" { @@ -1056,6 +1116,21 @@ func (f *flags) createLTSVTopNOptions(cmd *cobra.Command) (*options.Options, err return f.setLTSVOptions(cmd, opts) } +// alp ltsv count +func (f *flags) createLTSVCountOptions(cmd *cobra.Command) (*options.Options, error) { + if f.config != "" { + f.bindFlags(cmd) + return f.createOptionsFromConfig(cmd) + } + + opts, err := f.setCountSubCommandOptions(cmd, options.NewOptions()) + if err != nil { + return nil, err + } + + return f.setLTSVOptions(cmd, opts) +} + // alp regexp func (f *flags) createRegexpOptions(cmd *cobra.Command) (*options.Options, error) { if f.config != "" { @@ -1101,29 +1176,29 @@ func (f *flags) createRegexpTopNOptions(cmd *cobra.Command) (*options.Options, e return f.setRegexpOptions(cmd, opts) } -// alp pcap -func (f *flags) createPcapOptions(cmd *cobra.Command) (*options.Options, error) { +// alp regexp count +func (f *flags) createRegexpCountOptions(cmd *cobra.Command) (*options.Options, error) { if f.config != "" { f.bindFlags(cmd) return f.createOptionsFromConfig(cmd) } - opts, err := f.setProfileOptions(cmd, options.NewOptions()) + opts, err := f.setCountSubCommandOptions(cmd, options.NewOptions()) if err != nil { return nil, err } - return f.setPcapOptions(cmd, opts) + return f.setRegexpOptions(cmd, opts) } -// alp pcap diff -func (f *flags) createPcapDiffOptions(cmd *cobra.Command) (*options.Options, error) { +// alp pcap +func (f *flags) createPcapOptions(cmd *cobra.Command) (*options.Options, error) { if f.config != "" { f.bindFlags(cmd) return f.createOptionsFromConfig(cmd) } - opts, err := f.setDiffSubCommandOptions(cmd, options.NewOptions()) + opts, err := f.setProfileOptions(cmd, options.NewOptions()) if err != nil { return nil, err } @@ -1131,14 +1206,14 @@ func (f *flags) createPcapDiffOptions(cmd *cobra.Command) (*options.Options, err return f.setPcapOptions(cmd, opts) } -// alp pcap topN -func (f *flags) createPcapTopNOptions(cmd *cobra.Command) (*options.Options, error) { +// alp pcap diff +func (f *flags) createPcapDiffOptions(cmd *cobra.Command) (*options.Options, error) { if f.config != "" { f.bindFlags(cmd) return f.createOptionsFromConfig(cmd) } - opts, err := f.setTopNSubCommandOptions(cmd, options.NewOptions()) + opts, err := f.setDiffSubCommandOptions(cmd, options.NewOptions()) if err != nil { return nil, err } @@ -1146,24 +1221,19 @@ func (f *flags) createPcapTopNOptions(cmd *cobra.Command) (*options.Options, err return f.setPcapOptions(cmd, opts) } -// alp count -func (f *flags) createCountOptions(cmd *cobra.Command) (*options.Options, error) { +// alp pcap topN +func (f *flags) createPcapTopNOptions(cmd *cobra.Command) (*options.Options, error) { if f.config != "" { f.bindFlags(cmd) return f.createOptionsFromConfig(cmd) } - flags := []string{ - flagFile, - flagReverse, - } - - opts, err := f.setOptions(cmd, options.NewOptions(), flags) + opts, err := f.setTopNSubCommandOptions(cmd, options.NewOptions()) if err != nil { return nil, err } - return f.setCountOptions(cmd, opts) + return f.setPcapOptions(cmd, opts) } // alp diff diff --git a/cmd/alp/cmd/json.go b/cmd/alp/cmd/json.go index eee0e31..236dc8a 100644 --- a/cmd/alp/cmd/json.go +++ b/cmd/alp/cmd/json.go @@ -3,6 +3,8 @@ package cmd import ( "os" + "github.com/tkuchiki/alp/counter" + "github.com/tkuchiki/alp/log_reader" "github.com/spf13/cobra" @@ -145,3 +147,34 @@ func newJsonTopNCmd(flags *flags) *cobra.Command { return jsonTopNCmd } + +func newJsonCountCmd(flags *flags) *cobra.Command { + jsonCountCmd := newCountSubCmd() + jsonCountCmd.RunE = func(cmd *cobra.Command, args []string) error { + opts, err := flags.createJSONCountOptions(cmd) + if err != nil { + return err + } + + counter := counter.NewCounter(os.Stdout, os.Stderr, opts) + + f, err := counter.Open(opts.File) + if err != nil { + return err + } + defer f.Close() + + parser := newJsonParser(opts, f) + + return runCount(counter, parser, opts) + } + + flags.defineCountSubCommandOptions(jsonCountCmd) + flags.defineJSONOptions(jsonCountCmd) + + jsonCountCmd.Flags().SortFlags = false + jsonCountCmd.PersistentFlags().SortFlags = false + jsonCountCmd.InheritedFlags().SortFlags = false + + return jsonCountCmd +} diff --git a/cmd/alp/cmd/json_test.go b/cmd/alp/cmd/json_test.go index 4949d67..3f7a33e 100644 --- a/cmd/alp/cmd/json_test.go +++ b/cmd/alp/cmd/json_test.go @@ -147,3 +147,28 @@ func TestJSONTopNCmd(t *testing.T) { t.Fatal(err) } } + +func TestJSONCountCmd(t *testing.T) { + keys := testutil.NewJsonLogKeys() + + jsonLog := testutil.JsonLog(keys) + + tempFile, err := testutil.CreateTempDirAndFile(t.TempDir(), "test_json_count_cmd_temp_file", jsonLog) + if err != nil { + t.Fatal(err) + } + + args := []string{"json", "count", + "--file", tempFile, + "--reverse", + "--keys", "ua", + } + + command := NewCommand("test") + command.setArgs(args) + + err = command.Execute() + if err != nil { + t.Fatal(err) + } +} diff --git a/cmd/alp/cmd/ltsv.go b/cmd/alp/cmd/ltsv.go index c99d177..3ac37b4 100644 --- a/cmd/alp/cmd/ltsv.go +++ b/cmd/alp/cmd/ltsv.go @@ -3,6 +3,8 @@ package cmd import ( "os" + "github.com/tkuchiki/alp/counter" + "github.com/tkuchiki/alp/log_reader" "github.com/spf13/cobra" @@ -146,3 +148,34 @@ func newLTSVTopNCmd(flags *flags) *cobra.Command { return ltsvTopNCmd } + +func newLTSVCountCmd(flags *flags) *cobra.Command { + ltsvCountCmd := newCountSubCmd() + ltsvCountCmd.RunE = func(cmd *cobra.Command, args []string) error { + opts, err := flags.createLTSVCountOptions(cmd) + if err != nil { + return err + } + + counter := counter.NewCounter(os.Stdout, os.Stderr, opts) + + f, err := counter.Open(opts.File) + if err != nil { + return err + } + defer f.Close() + + parser := newLTSVParser(opts, f) + + return runCount(counter, parser, opts) + } + + flags.defineCountSubCommandOptions(ltsvCountCmd) + flags.defineLTSVOptions(ltsvCountCmd) + + ltsvCountCmd.Flags().SortFlags = false + ltsvCountCmd.PersistentFlags().SortFlags = false + ltsvCountCmd.InheritedFlags().SortFlags = false + + return ltsvCountCmd +} diff --git a/cmd/alp/cmd/ltsv_test.go b/cmd/alp/cmd/ltsv_test.go index e99832b..6ed2af4 100644 --- a/cmd/alp/cmd/ltsv_test.go +++ b/cmd/alp/cmd/ltsv_test.go @@ -147,3 +147,28 @@ func TestLTSVTopNCmd(t *testing.T) { t.Fatal(err) } } + +func TestLTSVCountCmd(t *testing.T) { + keys := testutil.NewLTSVLogKeys() + + ltsvLog := testutil.LTSVLog(keys) + + tempFile, err := testutil.CreateTempDirAndFile(t.TempDir(), "test_ltsv_count_cmd_temp_file", ltsvLog) + if err != nil { + t.Fatal(err) + } + + args := []string{"ltsv", "count", + "--file", tempFile, + "--reverse", + "--keys", "ua", + } + + command := NewCommand("test") + command.setArgs(args) + + err = command.Execute() + if err != nil { + t.Fatal(err) + } +} diff --git a/cmd/alp/cmd/regexp.go b/cmd/alp/cmd/regexp.go index df9d146..81fe963 100644 --- a/cmd/alp/cmd/regexp.go +++ b/cmd/alp/cmd/regexp.go @@ -3,6 +3,8 @@ package cmd import ( "os" + "github.com/tkuchiki/alp/counter" + "github.com/tkuchiki/alp/log_reader" "github.com/spf13/cobra" @@ -156,3 +158,37 @@ func newRegexpTopNCmd(flags *flags) *cobra.Command { return regexpTopNCmd } + +func newRegexpCountCmd(flags *flags) *cobra.Command { + regexpCountCmd := newCountSubCmd() + regexpCountCmd.RunE = func(cmd *cobra.Command, args []string) error { + opts, err := flags.createRegexpCountOptions(cmd) + if err != nil { + return err + } + + counter := counter.NewCounter(os.Stdout, os.Stderr, opts) + + f, err := counter.Open(opts.File) + if err != nil { + return err + } + defer f.Close() + + parser, err := newRegexpParser(opts, f) + if err != nil { + return err + } + + return runCount(counter, parser, opts) + } + + flags.defineCountSubCommandOptions(regexpCountCmd) + flags.defineRegexpOptions(regexpCountCmd) + + regexpCountCmd.Flags().SortFlags = false + regexpCountCmd.PersistentFlags().SortFlags = false + regexpCountCmd.InheritedFlags().SortFlags = false + + return regexpCountCmd +} diff --git a/cmd/alp/cmd/regexp_test.go b/cmd/alp/cmd/regexp_test.go index 679112a..d4b7380 100644 --- a/cmd/alp/cmd/regexp_test.go +++ b/cmd/alp/cmd/regexp_test.go @@ -139,3 +139,29 @@ func TestRegexpTopNCmd(t *testing.T) { t.Fatal(err) } } + +func TestRegexpCountCmd(t *testing.T) { + keys := testutil.NewRegexpLogKeys() + + regexpLog := testutil.RegexpLog() + + tempFile, err := testutil.CreateTempDirAndFile(t.TempDir(), "test_regexp_count_cmd_temp_file", regexpLog) + if err != nil { + t.Fatal(err) + } + + args := []string{"regexp", "count", + "--pattern", testutil.RegexpPattern(keys), + "--file", tempFile, + "--reverse", + "--keys", "ua", + } + + command := NewCommand("test") + command.setArgs(args) + + err = command.Execute() + if err != nil { + t.Fatal(err) + } +} diff --git a/counter/counter.go b/counter/counter.go index 35649cc..47735f3 100644 --- a/counter/counter.go +++ b/counter/counter.go @@ -7,6 +7,8 @@ import ( "strings" "sync" + "github.com/tkuchiki/alp/options" + "github.com/tkuchiki/alp/errors" "github.com/tkuchiki/alp/parsers" ) @@ -18,17 +20,18 @@ type Counter struct { parser parsers.Parser groups *groups printer *Printer - reverse bool + options *options.Options } -func NewCounter(outw, errw io.Writer, reverse bool) *Counter { +func NewCounter(outw, errw io.Writer, opts *options.Options) *Counter { + printOptions := NewPrintOptions(opts.NoHeaders, false, opts.PaginationLimit) return &Counter{ outWriter: outw, errWriter: errw, inReader: os.Stdin, - printer: NewPrinter(outw), + printer: NewPrinter(outw, opts.Format, printOptions), groups: newGroups(), - reverse: reverse, + options: opts, } } @@ -94,7 +97,7 @@ Loop: } func (c *Counter) Sort() { - if c.reverse { + if c.options.Reverse { sort.Slice(c.groups.groups, func(i, j int) bool { return c.groups.groups[i].count > c.groups.groups[j].count }) diff --git a/counter/printer.go b/counter/printer.go index 884f699..ca6fdca 100644 --- a/counter/printer.go +++ b/counter/printer.go @@ -3,6 +3,9 @@ package counter import ( "fmt" "io" + "strings" + + "github.com/tkuchiki/alp/html" "github.com/olekukonko/tablewriter" ) @@ -31,10 +34,11 @@ func NewPrintOptions(noHeaders, showFooters bool, paginationLimit int) *PrintOpt } } -func NewPrinter(w io.Writer) *Printer { +func NewPrinter(w io.Writer, format string, printOptions *PrintOptions) *Printer { return &Printer{ - format: "table", - writer: w, + format: format, + writer: w, + printOptions: printOptions, } } @@ -43,14 +47,14 @@ func (p *Printer) Print(groups *groups) { switch p.format { case "table": p.printTable(groups) - /*case "md", "markdown": - p.printMarkdown(groups) - case "tsv": - p.printTSV(groups) - case "csv": - p.printCSV(groups) - case "html": - p.printHTML(groups)*/ + case "md", "markdown": + p.printMarkdown(groups) + case "tsv": + p.printTSV(groups) + case "csv": + p.printCSV(groups) + case "html": + p.printHTML(groups) } } @@ -76,130 +80,69 @@ func (p *Printer) printTable(groups *groups) { table.Append(data) } - /* - if p.printOptions.showFooters { - var footer []string - if hsTo == nil { - footer = p.GenerateFooter(hsFrom.CountAll()) - } else { - footer = p.GenerateFooterWithDiff(hsFrom.CountAll(), hsTo.CountAll()) - } - table.SetFooter(footer) - table.SetFooterAlignment(tablewriter.ALIGN_LEFT) - }*/ - table.SetAlignment(tablewriter.ALIGN_LEFT) table.Render() } -/* -func (p *Printer) printMarkdown(groups) { +func (p *Printer) printMarkdown(groups *groups) { table := tablewriter.NewWriter(p.writer) - table.SetHeader(p.headers) + var headers []string + headers = append(headers, defaultSumHeader) + headers = append(headers, groups.keys...) + + table.SetHeader(headers) table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) table.SetCenterSeparator("|") - if hsTo == nil { - for _, s := range hsFrom.stats { - data := p.GenerateLine(s, false) - table.Append(data) - } - } else { - for _, to := range hsTo.stats { - from := findHTTPStatFrom(hsFrom, to) - - var data []string - if from == nil { - data = p.GenerateLine(to, false) - } else { - data = p.GenerateLineWithDiff(from, to, false) - } - table.Append(data) - } - } - if p.printOptions.showFooters { - var footer []string - if hsTo == nil { - footer = p.GenerateFooter(hsFrom.CountAll()) - } else { - footer = p.GenerateFooterWithDiff(hsFrom.CountAll(), hsTo.CountAll()) - } - table.Append(footer) + for _, group := range groups.groups { + data := p.generateLine(groups.keys, group) + table.Append(data) } table.SetAlignment(tablewriter.ALIGN_LEFT) table.Render() } -func (p *Printer) printTSV(groups) { +func (p *Printer) printTSV(groups *groups) { + var headers []string + headers = append(headers, defaultSumHeader) + headers = append(headers, groups.keys...) + if !p.printOptions.noHeaders { - fmt.Println(strings.Join(p.headers, "\t")) + fmt.Println(strings.Join(headers, "\t")) } - var data []string - if hsTo == nil { - for _, s := range hsFrom.stats { - data = p.GenerateLine(s, false) - fmt.Println(strings.Join(data, "\t")) - } - } else { - for _, to := range hsTo.stats { - from := findHTTPStatFrom(hsFrom, to) - - if from == nil { - data = p.GenerateLine(to, false) - } else { - data = p.GenerateLineWithDiff(from, to, false) - } - fmt.Println(strings.Join(data, "\t")) - } + for _, group := range groups.groups { + data := p.generateLine(groups.keys, group) + fmt.Println(strings.Join(data, "\t")) } } -func (p *Printer) printCSV(groups) { +func (p *Printer) printCSV(groups *groups) { + var headers []string + headers = append(headers, defaultSumHeader) + headers = append(headers, groups.keys...) + if !p.printOptions.noHeaders { - fmt.Println(strings.Join(p.headers, ",")) + fmt.Println(strings.Join(headers, ",")) } - var data []string - if hsTo == nil { - for _, s := range hsFrom.stats { - data = p.GenerateLine(s, true) - fmt.Println(strings.Join(data, ",")) - } - } else { - for _, to := range hsTo.stats { - from := findHTTPStatFrom(hsFrom, to) - - if from == nil { - data = p.GenerateLine(to, false) - } else { - data = p.GenerateLineWithDiff(from, to, false) - } - fmt.Println(strings.Join(data, ",")) - } + for _, group := range groups.groups { + data := p.generateLine(groups.keys, group) + fmt.Println(strings.Join(data, ",")) } } -func (p *Printer) printHTML(groups) { - var data [][]string +func (p *Printer) printHTML(groups *groups) { + var headers []string + headers = append(headers, defaultSumHeader) + headers = append(headers, groups.keys...) - if hsTo == nil { - for _, s := range hsFrom.stats { - data = append(data, p.GenerateLine(s, true)) - } - } else { - for _, to := range hsTo.stats { - from := findHTTPStatFrom(hsFrom, to) - - if from == nil { - data = append(data, p.GenerateLine(to, false)) - } else { - data = append(data, p.GenerateLineWithDiff(from, to, false)) - } - } + var data [][]string + for _, group := range groups.groups { + data = append(data, p.generateLine(groups.keys, group)) } - content, _ := html.RenderTableWithGridJS("alp", p.headers, data, p.printOptions.paginationLimit) + + content, _ := html.RenderTableWithGridJS("alp", headers, data, p.printOptions.paginationLimit) fmt.Println(content) } -*/ diff --git a/example/logs/ltsv_access.log b/example/logs/ltsv_access.log index 45ea7d3..53320e5 100644 --- a/example/logs/ltsv_access.log +++ b/example/logs/ltsv_access.log @@ -1,14 +1,14 @@ -time:2015-09-06T05:58:05+09:00 method:POST uri:/foo/bar?token=xxx&uuid=1234 status:200 size:12 apptime:0.057 -time:2015-09-06T05:58:41+09:00 method:POST uri:/foo/bar?token=yyy status:200 size:34 apptime:0.100 -time:2015-09-06T06:00:42+09:00 method:GET uri:/foo/bar?token=zzz status:200 size:56 apptime:0.123 -time:2015-09-06T06:00:43+09:00 method:GET uri:/foo/bar status:400 size:15 apptime:- -time:2015-09-06T05:58:44+09:00 method:POST uri:/foo/bar?token=yyy status:200 size:34 apptime:0.234 -time:2015-09-06T05:58:44+09:00 method:POST uri:/hoge/piyo?id=yyy status:200 size:34 apptime:0.234 -time:2015-09-06T05:58:05+09:00 method:POST uri:/foo/bar?token=xxx&uuid=1234 status:200 size:12 apptime:0.057 -time:2015-09-06T05:58:41+09:00 method:POST uri:/foo/bar?token=yyy status:200 size:34 apptime:0.100 -time:2015-09-06T06:00:42+09:00 method:GET uri:/foo/bar?token=zzz status:200 size:56 apptime:0.123 -time:2015-09-06T06:00:43+09:00 method:GET uri:/foo/bar status:400 size:15 apptime:- -time:2015-09-06T06:00:43+09:00 method:GET uri:/diary/entry/1234 status:200 size:15 apptime:0.135 -time:2015-09-06T06:00:43+09:00 method:GET uri:/diary/entry/5678 status:200 size:30 apptime:0.432 -time:2015-09-06T06:00:43+09:00 method:GET uri:/foo/bar/5xx status:504 size:15 apptime:60.000 -time:2015-09-06T06:00:43+09:00 method:GET uri:/req status:200 size:15 apptime:- reqtime:0.321 +time:2015-09-06T05:58:05+09:00 method:POST uri:/foo/bar?token=xxx&uuid=1234 status:200 size:12 apptime:0.057 ua:UA1 +time:2015-09-06T05:58:41+09:00 method:POST uri:/foo/bar?token=yyy status:200 size:34 apptime:0.100 ua:UA1 +time:2015-09-06T06:00:42+09:00 method:GET uri:/foo/bar?token=zzz status:200 size:56 apptime:0.123 ua:UA3 +time:2015-09-06T06:00:43+09:00 method:GET uri:/foo/bar status:400 size:15 apptime:- ua:UA2 +time:2015-09-06T05:58:44+09:00 method:POST uri:/foo/bar?token=yyy status:200 size:34 apptime:0.234 ua:UA4 +time:2015-09-06T05:58:44+09:00 method:POST uri:/hoge/piyo?id=yyy status:200 size:34 apptime:0.234 ua:UA2 +time:2015-09-06T05:58:05+09:00 method:POST uri:/foo/bar?token=xxx&uuid=1234 status:200 size:12 apptime:0.057 ua:UA1 +time:2015-09-06T05:58:41+09:00 method:POST uri:/foo/bar?token=yyy status:200 size:34 apptime:0.100 ua:UA3 +time:2015-09-06T06:00:42+09:00 method:GET uri:/foo/bar?token=zzz status:200 size:56 apptime:0.123 ua:UA1 +time:2015-09-06T06:00:43+09:00 method:GET uri:/foo/bar status:400 size:15 apptime:- ua:UA3 +time:2015-09-06T06:00:43+09:00 method:GET uri:/diary/entry/1234 status:200 size:15 apptime:0.135 ua:UA4 +time:2015-09-06T06:00:43+09:00 method:GET uri:/diary/entry/5678 status:200 size:30 apptime:0.432 ua:UA1 +time:2015-09-06T06:00:43+09:00 method:GET uri:/foo/bar/5xx status:504 size:15 apptime:60.000 ua:UA1 +time:2015-09-06T06:00:43+09:00 method:GET uri:/req status:200 size:15 apptime:- reqtime:0.321 ua:UA2 diff --git a/internal/testutil/cmd.go b/internal/testutil/cmd.go index 0a5ff9b..74b9753 100644 --- a/internal/testutil/cmd.go +++ b/internal/testutil/cmd.go @@ -45,6 +45,18 @@ func NewLTSVLogKeys() LogKeys { } } +func NewRegexpLogKeys() LogKeys { + return LogKeys{ + Uri: "uri", + Method: "method", + Time: "time", + ResponseTime: "response_time", + RequestTime: "request_time", + BodyBytes: "body_bytes", + Status: "status", + } +} + func CreateTempDirAndFile(dir, filename, content string) (string, error) { fpath := filepath.Join(dir, filename) err := os.WriteFile(fpath, []byte(content), 0644) @@ -61,8 +73,8 @@ func JsonLog(keys LogKeys) string { "__response_time__", keys.ResponseTime, "__request_time__", keys.RequestTime) - return r.Replace(`{"__time__":"2015-09-06T05:58:05+09:00","__method__":"POST","__uri__":"/foo/bar/123?token=xxx&uuid=1234","__status__":200,"__body_bytes__":12,"__response_time__":0.057,"__request_time__":0.057} -{"__time__":"2015-09-06T05:58:05+09:00","__method__":"POST","__uri__":"/foo/bar/456?token=yyy","__status__":200,"__body_bytes__":34,"__response_time__":0.100,"__request_time__":0.100}`) + return r.Replace(`{"__time__":"2015-09-06T05:58:05+09:00","__method__":"POST","__uri__":"/foo/bar/123?token=xxx&uuid=1234","__status__":200,"__body_bytes__":12,"__response_time__":0.057,"__request_time__":0.057,"ua":"UA1"} +{"__time__":"2015-09-06T05:58:05+09:00","__method__":"POST","__uri__":"/foo/bar/456?token=yyy","__status__":200,"__body_bytes__":34,"__response_time__":0.100,"__request_time__":0.100,"ua":"UA2"}`) } func LTSVLog(keys LogKeys) string { @@ -74,14 +86,14 @@ func LTSVLog(keys LogKeys) string { "__apptime__", keys.ResponseTime, "__reqtime__", keys.RequestTime) - return r.Replace(`__time__:2015-09-06T05:58:05+09:00 __method__:POST __uri__:/foo/bar/123?token=xxx&uuid=1234 __status__:200 __size__:12 __apptime__:0.057 __reqtime__:0.0057 -__time__:2015-09-06T05:58:41+09:00 __method__:POST __uri__:/foo/bar/456?token=yyy __status__:200 __size__:34 __apptime__:0.100 __reqtime__:0.100 + return r.Replace(`__time__:2015-09-06T05:58:05+09:00 __method__:POST __uri__:/foo/bar/123?token=xxx&uuid=1234 __status__:200 __size__:12 __apptime__:0.057 __reqtime__:0.0057 ua:UA1 +__time__:2015-09-06T05:58:41+09:00 __method__:POST __uri__:/foo/bar/456?token=yyy __status__:200 __size__:34 __apptime__:0.100 __reqtime__:0.100 ua:UA2 `) } func RegexpLog() string { - return `127.0.0.1 - - [06/Sep/2015:05:58:05 +0900] "POST /foo/bar/123?token=xxx&uuid=1234 HTTP/1.1" 200 12 "-" "curl/7.54.0" "-" 0.057 -127.0.0.1 - - [06/Sep/2015:05:58:41 +0900] "POST /foo/bar/456?token=yyy HTTP/1.1" 200 34 "-" "curl/7.54.0" "-" 0.100` + return `127.0.0.1 - - [06/Sep/2015:05:58:05 +0900] "POST /foo/bar/123?token=xxx&uuid=1234 HTTP/1.1" 200 12 "-" "UA1" "-" 0.057 +127.0.0.1 - - [06/Sep/2015:05:58:41 +0900] "POST /foo/bar/456?token=yyy HTTP/1.1" 200 34 "-" "UA2" "-" 0.100` } func RegexpPattern(keys LogKeys) string { @@ -102,7 +114,7 @@ func RegexpPattern(keys LogKeys) string { `(?P<__status__>\S+)\s` + // status code `(?P<__body_bytes__>\S+)\s` + // bytes `"((?:[^"]*(?:\\")?)*)"\s` + // referer - `"(?:.+)"` + // user agent + `"(?P[^"]*)"` + // user agent `\s(?P<__response_time__>\S+)(?:\s(?P<__request_time__>\S+))?$`) }