From 3f10d6905d3c469c7f3eb32204ec386cee07bd7a Mon Sep 17 00:00:00 2001 From: Dmitry Mozzherin Date: Sat, 22 Jan 2022 09:19:20 -0600 Subject: [PATCH] enable NSQ log aggregation (close #218) --- README.md | 46 +++++++++++++++++++-- config.go | 94 ++++++++++++++++++++++++++++--------------- gnparser.go | 9 +++++ gnparser/cmd/flags.go | 86 +++++++++++++++++++++++---------------- gnparser/cmd/root.go | 30 +++++++++----- go.mod | 6 ++- go.sum | 8 +++- interface.go | 31 ++++++++------ io/web/server.go | 31 ++++++++++++-- 9 files changed, 239 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index bbe711a..372fb88 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,8 @@ gnparser -h * [R language package](#r-language-package) * [Ruby Gem](#ruby-gem) * [Node.js](#nodejs) - * [Usage as a REST API Interface](#usage-as-a-rest-api-interface) + * [Usage as a REST API Interface or Web-based User Graphical Interface](#usage-as-a-rest-api-interface-or-web-based-user-graphical-interface) + * [Enabling logs for GNparser's web-service](#enabling-logs-for-gnparsers-web-service) * [Use as a Docker image](#use-as-a-docker-image) * [Use as a library in Go](#use-as-a-library-in-go) * [Use as a shared C library](#use-as-a-shared-c-library) @@ -353,7 +354,7 @@ Relevant flags: This flag is ignored if parsing mode is set to streaming with ``-s`` flag. ``--cultivars -C`` -: Adds support for botanical cultivars like ``Sarracenia flava 'Maxima'`` +: Adds support for botanical cultivars like ``Sarracenia flava 'Maxima'`` and graft-chimaeras like ``+ Crataegomespilus`` ``--capitalize -c`` @@ -385,6 +386,15 @@ performance. ``--port -p`` : set a port to run web-interface and [RESTful API][OpenAPI]. +`` --web-logs`` +: requires `--port`. Enables output of logs for web-services. + +`` --nsqd-tcp`` +: requires `--port`. Allows to redirect web-service log output to [NSQ] + messaging server's TCP-based endpoint. It is handy for aggregations of logs + from GNparser web-services running inside of Docker containers or + in Kubernetes pods. + ``--stream -s`` : ``GNparser`` can be used from any language using pipe-in/pipe-out of the command line application. This approach requires sending 1 name at a time @@ -509,7 +519,7 @@ uses C-binding and does not require an installed `gnparser` app. @tobymarsden created a [wrapper for node.js][node-gnparser]. It uses C-binding and does not require an installed `gnparser` app. -### Usage as a REST API Interface +### Usage as a REST API Interface or Web-based User Graphical Interface Web-based user interface and API are invoked by ``--port`` or ``-p`` flag. To start web server on ``http://0.0.0.0:9000`` @@ -543,6 +553,33 @@ request.body = ['Solanum mariae Särkinen & S.Knapp', response = http.request(request) ``` +#### Enabling logs for GNparser's web-service + +There are several ways to enable logging from a web service. + +The following enables web-access logs to be printed to STDERR + +``` +gnparser -p 80 --web-logs +``` + +This next settings allows to send logs to a [NSQ] messaging service. +This option allows aggregation logs from several instances of GNparser together. +It is a great way for log aggregation and analysis if the instances run +inside Docker containers or as Kubernetes Pods. + +``` +gnparser -p 80 --nsqd-tcp=127.0.0.1:4150 +``` + +An **important note**: the address must point to the TCP service of nsqd. + +To enable logs to be sent to STDERR and [NSQ] run + +``` +gnparser -p 80 --web-logs --nsqd-tcp=127.0.0.1:4150 +``` + ### Use as a Docker image You need to have [docker runtime installed](https://docs.docker.com/install/) @@ -550,7 +587,7 @@ on your computer for these examples to work. ```bash # run as a website and a RESTful service -docker run -p 0.0.0.0:80:8080 gnames/gognparser -p 8080 +docker run -p 0.0.0.0:80:8080 gnames/gognparser -p 8080 --web-logs # just parse something docker run gnames/gognparser "Amaurorhinus bewichianus (Wollaston,1860) (s.str.)" @@ -650,6 +687,7 @@ Released under [MIT license] [CONTRIBUTING]: https://github.com/gnames/gnparser/blob/master/CONTRIBUTING.md [Dmitry Mozzherin]: https://github.com/dimus [Geoff Ower]: https://github.com/gdower +[NSQ]: https://nsq.io/overview/quick_start.html [Toby Marsden]: https://github.com/tobymarsden [Hernan Lucas Pereira]: https://github.com/LocoDelAssembly [Homebrew]: https://brew.sh/ diff --git a/config.go b/config.go index 8a02436..868b6b9 100644 --- a/config.go +++ b/config.go @@ -10,26 +10,52 @@ import ( // Config keeps settings that might affect how parsing is done, // of change the parsing output. type Config struct { + // BatchSize sets the maximum number of elements in names-strings slice. + BatchSize int + + // Debug sets a "debug" state for parsing. The debug state forces output + // format to showing parsed ast tree. + Debug bool + // Format sets the output format for CLI and Web interfaces. // There are 3 formats available: 'CSV', 'CompactJSON' and // 'PrettyJSON'. Format gnfmt.Format + // IgnoreHTMLTags can be set to true when it is desirable to clean up names + // from a few HTML tags often present in names-strings that were planned to + // be presented via an HTML page. + IgnoreHTMLTags bool + + // IsTest can be set to true when parsing functionality is used for tests. + // In such cases the `ParserVersion` field is presented as `test_version` + // instead of displaying the actual version of `gnparser`. + IsTest bool + // JobsNum sets a level of parallelism used during parsing of // a stream of name-strings. JobsNum int - // BatchSize sets the maximum number of elements in names-strings slice. - BatchSize int + // Port to run wer-service. + Port int - // WithStream changes from parsing a batch by batch, to parsing one name - // at a time. When WithStream is true, BatchSize setting is ignored. - WithStream bool + // WebLogsNsqdTCP provides an address to the NSQ messenger TCP service. If + // this value is set and valid, the web logs will be published to the NSQ. + // The option is ignored if `Port` is not set. + // + // If WithWebLogs option is set to `false`, but `WebLogsNsqdTCP` is set to a + // valid URL, the logs will be sent to the NSQ messanging service, but they + // wil not appear as STRERR output. + // Example: `127.0.0.1:4150` + WebLogsNsqdTCP string - // IgnoreHTMLTags can be set to true when it is desirable to clean up names - // from a few HTML tags often present in names-strings that were planned to - // be presented via an HTML page. - IgnoreHTMLTags bool + // WithCapitalization flag, when true, the first letter of a name-string + // is capitalized, if appropriate. + WithCapitalization bool + + // WithCultivars flag, when true, cultivar names will be parsed and + // modify cardinality, normalized and canonical output. + WithCultivars bool // WithDetails can be set to true when a simplified output is not sufficient // for obtaining a required information. @@ -38,28 +64,16 @@ type Config struct { // WithNoOrder flag, when true, output and input are in different order. WithNoOrder bool - // WithCapitalization flag, when true, the first letter of a name-string - // is capitalized, if appropriate. - WithCapitalization bool - // WithPreserveDiaereses flag, when true, diaereses will not be transliterated WithPreserveDiaereses bool - // WithCultivars flag, when true, cultivar names will be parsed and - // modify cardinality, normalized and canonical output. - WithCultivars bool - - // Port to run wer-service. - Port int - - // IsTest can be set to true when parsing functionality is used for tests. - // In such cases the `ParserVersion` field is presented as `test_version` - // instead of displaying the actual version of `gnparser`. - IsTest bool + // WithStream changes from parsing a batch by batch, to parsing one name + // at a time. When WithStream is true, BatchSize setting is ignored. + WithStream bool - // Debug sets a "debug" state for parsing. The debug state forces output - // format to showing parsed ast tree. - Debug bool + // WithWebLogs flag enables logs when running web-service. This flag is + // ignored if `Port` value is not set. + WithWebLogs bool } // Option is a type that has to be returned by all Option functions. Such @@ -129,17 +143,17 @@ func OptPort(i int) Option { } } -// OptWithCapitaliation sets the WithCapitalization field. -func OptWithCapitaliation(b bool) Option { +// OptWebLogsNsqdTCP provides a URL to NSQ messanging service. +func OptWebLogsNsqdTCP(s string) Option { return func(cfg *Config) { - cfg.WithCapitalization = b + cfg.WebLogsNsqdTCP = s } } -// OptPreserveDiaereses sets the PreserveDiaereses field. -func OptWithPreserveDiaereses(b bool) Option { +// OptWithCapitaliation sets the WithCapitalization field. +func OptWithCapitaliation(b bool) Option { return func(cfg *Config) { - cfg.WithPreserveDiaereses = b + cfg.WithCapitalization = b } } @@ -164,6 +178,13 @@ func OptWithNoOrder(b bool) Option { } } +// OptWithPreserveDiaereses sets the PreserveDiaereses field. +func OptWithPreserveDiaereses(b bool) Option { + return func(cfg *Config) { + cfg.WithPreserveDiaereses = b + } +} + // OptWithDetails sets the WithDetails field. func OptWithStream(b bool) Option { return func(cfg *Config) { @@ -171,6 +192,13 @@ func OptWithStream(b bool) Option { } } +// OptWithWebLogs sets the WithWebLogs field. +func OptWithWebLogs(b bool) Option { + return func(cfg *Config) { + cfg.WithWebLogs = b + } +} + // NewConfig generates a new Config object. It can take an arbitrary number // of `Option` functions to modify default configuration settings. func NewConfig(opts ...Option) Config { diff --git a/gnparser.go b/gnparser.go index 9576fe3..db36e37 100644 --- a/gnparser.go +++ b/gnparser.go @@ -104,6 +104,15 @@ func (gnp gnparser) Format() gnfmt.Format { return gnp.cfg.Format } +// WebLogs returns a boolean to show or not the web-service logs. +func (gnp gnparser) WebLogs() bool { + return gnp.cfg.WithWebLogs +} + +func (gnp gnparser) WebLogsNsqdTCP() string { + return gnp.cfg.WebLogsNsqdTCP +} + // ChangeConfig allows change configuration of already created // GNparser object. func (gnp gnparser) ChangeConfig(opts ...Option) GNparser { diff --git a/gnparser/cmd/flags.go b/gnparser/cmd/flags.go index 7d39a24..26fd269 100644 --- a/gnparser/cmd/flags.go +++ b/gnparser/cmd/flags.go @@ -9,17 +9,15 @@ import ( "github.com/spf13/cobra" ) -func versionFlag(cmd *cobra.Command) bool { - version, err := cmd.Flags().GetBool("version") +func batchSizeFlag(cmd *cobra.Command) { + bs, err := cmd.Flags().GetInt("batch_size") if err != nil { - log.Fatal(err) + fmt.Println(err) + os.Exit(1) } - if version { - fmt.Printf("\nversion: %s\n\nbuild: %s\n\n", - gnparser.Version, gnparser.Build) - return true + if bs > 0 { + opts = append(opts, gnparser.OptBatchSize(bs)) } - return false } func formatFlag(cmd *cobra.Command) { @@ -55,26 +53,38 @@ func ignoreHTMLTagsFlag(cmd *cobra.Command) { } } -func withDetailsFlag(cmd *cobra.Command) { - withDet, err := cmd.Flags().GetBool("details") +func portFlag(cmd *cobra.Command) int { + webPort, err := cmd.Flags().GetInt("port") if err != nil { fmt.Println(err) os.Exit(1) } - if withDet { - opts = append(opts, gnparser.OptWithDetails(true)) + if webPort > 0 { + opts = append(opts, gnparser.OptPort(webPort)) } + return webPort } -func withNoOrderFlag(cmd *cobra.Command) { - withOrd, err := cmd.Flags().GetBool("unordered") +func versionFlag(cmd *cobra.Command) bool { + version, err := cmd.Flags().GetBool("version") + if err != nil { + log.Fatal(err) + } + if version { + fmt.Printf("\nversion: %s\n\nbuild: %s\n\n", + gnparser.Version, gnparser.Build) + return true + } + return false +} + +func webLogsNsqdTCPFlag(cmd *cobra.Command) string { + u, err := cmd.Flags().GetString("nsqd-tcp") if err != nil { fmt.Println(err) os.Exit(1) } - if withOrd { - opts = append(opts, gnparser.OptWithNoOrder(true)) - } + return u } func withCapitalizeFlag(cmd *cobra.Command) { @@ -88,14 +98,14 @@ func withCapitalizeFlag(cmd *cobra.Command) { } } -func withPreserveDiaeresesFlag(cmd *cobra.Command) { - b, err := cmd.Flags().GetBool("diaereses") +func withDetailsFlag(cmd *cobra.Command) { + withDet, err := cmd.Flags().GetBool("details") if err != nil { fmt.Println(err) os.Exit(1) } - if b { - opts = append(opts, gnparser.OptWithPreserveDiaereses(true)) + if withDet { + opts = append(opts, gnparser.OptWithDetails(true)) } } @@ -110,36 +120,44 @@ func withEnableCultivarsFlag(cmd *cobra.Command) { } } -func withStreamFlag(cmd *cobra.Command) { - withDet, err := cmd.Flags().GetBool("stream") +func withNoOrderFlag(cmd *cobra.Command) { + withOrd, err := cmd.Flags().GetBool("unordered") if err != nil { fmt.Println(err) os.Exit(1) } - if withDet { - opts = append(opts, gnparser.OptWithStream(true)) + if withOrd { + opts = append(opts, gnparser.OptWithNoOrder(true)) } } -func batchSizeFlag(cmd *cobra.Command) { - bs, err := cmd.Flags().GetInt("batch_size") +func withPreserveDiaeresesFlag(cmd *cobra.Command) { + b, err := cmd.Flags().GetBool("diaereses") if err != nil { fmt.Println(err) os.Exit(1) } - if bs > 0 { - opts = append(opts, gnparser.OptBatchSize(bs)) + if b { + opts = append(opts, gnparser.OptWithPreserveDiaereses(true)) } } -func portFlag(cmd *cobra.Command) int { - webPort, err := cmd.Flags().GetInt("port") +func withStreamFlag(cmd *cobra.Command) { + withDet, err := cmd.Flags().GetBool("stream") if err != nil { fmt.Println(err) os.Exit(1) } - if webPort > 0 { - opts = append(opts, gnparser.OptPort(webPort)) + if withDet { + opts = append(opts, gnparser.OptWithStream(true)) } - return webPort +} + +func withWebLogsFlag(cmd *cobra.Command) bool { + withLogs, err := cmd.Flags().GetBool("web-logs") + if err != nil { + fmt.Println(err) + os.Exit(1) + } + return withLogs } diff --git a/gnparser/cmd/root.go b/gnparser/cmd/root.go index 6a2b1bb..4e71bfd 100644 --- a/gnparser/cmd/root.go +++ b/gnparser/cmd/root.go @@ -79,7 +79,12 @@ gnparser -j 5 -p 8080 batchSize = cfg.BatchSize if port != 0 { - cfg = gnparser.NewConfig(gnparser.OptFormat("compact")) + webopts := []gnparser.Option{ + gnparser.OptFormat("compact"), + gnparser.OptWithWebLogs(withWebLogsFlag(cmd)), + gnparser.OptWebLogsNsqdTCP(webLogsNsqdTCPFlag(cmd)), + } + cfg = gnparser.NewConfig(webopts...) gnp := gnparser.New(cfg) gnps := web.NewGNparserService(gnp, port) web.Run(gnps) @@ -114,12 +119,18 @@ func Execute() { } func init() { - rootCmd.PersistentFlags().BoolP("version", "V", false, - "shows build version and date, ignores other flags.") - rootCmd.Flags().IntP("batch_size", "b", 0, "maximum number of names in a batch send for processing.") + rootCmd.Flags().BoolP("cultivar", "C", false, + "include cultivar epithets and graft-chimeras in normalized and canonical outputs") + + rootCmd.Flags().BoolP("capitalize", "c", false, + "capitalize the first letter of input name-strings") + + rootCmd.Flags().BoolP("diaereses", "D", false, + "preserve diaereses in names") + rootCmd.Flags().BoolP("details", "d", false, "provides more details") formatHelp := "sets output format. Can be one of:\n" + @@ -144,15 +155,12 @@ func init() { rootCmd.Flags().BoolP("unordered", "u", false, "output and input are in different order") - rootCmd.Flags().BoolP("capitalize", "c", false, - "capitalize the first letter of input name-strings") - - rootCmd.Flags().BoolP("cultivar", "C", false, - "include cultivar epithets and graft-chimeras in normalized and canonical outputs") + rootCmd.PersistentFlags().BoolP("version", "V", false, + "shows build version and date, ignores other flags.") - rootCmd.Flags().BoolP("diaereses", "D", false, - "preserve diaereses in names") + rootCmd.Flags().BoolP("web-logs", "", false, "enable logs for the web service") + rootCmd.Flags().StringP("nsqd-tcp", "", "", "an addresss pointing to NSQ TCP service for logs redirection (e.g. 127.0.0.1:4150)") } func processStdin(cmd *cobra.Command, cfg gnparser.Config, quiet bool) { diff --git a/go.mod b/go.mod index d9213cc..5d0563e 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,13 @@ go 1.17 require ( github.com/dustin/go-humanize v1.0.0 github.com/gnames/gnfmt v0.2.0 - github.com/gnames/gnlib v0.3.2 + github.com/gnames/gnlib v0.7.0 github.com/gnames/gnsys v0.2.2 github.com/gnames/gnuuid v0.1.1 github.com/gnames/organizer v0.1.1 github.com/gnames/tribool v0.1.1 github.com/labstack/echo/v4 v4.6.1 + github.com/labstack/gommon v0.3.0 github.com/pointlander/peg v1.0.1 github.com/rendon/testcli v1.0.0 github.com/spf13/cobra v1.2.1 @@ -24,17 +25,18 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang/snappy v0.0.3 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/labstack/gommon v0.3.0 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/mattn/go-colorable v0.1.11 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/mapstructure v1.4.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/nsqio/go-nsq v1.1.0 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pointlander/compress v1.1.1-0.20190518213731-ff44bd196cc3 // indirect diff --git a/go.sum b/go.sum index 5186fce..e7bf142 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gnames/gnfmt v0.1.0/go.mod h1:WG9c3CoiVrGc1SDsxLk7zjmv2B4UIzI00m4K5Khc/d0= github.com/gnames/gnfmt v0.2.0 h1:CjE1HxdqyTwufua5wMCdILWnCsCfRiHe5G4TgxR8aAI= github.com/gnames/gnfmt v0.2.0/go.mod h1:0Aog37s1ZNpmUwVQOf+lnx0SQq8r2EvfE/pLYGiJlJQ= -github.com/gnames/gnlib v0.3.2 h1:lX8fp620ZKsDecuoKRjlk/l8PpPqPcCZ9dHcSGTDQdU= -github.com/gnames/gnlib v0.3.2/go.mod h1:IIo4lQ8hmW/pCmudLgwJ4KdoKWF6B4jFCdVcowG/evw= +github.com/gnames/gnlib v0.7.0 h1:VcwPODD2RItWsI38nQBDMpqjaG8Gnv2l7Q/tTuu2XyY= +github.com/gnames/gnlib v0.7.0/go.mod h1:19DRcg4P/oP623qYZjG1yIBUpdqIh0UXeq2GXdz6/TU= github.com/gnames/gnsys v0.2.2 h1:7IG4aKdCQzzP1tBFp6I9s75QSwM/qhMjpyQVQsedQUY= github.com/gnames/gnsys v0.2.2/go.mod h1:xCjepsCm9yJWTpIfDGt0sUJBLoFRGzH0I3F6ast+Bow= github.com/gnames/gnuuid v0.1.1 h1:UMRHYUSlD19qo8oVz1JyU57kg4yu+SJ/b+yvWYeqRiA= @@ -141,6 +141,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc= github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82/go.mod h1:PxC8OnwL11+aosOB5+iEPoV3picfs8tUpkVd0pDo+Kg= @@ -287,6 +289,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE= +github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= diff --git a/interface.go b/interface.go index 62b60ae..02d2665 100644 --- a/interface.go +++ b/interface.go @@ -12,29 +12,36 @@ import ( // GNparser is the main use-case interface. It provides methods required // for parsing scientific names. type GNparser interface { + // ChangeConfig allows to modify settings of GNparser. Changing settings + // might modify parsing process, and the final output of results. + ChangeConfig(opts ...Option) GNparser + + // Debug parses a string and outputs raw AST tree from PEG engine. + Debug(s string) []byte + + // Format returns currently chosen desired output format of a JSON or + // CSV output. + Format() gnfmt.Format + // GetVersion provides a version and a build timestamp of gnparser. GetVersion() gnvers.Version // ParseName takes a name-string, and returns parsed results for the name. ParseName(string) parsed.Parsed - // ParseNames takes a slice of name-strings, and returns a slice of - // parsed results in the same order as the input. - ParseNames([]string) []parsed.Parsed - // ParseNameStream takes a context, an input channel that takes a // a name-string and its position in the input. It returns parsed results // that come in the same order as the input. ParseNameStream(context.Context, <-chan nameidx.NameIdx, chan<- parsed.Parsed) - // Format returns currently chosen desired output format of a JSON or - // CSV output. - Format() gnfmt.Format + // ParseNames takes a slice of name-strings, and returns a slice of + // parsed results in the same order as the input. + ParseNames([]string) []parsed.Parsed - // ChangeConfig allows to modify settings of GNparser. Changing settings - // might modify parsing process, and the final output of results. - ChangeConfig(opts ...Option) GNparser + // WebLogs returns a boolean to show or not the web-service logs. + WebLogs() bool - // Debug parses a string and outputs raw AST tree from PEG engine. - Debug(s string) []byte + // WebLogsNsqdTCP returns a URL to NSQ messanging service if it is given. + // WebLogsNsqdTCP is used for publishing web logs as messages to NSQ server. + WebLogsNsqdTCP() string } diff --git a/io/web/server.go b/io/web/server.go index 0da2bf7..4933ea0 100644 --- a/io/web/server.go +++ b/io/web/server.go @@ -9,14 +9,14 @@ import ( "time" "github.com/gnames/gnfmt" + "github.com/gnames/gnlib/io/lognsq" "github.com/gnames/gnparser" "github.com/gnames/gnparser/ent/parsed" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" + "github.com/labstack/gommon/log" ) -const withLogs = false - //go:embed static var static embed.FS @@ -31,9 +31,11 @@ type inputREST struct { // Run starts the GNparser web service and servies both RESTful API and // a website. func Run(gnps GNparserService) { + var remote *lognsq.LogNSQ + var err error + e := echo.New() - var err error e.Renderer, err = NewTemplate() if err != nil { e.Logger.Fatal(err) @@ -41,9 +43,30 @@ func Run(gnps GNparserService) { e.Use(middleware.Gzip()) e.Use(middleware.CORS()) - if withLogs { + + nsqAddr := gnps.WebLogsNsqdTCP() + withLogs := gnps.WebLogs() + + if nsqAddr != "" { + cfg := lognsq.Config{ + PrintLogs: withLogs, + Topic: "gnparser", + NsqdURL: nsqAddr, + } + remote, err = lognsq.New(cfg) + logCfg := middleware.DefaultLoggerConfig + if err == nil { + logCfg.Output = remote + } + e.Use(middleware.LoggerWithConfig(logCfg)) + if err != nil { + log.Warn(err) + } + defer remote.Stop() + } else if withLogs { e.Use(middleware.Logger()) } + e.GET("/", homeGET(gnps)) e.POST("/", homePOST(gnps)) e.GET("/doc/api", docAPI())