From b86a827abdd66ff09f0c413c916e4ee65fcde516 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 25 May 2023 21:15:52 +0200 Subject: [PATCH 1/2] feat: per HTTP handler metrics This adds generic HTTP metrics per handler/method/code: - ipfs_http_requests_inflight (gauge) - partially addresses #112 (outgoing tracked in caboose) - ipfs_http_requests_total (counter) - ipfs_http_request_duration_seconds (summary) - partially addresses #120 - ipfs_http_request_size_bytes (summary) - ipfs_http_response_size_bytes (summary) --- go.mod | 10 +++---- go.sum | 19 ++++++------ handlers.go | 29 +++++++----------- metrics.go | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++ version.go | 14 --------- 5 files changed, 110 insertions(+), 47 deletions(-) create mode 100644 metrics.go diff --git a/go.mod b/go.mod index f56d0bf..7b04404 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/mitchellh/go-server-timing v1.0.1 github.com/multiformats/go-multicodec v0.8.1 github.com/multiformats/go-multihash v0.2.1 - github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_golang v1.15.1 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.2 @@ -36,7 +36,7 @@ require ( ) require ( - github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -61,7 +61,7 @@ require ( github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 // indirect github.com/golang/mock v1.6.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect github.com/google/uuid v1.3.0 // indirect @@ -127,7 +127,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.39.0 // indirect + github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-19 v0.2.1 // indirect @@ -171,7 +171,7 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect nhooyr.io/websocket v1.8.7 // indirect diff --git a/go.sum b/go.sum index 4d29393..a72c9ef 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGy github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= -github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d h1:OD9AM8JZUHPrkEa9/SgQW2cCX6Om8d6n0akPS5leWJA= @@ -199,8 +199,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -513,15 +514,15 @@ github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXx github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= -github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= @@ -1038,8 +1039,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/handlers.go b/handlers.go index 15c7d19..9537b16 100644 --- a/handlers.go +++ b/handlers.go @@ -117,10 +117,17 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC } gwHandler := gateway.NewHandler(gwConf, gwAPI) + ipfsHandler := withHTTPMetrics(gwHandler, "ipfs") + ipnsHandler := withHTTPMetrics(gwHandler, "ipns") + mux := http.NewServeMux() - mux.Handle("/ipfs/", gwHandler) - mux.Handle("/ipns/", gwHandler) - mux.Handle("/api/v0/", newKuboRPCHandler(kuboRPC)) + mux.Handle("/ipfs/", ipfsHandler) + mux.Handle("/ipns/", ipnsHandler) + + // TODO: below is legacy which we want to remove, measuring this separately + // allows us to decide when is the time to do it. + legacyKuboRpcHandler := withHTTPMetrics(newKuboRPCHandler(kuboRPC), "legacyKuboRpc") + mux.Handle("/api/v0/", legacyKuboRpcHandler) // Note: in the future we may want to make this more configurable. noDNSLink := false @@ -146,26 +153,10 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC } } - // Creates metrics handler for total response size. Matches the same metrics - // from Kubo: - // https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152 - sum := prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Namespace: "ipfs", - Subsystem: "http", - Name: "response_size_bytes", - Help: "The HTTP response sizes in bytes.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, []string{"code"}) - err = prometheus.Register(sum) - if err != nil { - return nil, err - } - // Construct the HTTP handler for the gateway. handler := withConnect(mux) handler = http.Handler(gateway.WithHostname(handler, gwAPI, publicGateways, noDNSLink)) handler = servertiming.Middleware(handler, nil) - handler = promhttp.InstrumentHandlerResponseSize(sum, handler) // Add logging. handler = withRequestLogger(handler) diff --git a/metrics.go b/metrics.go new file mode 100644 index 0000000..0a9f60f --- /dev/null +++ b/metrics.go @@ -0,0 +1,85 @@ +package main + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func registerVersionMetric(version string) { + m := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "ipfs", + Subsystem: "bifrost_gateway", + Name: "info", + Help: "Information about bifrost-gateway instance.", + ConstLabels: prometheus.Labels{"version": version}, + }) + prometheus.MustRegister(m) + m.Set(1) +} + +// withHTTPMetrics collects metrics around HTTP request/response count, duration, and size +// per specific handler. Allows us to track them separately for /ipns and /ipfs. +func withHTTPMetrics(handler http.Handler, handlerName string) http.Handler { + + // TODO: move HTTP metrics below to separate file + // HTTP metric template names match Kubo: + // https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152 + // This allows Kubo users to migrate to bifrost-gateway and compare global totals. + opts := prometheus.SummaryOpts{ + Namespace: "ipfs", + Subsystem: "http", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + ConstLabels: prometheus.Labels{"handler": handlerName}, + } + // Dynamic labels 'method or 'code' are auto-filled + // by https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/promhttp#InstrumentHandlerResponseSize + labels := []string{"method", "code"} + + reqWip := prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: opts.Namespace, + Subsystem: opts.Subsystem, + Name: "requests_inflight", + Help: "Tracks the number of HTTP client requests currently in progress.", + ConstLabels: opts.ConstLabels, + }, + ) + prometheus.MustRegister(reqWip) + + reqCnt := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: opts.Namespace, + Subsystem: opts.Subsystem, + Name: "requests_total", + Help: "Total number of HTTP requests made.", + ConstLabels: opts.ConstLabels, + }, + labels, + ) + prometheus.MustRegister(reqCnt) + + opts.Name = "request_duration_seconds" + opts.Help = "The HTTP request latencies in seconds." + reqDur := prometheus.NewSummaryVec(opts, labels) + prometheus.MustRegister(reqDur) + + opts.Name = "request_size_bytes" + opts.Help = "The HTTP request sizes in bytes." + reqSz := prometheus.NewSummaryVec(opts, labels) + prometheus.MustRegister(reqSz) + + opts.Name = "response_size_bytes" + opts.Help = "The HTTP response sizes in bytes." + resSz := prometheus.NewSummaryVec(opts, labels) + prometheus.MustRegister(resSz) + + handler = promhttp.InstrumentHandlerInFlight(reqWip, handler) + handler = promhttp.InstrumentHandlerCounter(reqCnt, handler) + handler = promhttp.InstrumentHandlerDuration(reqDur, handler) + handler = promhttp.InstrumentHandlerRequestSize(reqSz, handler) + handler = promhttp.InstrumentHandlerResponseSize(resSz, handler) + + return handler +} diff --git a/version.go b/version.go index 0bc8946..b9982fb 100644 --- a/version.go +++ b/version.go @@ -3,8 +3,6 @@ package main import ( "runtime/debug" "time" - - "github.com/prometheus/client_golang/prometheus" ) var name = "bifrost-gateway" @@ -39,15 +37,3 @@ func buildVersion() string { } return "dev-build" } - -func registerVersionMetric(version string) { - m := prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "ipfs", - Subsystem: "bifrost_gateway", - Name: "info", - Help: "Information about bifrost-gateway instance.", - ConstLabels: prometheus.Labels{"version": version}, - }) - prometheus.MustRegister(m) - m.Set(1) -} From e55c2d06208cd7201509a2660e82dc46809bbf35 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 29 May 2023 13:13:46 +0200 Subject: [PATCH 2/2] feat(metrics): add P75 and P95 to http objectives --- metrics.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/metrics.go b/metrics.go index 0a9f60f..615ee6d 100644 --- a/metrics.go +++ b/metrics.go @@ -19,18 +19,34 @@ func registerVersionMetric(version string) { m.Set(1) } +// httpMetricsObjectives Objectives map defines the quantile objectives for a +// summary metric in Prometheus. Each key-value pair in the map represents a +// quantile level and the desired maximum error allowed for that quantile. +// +// Adjusting the objectives control the trade-off between +// accuracy and resource usage for the summary metric. +// +// Example: 0.95: 0.005 means that the 95th percentile (P95) should have a +// maximum error of 0.005, which represents a 0.5% error margin. +var httpMetricsObjectives = map[float64]float64{ + 0.5: 0.05, + 0.75: 0.025, + 0.9: 0.01, + 0.95: 0.005, + 0.99: 0.001, +} + // withHTTPMetrics collects metrics around HTTP request/response count, duration, and size // per specific handler. Allows us to track them separately for /ipns and /ipfs. func withHTTPMetrics(handler http.Handler, handlerName string) http.Handler { - // TODO: move HTTP metrics below to separate file // HTTP metric template names match Kubo: // https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152 // This allows Kubo users to migrate to bifrost-gateway and compare global totals. opts := prometheus.SummaryOpts{ Namespace: "ipfs", Subsystem: "http", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + Objectives: httpMetricsObjectives, ConstLabels: prometheus.Labels{"handler": handlerName}, } // Dynamic labels 'method or 'code' are auto-filled