feat: per HTTP handler metrics

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/mitchellh/go-server-timing v1.0.1
 	github.com/multiformats/go-multicodec v0.8.1
 	github.com/multiformats/go-multihash v0.2.1
-	github.com/prometheus/client_golang v1.14.0
+	github.com/prometheus/client_golang v1.15.1
 	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
 	github.com/spf13/cobra v1.6.1
 	github.com/stretchr/testify v1.8.2
@@ -36,7 +36,7 @@ require (
 )
 
 require (
-	github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect
+	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -61,7 +61,7 @@ require (
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 // indirect
 	github.com/golang/mock v1.6.0 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
 	github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect
 	github.com/google/uuid v1.3.0 // indirect
@@ -127,7 +127,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.39.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
 	github.com/quic-go/qtls-go1-19 v0.2.1 // indirect
@@ -171,7 +171,7 @@ require (
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
 	google.golang.org/grpc v1.53.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.1.7 // indirect
 	nhooyr.io/websocket v1.8.7 // indirect

go.sum

@@ -41,8 +41,8 @@ git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGy
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc=
-github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
+github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d h1:OD9AM8JZUHPrkEa9/SgQW2cCX6Om8d6n0akPS5leWJA=
@@ -199,8 +199,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -513,15 +514,15 @@ github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXx
 github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4=
 github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw=
 github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
-github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
+github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
+github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
 github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI=
-github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y=
+github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
+github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
@@ -1038,8 +1039,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
-google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

handlers.go

@@ -117,10 +117,17 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC
 	}
 
 	gwHandler := gateway.NewHandler(gwConf, gwAPI)
+	ipfsHandler := withHTTPMetrics(gwHandler, "ipfs")
+	ipnsHandler := withHTTPMetrics(gwHandler, "ipns")
+
 	mux := http.NewServeMux()
-	mux.Handle("/ipfs/", gwHandler)
-	mux.Handle("/ipns/", gwHandler)
-	mux.Handle("/api/v0/", newKuboRPCHandler(kuboRPC))
+	mux.Handle("/ipfs/", ipfsHandler)
+	mux.Handle("/ipns/", ipnsHandler)
+
+	// TODO: below is legacy which we want to remove, measuring this separately
+	// allows us to decide when is the time to do it.
+	legacyKuboRpcHandler := withHTTPMetrics(newKuboRPCHandler(kuboRPC), "legacyKuboRpc")
+	mux.Handle("/api/v0/", legacyKuboRpcHandler)
 
 	// Note: in the future we may want to make this more configurable.
 	noDNSLink := false
@@ -146,26 +153,10 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC
 		}
 	}
 
-	// Creates metrics handler for total response size. Matches the same metrics
-	// from Kubo:
-	// https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152
-	sum := prometheus.NewSummaryVec(prometheus.SummaryOpts{
-		Namespace:  "ipfs",
-		Subsystem:  "http",
-		Name:       "response_size_bytes",
-		Help:       "The HTTP response sizes in bytes.",
-		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-	}, []string{"code"})
-	err = prometheus.Register(sum)
-	if err != nil {
-		return nil, err
-	}
-
 	// Construct the HTTP handler for the gateway.
 	handler := withConnect(mux)
 	handler = http.Handler(gateway.WithHostname(handler, gwAPI, publicGateways, noDNSLink))
 	handler = servertiming.Middleware(handler, nil)
-	handler = promhttp.InstrumentHandlerResponseSize(sum, handler)
 
 	// Add logging.
 	handler = withRequestLogger(handler)

metrics.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+func registerVersionMetric(version string) {
+	m := prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   "ipfs",
+		Subsystem:   "bifrost_gateway",
+		Name:        "info",
+		Help:        "Information about bifrost-gateway instance.",
+		ConstLabels: prometheus.Labels{"version": version},
+	})
+	prometheus.MustRegister(m)
+	m.Set(1)
+}
+
+// httpMetricsObjectives Objectives map defines the quantile objectives for a
+// summary metric in Prometheus. Each key-value pair in the map represents a
+// quantile level and the desired maximum error allowed for that quantile.
+//
+// Adjusting the objectives control the trade-off between
+// accuracy and resource usage for the summary metric.
+//
+// Example: 0.95: 0.005 means that the 95th percentile (P95) should have a
+// maximum error of 0.005, which represents a 0.5% error margin.
+var httpMetricsObjectives = map[float64]float64{
+	0.5:  0.05,
+	0.75: 0.025,
+	0.9:  0.01,
+	0.95: 0.005,
+	0.99: 0.001,
+}
+
+// withHTTPMetrics collects metrics around HTTP request/response count, duration, and size
+// per specific handler. Allows us to track them separately for /ipns and /ipfs.
+func withHTTPMetrics(handler http.Handler, handlerName string) http.Handler {
+
+	// HTTP metric template names match Kubo:
+	// https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152
+	// This allows Kubo users to migrate to bifrost-gateway and compare global totals.
+	opts := prometheus.SummaryOpts{
+		Namespace:   "ipfs",
+		Subsystem:   "http",
+		Objectives:  httpMetricsObjectives,
+		ConstLabels: prometheus.Labels{"handler": handlerName},
+	}
+	// Dynamic labels 'method or 'code' are auto-filled
+	// by https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/promhttp#InstrumentHandlerResponseSize
+	labels := []string{"method", "code"}
+
+	reqWip := prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace:   opts.Namespace,
+			Subsystem:   opts.Subsystem,
+			Name:        "requests_inflight",
+			Help:        "Tracks the number of HTTP client requests currently in progress.",
+			ConstLabels: opts.ConstLabels,
+		},
+	)
+	prometheus.MustRegister(reqWip)
+
+	reqCnt := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace:   opts.Namespace,
+			Subsystem:   opts.Subsystem,
+			Name:        "requests_total",
+			Help:        "Total number of HTTP requests made.",
+			ConstLabels: opts.ConstLabels,
+		},
+		labels,
+	)
+	prometheus.MustRegister(reqCnt)
+
+	opts.Name = "request_duration_seconds"
+	opts.Help = "The HTTP request latencies in seconds."
+	reqDur := prometheus.NewSummaryVec(opts, labels)
+	prometheus.MustRegister(reqDur)
+
+	opts.Name = "request_size_bytes"
+	opts.Help = "The HTTP request sizes in bytes."
+	reqSz := prometheus.NewSummaryVec(opts, labels)
+	prometheus.MustRegister(reqSz)
+
+	opts.Name = "response_size_bytes"
+	opts.Help = "The HTTP response sizes in bytes."
+	resSz := prometheus.NewSummaryVec(opts, labels)
+	prometheus.MustRegister(resSz)
+
+	handler = promhttp.InstrumentHandlerInFlight(reqWip, handler)
+	handler = promhttp.InstrumentHandlerCounter(reqCnt, handler)
+	handler = promhttp.InstrumentHandlerDuration(reqDur, handler)
+	handler = promhttp.InstrumentHandlerRequestSize(reqSz, handler)
+	handler = promhttp.InstrumentHandlerResponseSize(resSz, handler)
+
+	return handler
+}

version.go

@@ -3,8 +3,6 @@ package main
 import (
 	"runtime/debug"
 	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 var name = "bifrost-gateway"
@@ -39,15 +37,3 @@ func buildVersion() string {
 	}
 	return "dev-build"
 }
-
-func registerVersionMetric(version string) {
-	m := prometheus.NewGauge(prometheus.GaugeOpts{
-		Namespace:   "ipfs",
-		Subsystem:   "bifrost_gateway",
-		Name:        "info",
-		Help:        "Information about bifrost-gateway instance.",
-		ConstLabels: prometheus.Labels{"version": version},
-	})
-	prometheus.MustRegister(m)
-	m.Set(1)
-}