From 60b80b7ef5c7ed31bdc5332529a642a38e11e586 Mon Sep 17 00:00:00 2001 From: Po-Yao Chen Date: Wed, 13 Nov 2024 23:19:44 +0000 Subject: [PATCH] Add otel instrumentation for http-grpc gateway (#522) --- go.mod | 4 ++- go.sum | 4 +++ otellib/http.go | 76 +++++++++++++++++++++++++++++++++++++++ {otel => otellib}/otel.go | 0 server/server.go | 6 ++-- 5 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 otellib/http.go rename {otel => otellib}/otel.go (100%) diff --git a/go.mod b/go.mod index 754ed06..53185fd 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,10 @@ require ( github.com/miekg/pkcs11 v1.1.1 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go.opentelemetry.io/otel v1.32.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0 + go.opentelemetry.io/otel/metric v1.32.0 go.opentelemetry.io/otel/sdk v1.32.0 go.opentelemetry.io/otel/sdk/metric v1.32.0 golang.org/x/crypto v0.29.0 @@ -25,12 +27,12 @@ require ( require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - go.opentelemetry.io/otel/metric v1.32.0 // indirect go.opentelemetry.io/otel/trace v1.32.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/mod v0.20.0 // indirect diff --git a/go.sum b/go.sum index f077272..1ba7f01 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -73,6 +75,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0 h1:yMkBS9yViCc7U7yeLzJPM2XizlfdVvBRSmsQDWu6qc0= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0/go.mod h1:n8MR6/liuGB5EmTETUBeU5ZgqMOlqKRxUaqPQBOANZ8= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0 h1:t/Qur3vKSkUCcDVaSumWF2PKHt85pc7fRvFuoVT8qFU= diff --git a/otellib/http.go b/otellib/http.go new file mode 100644 index 0000000..fda865c --- /dev/null +++ b/otellib/http.go @@ -0,0 +1,76 @@ +// Copyright 2024 Yahoo Inc. +// Licensed under the terms of the Apache License 2.0. Please see LICENSE file in project root for terms. + +package otellib + +import ( + "fmt" + "log" + "net/http" + "runtime/debug" + + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + semconv "go.opentelemetry.io/otel/semconv/v1.20.0" +) + +const scopeName = "github.com/theparanoids/crypki/otellib" + +const handlerPanic = "http.handler.panic" + +type httpMiddleware struct { + next http.Handler + meter metric.Meter + + panicCounter metric.Int64Counter +} + +// NewHTTPMiddleware wraps an http.Handler with OpenTelemetry instrumentation. +func NewHTTPMiddleware(handler http.Handler, operation string) http.Handler { + h := newHTTPMiddleware(handler) + return otelhttp.NewHandler(h, operation) +} + +func newHTTPMiddleware(nextHandler http.Handler) *httpMiddleware { + middleware := &httpMiddleware{ + next: nextHandler, + } + middleware.meter = otel.GetMeterProvider().Meter(scopeName) + + var err error + if middleware.panicCounter, err = middleware.meter.Int64Counter( + handlerPanic, + metric.WithUnit("1"), + metric.WithDescription("Count the number of HTTP handler panic"), + ); err != nil { + otel.Handle(err) + } + + return middleware +} + +// ServeHTTP implements http.Handler. +func (h *httpMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // A labeler is always available in the request context by otelhttp package. + labeler, _ := otelhttp.LabelerFromContext(r.Context()) + // Add the http.target attribute to the OTel labeler. + if r.URL != nil { + labeler.Add(semconv.HTTPTargetKey.String(r.URL.RequestURI())) + } + defer func() { + if rec := recover(); rec != nil { + log.Printf(`panic captured, %v, %v`, string(debug.Stack()), rec) + + ctx := r.Context() + labels := append(labeler.Get(), []attribute.KeyValue{ + attribute.String("http.method", r.Method), + attribute.String("panic.message", fmt.Sprintf("%v", rec)), + }...) + h.panicCounter.Add(ctx, 1, metric.WithAttributes(labels...)) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + } + }() + h.next.ServeHTTP(w, r) +} diff --git a/otel/otel.go b/otellib/otel.go similarity index 100% rename from otel/otel.go rename to otellib/otel.go diff --git a/server/server.go b/server/server.go index d26174c..f2f5ef1 100644 --- a/server/server.go +++ b/server/server.go @@ -35,7 +35,7 @@ import ( "github.com/theparanoids/crypki/config" "github.com/theparanoids/crypki/healthcheck" "github.com/theparanoids/crypki/oor" - otellib "github.com/theparanoids/crypki/otel" + "github.com/theparanoids/crypki/otellib" "github.com/theparanoids/crypki/pkcs11" "github.com/theparanoids/crypki/proto" "github.com/theparanoids/crypki/server/interceptor" @@ -58,14 +58,14 @@ func grpcHandlerFunc(ctx context.Context, grpcServer *grpc.Server, otherHandler // initHTTPServer initializes HTTP server with TLS credentials and returns http.Server. func initHTTPServer(ctx context.Context, tlsConfig *tls.Config, - grpcServer *grpc.Server, gwmux *runtime.ServeMux, addr string, + grpcServer *grpc.Server, gwmux http.Handler, addr string, idleTimeout, readTimeout, writeTimeout uint) *http.Server { mux := http.NewServeMux() // handler to check if service is up mux.HandleFunc("/ruok", func(w http.ResponseWriter, req *http.Request) { fmt.Fprintln(w, "imok") }) - mux.Handle("/", gwmux) + mux.Handle("/", otellib.NewHTTPMiddleware(gwmux, "crypki-gateway")) srv := &http.Server{ Addr: addr,