Export metrics to a working endpoint when is OTEL_REALM set (#2637)

CHANGELOG.md

@@ -8,10 +8,14 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 ## [Unreleased]
 
+The release adds support for sending metrics directly to Splunk Observability Cloud.
+
 ### Add
 
 - Add the `WithIDGenerator` option to
   `github.com/signalfx/splunk-otel-go/distro`. (#2634)
+- Metrics are sent directly to Splunk Observability Cloud when `SPLUNK_REALM` is
+  set. (#2637)
 
 ## [1.10.0] - 2023-11-10
 

distro/config.go

@@ -60,10 +60,13 @@ const (
 	defaultMetricsExporter = "otlp"
 	defaultLogLevel        = "info"
 
-	defaultJaegerEndpoint = "http://127.0.0.1:9080/v1/trace"
+	jaegerDefaultEndpoint     = "http://127.0.0.1:9080/v1/trace"
+	jaegerRealmEndpointFormat = "https://ingest.%s.signalfx.com/v2/trace"
 
-	realmEndpointFormat     = "https://ingest.%s.signalfx.com/v2/trace"
-	otlpRealmEndpointFormat = "ingest.%s.signalfx.com:443"
+	otlpRealmTracesEndpointFormat  = "ingest.%s.signalfx.com"
+	otlpRealmTracesEndpointPath    = "/v2/trace/otlp"
+	otlpRealmMetricsEndpointFormat = "ingest.%s.signalfx.com"
+	otlpRealmMetricsEndpointPath   = "/v2/datapoint/otlp"
 )
 
 type exporterConfig struct {

distro/exporter.go

@@ -23,7 +23,9 @@ import (
 	"github.com/go-logr/logr"
 	"go.opentelemetry.io/otel/exporters/jaeger" //nolint:staticcheck // Jaeger is deprecated, but we still support it to not break existing users.
 	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
 	"go.opentelemetry.io/otel/sdk/metric"
 	"go.opentelemetry.io/otel/sdk/trace"
 	"google.golang.org/grpc/credentials"
@@ -56,13 +58,22 @@ func tracesExporter(log logr.Logger) traceExporterFunc {
 }
 
 func newOTLPTracesExporter(c *exporterConfig) (trace.SpanExporter, error) {
-	var opts []otlptracegrpc.Option
-
-	endpoint := otlpTracesEndpoint()
-	if endpoint != "" {
-		opts = append(opts, otlptracegrpc.WithEndpoint(endpoint))
+	ctx := context.Background()
+
+	splunkEndpoint := otlpRealmTracesEndpoint()
+	if splunkEndpoint != "" {
+		// Direct ingest to Splunk Observabilty Cloud using HTTP/protobuf.
+		return otlptracehttp.New(ctx,
+			otlptracehttp.WithEndpoint(splunkEndpoint),
+			otlptracehttp.WithURLPath(otlpRealmTracesEndpointPath),
+			otlptracehttp.WithHeaders(map[string]string{
+				"X-Sf-Token": c.AccessToken,
+			}),
+		)
 	}
 
+	var opts []otlptracegrpc.Option
+
 	if c.AccessToken != "" {
 		opts = append(opts, otlptracegrpc.WithHeaders(map[string]string{
 			"X-Sf-Token": c.AccessToken,
@@ -77,11 +88,11 @@ func newOTLPTracesExporter(c *exporterConfig) (trace.SpanExporter, error) {
 		opts = append(opts, otlptracegrpc.WithTLSCredentials(insecure.NewCredentials()))
 	}
 
-	return otlptracegrpc.New(context.Background(), opts...)
+	return otlptracegrpc.New(ctx, opts...)
 }
 
-// otlpTracesEndpoint returns the endpoint to use for the OTLP gRPC traces exporter.
-func otlpTracesEndpoint() string {
+// otlpRealmTracesEndpoint returns the endpoint to use for the OTLP HTTP/protobuf traces exporter.
+func otlpRealmTracesEndpoint() string {
 	// Allow the exporter to interpret these environment variables directly.
 	envs := []string{otelExporterOTLPEndpointKey, otelExporterOTLPTracesEndpointKey}
 	for _, env := range envs {
@@ -92,18 +103,18 @@ func otlpTracesEndpoint() string {
 
 	// Use the realm only if OTEL_EXPORTER_OTLP*_ENDPOINT are not defined.
 	// Also, be sure to communicate local is false so the default behavior of
-	// the OTLP gRPC exporter (using the system CA for authentication and
+	// the OTLP HTTP/protobuf exporter (using the system CA for authentication and
 	// encryption) is used.
 	if realm, ok := os.LookupEnv(splunkRealmKey); ok && notNone(realm) {
-		return fmt.Sprintf(otlpRealmEndpointFormat, realm)
+		return fmt.Sprintf(otlpRealmTracesEndpointFormat, realm)
 	}
 
 	// The OTel default is the same as Splunk's (localhost:4317)
 	return ""
 }
 
-// otlpMetricsEndpoint returns the endpoint to use for the OTLP gRPC metrics exporter.
-func otlpMetricsEndpoint() string {
+// otlpRealmMetricsEndpoint returns the endpoint to use for the OTLP HTTP/protobuf metrics exporter.
+func otlpRealmMetricsEndpoint() string {
 	// Allow the exporter to interpret these environment variables directly.
 	envs := []string{otelExporterOTLPEndpointKey, otelExporterOTLPMetricsEndpointKey}
 	for _, env := range envs {
@@ -114,10 +125,10 @@ func otlpMetricsEndpoint() string {
 
 	// Use the realm only if OTEL_EXPORTER_OTLP*_ENDPOINT are not defined.
 	// Also, be sure to communicate local is false so the default behavior of
-	// the OTLP gRPC exporter (using the system CA for authentication and
+	// the OTLP HTTP/protobuf exporter (using the system CA for authentication and
 	// encryption) is used.
 	if realm, ok := os.LookupEnv(splunkRealmKey); ok && notNone(realm) {
-		return fmt.Sprintf(otlpRealmEndpointFormat, realm)
+		return fmt.Sprintf(otlpRealmMetricsEndpointFormat, realm)
 	}
 
 	// The OTel default is the same as Splunk's (localhost:4317)
@@ -159,11 +170,11 @@ func jaegerEndpoint() string {
 
 	// Use the realm only if OTEL_EXPORTER_JAGER_ENDPOINT is not defined.
 	if realm, ok := os.LookupEnv(splunkRealmKey); ok && notNone(realm) {
-		return fmt.Sprintf(realmEndpointFormat, realm)
+		return fmt.Sprintf(jaegerRealmEndpointFormat, realm)
 	}
 
 	// Use Splunk specific default (locally running collector).
-	return defaultJaegerEndpoint
+	return jaegerDefaultEndpoint
 }
 
 type metricsExporterFunc func(*exporterConfig) (metric.Exporter, error)
@@ -190,13 +201,22 @@ func metricsExporter(log logr.Logger) metricsExporterFunc {
 }
 
 func newOTLPMetricsExporter(c *exporterConfig) (metric.Exporter, error) {
-	var opts []otlpmetricgrpc.Option
-
-	endpoint := otlpMetricsEndpoint()
-	if endpoint != "" {
-		opts = append(opts, otlpmetricgrpc.WithEndpoint(endpoint))
+	ctx := context.Background()
+
+	splunkEndpoint := otlpRealmMetricsEndpoint()
+	if splunkEndpoint != "" {
+		// Direct ingest to Splunk Observabilty Cloud using HTTP/protobuf.
+		return otlpmetrichttp.New(ctx,
+			otlpmetrichttp.WithEndpoint(splunkEndpoint),
+			otlpmetrichttp.WithURLPath(otlpRealmMetricsEndpointPath),
+			otlpmetrichttp.WithHeaders(map[string]string{
+				"X-Sf-Token": c.AccessToken,
+			}),
+		)
 	}
 
+	var opts []otlpmetricgrpc.Option
+
 	if c.AccessToken != "" {
 		opts = append(opts, otlpmetricgrpc.WithHeaders(map[string]string{
 			"X-Sf-Token": c.AccessToken,
@@ -211,7 +231,7 @@ func newOTLPMetricsExporter(c *exporterConfig) (metric.Exporter, error) {
 		opts = append(opts, otlpmetricgrpc.WithTLSCredentials(insecure.NewCredentials()))
 	}
 
-	return otlpmetricgrpc.New(context.Background(), opts...)
+	return otlpmetricgrpc.New(ctx, opts...)
 }
 
 // noneEnvVarSet returns true if none of provided env vars is set.

distro/exporter_test.go

@@ -29,107 +29,107 @@ const (
 
 func TestOTLPTracesEndpoint(t *testing.T) {
 	t.Run("default", func(t *testing.T) {
-		assert.Equal(t, "", otlpTracesEndpoint())
+		assert.Equal(t, "", otlpRealmTracesEndpoint())
 	})
 
 	t.Run("none realm", func(t *testing.T) {
 		t.Setenv(splunkRealmKey, noneRealm)
 
-		assert.Equal(t, "", otlpTracesEndpoint())
+		assert.Equal(t, "", otlpRealmTracesEndpoint())
 	})
 
 	t.Run("realm", func(t *testing.T) {
 		t.Setenv(splunkRealmKey, invalidRealm)
 
-		want := fmt.Sprintf(otlpRealmEndpointFormat, invalidRealm)
-		assert.Equal(t, want, otlpTracesEndpoint())
+		want := fmt.Sprintf(otlpRealmTracesEndpointFormat, invalidRealm)
+		assert.Equal(t, want, otlpRealmTracesEndpoint())
 	})
 
 	t.Run(otelExporterOTLPEndpointKey, func(t *testing.T) {
 		t.Setenv(splunkRealmKey, invalidRealm)
 		t.Setenv(otelExporterOTLPEndpointKey, fakeEndpoint)
 
 		// SPLUNK_REALM is set, make sure it does not take precedence.
-		assert.Equal(t, "", otlpTracesEndpoint())
+		assert.Equal(t, "", otlpRealmTracesEndpoint())
 	})
 
 	t.Run(otelExporterOTLPTracesEndpointKey, func(t *testing.T) {
 		t.Setenv(splunkRealmKey, invalidRealm)
 		t.Setenv(otelExporterOTLPTracesEndpointKey, "some non-zero value")
 
 		// SPLUNK_REALM is set, make sure it does not take precedence.
-		assert.Equal(t, "", otlpTracesEndpoint())
+		assert.Equal(t, "", otlpRealmTracesEndpoint())
 	})
 
 	t.Run(otelExporterOTLPMetricsEndpointKey, func(t *testing.T) {
 		t.Setenv(splunkRealmKey, invalidRealm)
 		t.Setenv(otelExporterOTLPMetricsEndpointKey, "some non-zero value")
 
 		// OTEL_EXPORTER_OTLP_METRICS_ENDPOINT is ignored for traces exporter.
-		want := fmt.Sprintf(otlpRealmEndpointFormat, invalidRealm)
-		assert.Equal(t, want, otlpTracesEndpoint())
+		want := fmt.Sprintf(otlpRealmTracesEndpointFormat, invalidRealm)
+		assert.Equal(t, want, otlpRealmTracesEndpoint())
 	})
 }
 
 func TestOTLPMetricsEndpoint(t *testing.T) {
 	t.Run("default", func(t *testing.T) {
-		assert.Equal(t, "", otlpMetricsEndpoint())
+		assert.Equal(t, "", otlpRealmMetricsEndpoint())
 	})
 
 	t.Run("none realm", func(t *testing.T) {
 		t.Setenv(splunkRealmKey, noneRealm)
 
-		assert.Equal(t, "", otlpMetricsEndpoint())
+		assert.Equal(t, "", otlpRealmMetricsEndpoint())
 	})
 
 	t.Run("realm", func(t *testing.T) {
 		t.Setenv(splunkRealmKey, invalidRealm)
 
-		want := fmt.Sprintf(otlpRealmEndpointFormat, invalidRealm)
-		assert.Equal(t, want, otlpMetricsEndpoint())
+		want := fmt.Sprintf(otlpRealmMetricsEndpointFormat, invalidRealm)
+		assert.Equal(t, want, otlpRealmMetricsEndpoint())
 	})
 
 	t.Run(otelExporterOTLPEndpointKey, func(t *testing.T) {
 		t.Setenv(splunkRealmKey, invalidRealm)
 		t.Setenv(otelExporterOTLPEndpointKey, fakeEndpoint)
 
 		// SPLUNK_REALM is set, make sure it does not take precedence.
-		assert.Equal(t, "", otlpMetricsEndpoint())
+		assert.Equal(t, "", otlpRealmMetricsEndpoint())
 	})
 
 	t.Run(otelExporterOTLPMetricsEndpointKey, func(t *testing.T) {
 		t.Setenv(splunkRealmKey, invalidRealm)
 		t.Setenv(otelExporterOTLPMetricsEndpointKey, "some non-zero value")
 
 		// SPLUNK_REALM is set, make sure it does not take precedence.
-		assert.Equal(t, "", otlpMetricsEndpoint())
+		assert.Equal(t, "", otlpRealmMetricsEndpoint())
 	})
 
 	t.Run(otelExporterOTLPTracesEndpointKey, func(t *testing.T) {
 		t.Setenv(splunkRealmKey, invalidRealm)
 		t.Setenv(otelExporterOTLPTracesEndpointKey, "some non-zero value")
 
 		// OTEL_EXPORTER_OTLP_TRACES_ENDPOINT is ignored for metrics exporter.
-		want := fmt.Sprintf(otlpRealmEndpointFormat, invalidRealm)
-		assert.Equal(t, want, otlpMetricsEndpoint())
+		want := fmt.Sprintf(otlpRealmMetricsEndpointFormat, invalidRealm)
+		assert.Equal(t, want, otlpRealmMetricsEndpoint())
 	})
 }
 
 func TestJaegerEndpoint(t *testing.T) {
 	t.Run("default", func(t *testing.T) {
-		assert.Equal(t, defaultJaegerEndpoint, jaegerEndpoint())
+		assert.Equal(t, jaegerDefaultEndpoint, jaegerEndpoint())
 	})
 
 	t.Run("none realm", func(t *testing.T) {
 		t.Setenv(splunkRealmKey, noneRealm)
 
-		assert.Equal(t, defaultJaegerEndpoint, jaegerEndpoint())
+		assert.Equal(t, jaegerDefaultEndpoint, jaegerEndpoint())
 	})
 
 	t.Run("realm", func(t *testing.T) {
 		t.Setenv(splunkRealmKey, invalidRealm)
 
-		want := fmt.Sprintf(realmEndpointFormat, invalidRealm)
+		want := fmt.Sprintf(jaegerRealmEndpointFormat, invalidRealm)
 		assert.Equal(t, want, jaegerEndpoint())
 	})
 

distro/go.mod

@@ -12,7 +12,9 @@ require (
 	go.opentelemetry.io/otel v1.21.0
 	go.opentelemetry.io/otel/exporters/jaeger v1.17.0
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0
 	go.opentelemetry.io/otel/sdk v1.21.0
 	go.opentelemetry.io/otel/sdk/metric v1.21.0
 	go.opentelemetry.io/otel/trace v1.21.0

distro/go.sum

@@ -44,10 +44,14 @@ go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM
 go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 h1:jd0+5t/YynESZqsSyPz+7PAFdEop0dlN0+PkyHYo8oI=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0/go.mod h1:U707O40ee1FpQGyhvqnzmCJm1Wh6OX6GGBVn0E6Uyyk=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0 h1:bflGWrfYyuulcdxf14V6n9+CoQcu5SAAdHmDPAJnlps=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0/go.mod h1:qcTO4xHAxZLaLxPd60TdE88rxtItPHgHWqOhOGRr0as=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I=
 go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
 go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
 go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=

example/go.mod

@@ -26,8 +26,10 @@ require (
 	go.opentelemetry.io/otel v1.21.0 // indirect
 	go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 // indirect
 	go.opentelemetry.io/otel/metric v1.21.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.21.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.21.0 // indirect

example/go.sum

@@ -41,10 +41,14 @@ go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM
 go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 h1:jd0+5t/YynESZqsSyPz+7PAFdEop0dlN0+PkyHYo8oI=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0/go.mod h1:U707O40ee1FpQGyhvqnzmCJm1Wh6OX6GGBVn0E6Uyyk=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0 h1:bflGWrfYyuulcdxf14V6n9+CoQcu5SAAdHmDPAJnlps=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0/go.mod h1:qcTO4xHAxZLaLxPd60TdE88rxtItPHgHWqOhOGRr0as=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I=
 go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
 go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
 go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=