feat!: use OTLP sources by default

SumoLogic · Sep 25, 2023 · 7acfadf · 7acfadf
1 parent 8b109d2
commit 7acfadf
Show file tree

Hide file tree

Showing 33 changed files with 246 additions and 432 deletions.
diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -272,7 +272,7 @@ sumologic:
         pvcLabels: {}
         # storageClass:
 
-    sourceType: http
+    sourceType: otlp
 
   ### Logs configuration
   ## Set the enabled flag to false for disabling logs ingestion altogether.
@@ -435,7 +435,7 @@ sumologic:
     ## https://help.sumologic.com/docs/manage/fields/#manage-fields
     additionalFields: []
 
-    sourceType: http
+    sourceType: otlp
 
   ### Metrics configuration
   ## Set the enabled flag to false for disabling metrics ingestion altogether.
@@ -666,7 +666,7 @@ sumologic:
 
     ## The type of source we send to in Sumo. The possible values are http and otlp.
     ## Consult the documentation for more information.
-    sourceType: http
+    sourceType: otlp
 
   ### Traces configuration
   ## Set the enabled flag to false to disable traces ingestion.
@@ -675,7 +675,7 @@ sumologic:
     ## How many spans per request should be send to receiver
     spans_per_request: 100
 
-    sourceType: http
+    sourceType: otlp
 
 ## Configure metrics-server
 ## ref: https://github.com/bitnami/charts/blob/master/bitnami/metrics-server/values.yaml
@@ -1583,7 +1583,7 @@ tracesSampler:
       # logging:
       #   loglevel: debug
       otlphttp:
-        traces_endpoint: ${SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE}
+        traces_endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_TRACES_SOURCE}/v1/traces
         compression: gzip
     service:
       extensions: [health_check, memory_ballast, pprof]

diff --git a/tests/helm/logs_test.go b/tests/helm/logs_test.go
@@ -78,7 +78,7 @@ sumologic:
 	err := yaml.Unmarshal([]byte(otelConfigYaml), &otelConfig)
 	require.NoError(t, err)
 
-	require.ElementsMatch(t, []string{"sumologic/containers"}, keys(otelConfig.Exporters))
+	require.ElementsMatch(t, []string{"sumologic"}, keys(otelConfig.Exporters))
 	require.ElementsMatch(t, []string{"logs/otlp/containers"}, keys(otelConfig.Service.Pipelines))
 	for processorName := range otelConfig.Processors {
 		require.NotContains(t, processorName, "systemd")
@@ -126,7 +126,7 @@ sumologic:
 	require.Contains(t, containersPipeline.Processors, "filter/include-host")
 }
 
-func TestMetadataLogFormat(t *testing.T) {
+func TestMetadataLogFormatHTTP(t *testing.T) {
 	t.Parallel()
 	templatePath := "templates/logs/otelcol/configmap.yaml"
 
@@ -174,6 +174,7 @@ sumologic:
   logs:
     container:
       format: %s
+    sourceType: http
 `
 			valuesYaml := fmt.Sprintf(valuesYamlTemplate, testCase.logFormat)
 			otelConfigYaml := GetOtelConfigYaml(t, valuesYaml, templatePath)

diff --git a/tests/helm/metrics_test.go b/tests/helm/metrics_test.go
@@ -137,7 +137,6 @@ func TestMetadataMetricsOtelConfigExtraProcessors(t *testing.T) {
 		"transform/remove_name",
 		"filter/drop_unnecessary_metrics",
 		"batch",
-		"routing",
 	}
 
 	require.Equal(t, expectedPipelineValue, otelConfig.Service.Pipelines.Metrics.Processors)
@@ -180,12 +179,62 @@ sumologic:
 	assert.Equal(t, otelConfig.Exporters.Default.Endpoint, "${SUMO_ENDPOINT_DEFAULT_OTLP_METRICS_SOURCE}")
 	assert.Len(t, otelConfig.Exporters.Rest, 0)
 	assert.NotContains(t, otelConfig.Processors, "routing")
-	assert.NotContains(t, otelConfig.Processors, "transform/prepare_routing")
 	assert.NotContains(t, otelConfig.Service.Pipelines.Metrics.Processors, "routing")
-	assert.NotContains(t, otelConfig.Service.Pipelines.Metrics.Processors, "transform/prepare_routing")
 	assert.Equal(t, otelConfig.Service.Pipelines.Metrics.Exporters, []string{"sumologic/default"})
 }
 
+func TestMetadataSourceTypeHTTP(t *testing.T) {
+	t.Parallel()
+	templatePath := "templates/metrics/otelcol/configmap.yaml"
+
+	type OtelConfig struct {
+		Exporters map[string]struct {
+			MetricFormat string `yaml:"metric_format"`
+			Endpoint     string
+		} `yaml:"exporters"`
+		Processors map[string]interface{}
+		Service    struct {
+			Pipelines struct {
+				Metrics struct {
+					Processors []string `yaml:"processors"`
+					Exporters  []string `yaml:"exporters"`
+				}
+			}
+		}
+	}
+
+	var otelConfig OtelConfig
+	valuesYaml := `
+sumologic:
+  metrics:
+    sourceType: http
+`
+	otelConfigYaml := GetOtelConfigYaml(t, valuesYaml, templatePath)
+	err := yaml.Unmarshal([]byte(otelConfigYaml), &otelConfig)
+	require.NoError(t, err)
+
+	require.Contains(t, otelConfig.Exporters, "sumologic/default")
+	defaultExporter := otelConfig.Exporters["sumologic/default"]
+	assert.Equal(t, "prometheus", defaultExporter.MetricFormat)
+	assert.Equal(t, "${SUMO_ENDPOINT_DEFAULT_METRICS_SOURCE}", defaultExporter.Endpoint)
+	assert.Contains(t, otelConfig.Processors, "routing")
+	assert.Contains(t, otelConfig.Service.Pipelines.Metrics.Processors, "routing")
+	assert.Equal(
+		t,
+		[]string{
+			"sumologic/default",
+			"sumologic/apiserver",
+			"sumologic/control_plane",
+			"sumologic/controller",
+			"sumologic/kubelet",
+			"sumologic/node",
+			"sumologic/scheduler",
+			"sumologic/state",
+		},
+		otelConfig.Service.Pipelines.Metrics.Exporters,
+	)
+}
+
 func TestNoPrometheusServiceMonitors(t *testing.T) {
 	t.Parallel()
 	allTemplatePaths := []string{

diff --git a/tests/helm/testdata/goldenfile/events_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/events_otc/basic.output.yaml
@@ -15,8 +15,8 @@ data:
     exporters:
       sumologic:
         clear_logs_timestamp: false
-        endpoint: ${SUMO_ENDPOINT_DEFAULT_EVENTS_SOURCE}
-        log_format: json
+        endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_EVENTS_SOURCE}
+        log_format: otlp
         sending_queue:
           enabled: true
           storage: file_storage

diff --git a/tests/helm/testdata/goldenfile/events_otc/options.output.yaml b/tests/helm/testdata/goldenfile/events_otc/options.output.yaml
@@ -15,8 +15,8 @@ data:
     exporters:
       sumologic:
         clear_logs_timestamp: false
-        endpoint: ${SUMO_ENDPOINT_DEFAULT_EVENTS_SOURCE}
-        log_format: json
+        endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_EVENTS_SOURCE}
+        log_format: otlp
         sending_queue:
           enabled: true
     extensions:

diff --git a/tests/helm/testdata/goldenfile/events_otc_statefulset/basic.output.yaml b/tests/helm/testdata/goldenfile/events_otc_statefulset/basic.output.yaml
@@ -126,6 +126,11 @@ spec:
                 secretKeyRef:
                   name: sumologic
                   key: endpoint-events
+            - name: SUMO_ENDPOINT_DEFAULT_OTLP_EVENTS_SOURCE
+              valueFrom:
+                secretKeyRef:
+                  name: sumologic
+                  key: endpoint-events-otlp
 
             - name: NO_PROXY
               value: kubernetes.default.svc

diff --git a/tests/helm/testdata/goldenfile/events_otc_statefulset/custom.input.yaml b/tests/helm/testdata/goldenfile/events_otc_statefulset/custom.input.yaml
@@ -1,6 +1,6 @@
 sumologic:
   events:
-    sourceType: otlp
+    sourceType: http
   podAnnotations:
     sumoAnnotation: sumoValue
   podLabels:

diff --git a/tests/helm/testdata/goldenfile/events_otc_statefulset/custom.output.yaml b/tests/helm/testdata/goldenfile/events_otc_statefulset/custom.output.yaml
@@ -114,11 +114,6 @@ spec:
                 secretKeyRef:
                   name: sumologic
                   key: endpoint-events
-            - name: SUMO_ENDPOINT_DEFAULT_OTLP_EVENTS_SOURCE
-              valueFrom:
-                secretKeyRef:
-                  name: sumologic
-                  key: endpoint-events-otlp
 
             - name: NO_PROXY
               value: kubernetes.default.svc

diff --git a/tests/helm/testdata/goldenfile/events_otc_statefulset/proxy.output.yaml b/tests/helm/testdata/goldenfile/events_otc_statefulset/proxy.output.yaml
@@ -110,6 +110,11 @@ spec:
                 secretKeyRef:
                   name: sumologic
                   key: endpoint-events
+            - name: SUMO_ENDPOINT_DEFAULT_OTLP_EVENTS_SOURCE
+              valueFrom:
+                secretKeyRef:
+                  name: sumologic
+                  key: endpoint-events-otlp
 
             - name: HTTP_PROXY
               value: http://proxy.internal

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/otel.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/otel.output.yaml
@@ -13,21 +13,10 @@ metadata:
 data:
   config.yaml: |
     exporters:
-      sumologic/containers:
-        endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
-        json_logs:
-          add_timestamp: false
-        log_format: json
-        sending_queue:
-          enabled: true
-          num_consumers: 10
-          queue_size: 10000
-          storage: file_storage
-      sumologic/systemd:
-        endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
-        json_logs:
-          add_timestamp: false
-        log_format: json
+      sumologic:
+        clear_logs_timestamp: false
+        endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE}
+        log_format: otlp
         sending_queue:
           enabled: true
           num_consumers: 10
@@ -291,7 +280,7 @@ data:
       pipelines:
         logs/otlp/containers:
           exporters:
-          - sumologic/containers
+          - sumologic
           processors:
           - memory_limiter
           - filter/include_containers
@@ -311,7 +300,7 @@ data:
           - otlp
         logs/otlp/kubelet:
           exporters:
-          - sumologic/systemd
+          - sumologic
           processors:
           - memory_limiter
           - filter/include_fluent_tag_host
@@ -333,7 +322,7 @@ data:
           - otlp
         logs/otlp/systemd:
           exporters:
-          - sumologic/systemd
+          - sumologic
           processors:
           - memory_limiter
           - filter/include_fluent_tag_host

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc/templates.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc/templates.output.yaml
@@ -13,21 +13,10 @@ metadata:
 data:
   config.yaml: |
     exporters:
-      sumologic/containers:
-        endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
-        json_logs:
-          add_timestamp: false
-        log_format: json
-        sending_queue:
-          enabled: true
-          num_consumers: 10
-          queue_size: 10000
-          storage: file_storage
-      sumologic/systemd:
-        endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
-        json_logs:
-          add_timestamp: false
-        log_format: json
+      sumologic:
+        clear_logs_timestamp: false
+        endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE}
+        log_format: otlp
         sending_queue:
           enabled: true
           num_consumers: 10
@@ -318,7 +307,7 @@ data:
       pipelines:
         logs/otlp/containers:
           exporters:
-          - sumologic/containers
+          - sumologic
           processors:
           - memory_limiter
           - filter/include_containers
@@ -340,7 +329,7 @@ data:
           - otlp
         logs/otlp/kubelet:
           exporters:
-          - sumologic/systemd
+          - sumologic
           processors:
           - memory_limiter
           - filter/include_fluent_tag_host
@@ -364,7 +353,7 @@ data:
           - otlp
         logs/otlp/systemd:
           exporters:
-          - sumologic/systemd
+          - sumologic
           processors:
           - memory_limiter
           - filter/include_fluent_tag_host

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc_statefulset/basic.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc_statefulset/basic.output.yaml
@@ -98,6 +98,11 @@ spec:
                 secretKeyRef:
                   name: sumologic
                   key: endpoint-logs
+            - name: SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE
+              valueFrom:
+                secretKeyRef:
+                  name: sumologic
+                  key: endpoint-logs-otlp
 
             - name: NO_PROXY
               value: kubernetes.default.svc

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc_statefulset/custom.input.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc_statefulset/custom.input.yaml
@@ -1,6 +1,6 @@
 sumologic:
   logs:
-    sourceType: otlp
+    sourceType: http
 
 metadata:
   image:

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc_statefulset/custom.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc_statefulset/custom.output.yaml
@@ -122,11 +122,6 @@ spec:
                 secretKeyRef:
                   name: sumologic
                   key: endpoint-logs
-            - name: SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE
-              valueFrom:
-                secretKeyRef:
-                  name: sumologic
-                  key: endpoint-logs-otlp
 
             - name: NO_PROXY
               value: kubernetes.default.svc