From 5ec98e2753b918a5aa468648004c5e19cdf6e6ae Mon Sep 17 00:00:00 2001 From: Milan Kuba <43672087+akumetsu183@users.noreply.github.com> Date: Tue, 21 Jan 2025 14:29:18 +0100 Subject: [PATCH] feature: adding attribute without_entity (#25) * feature: adding attribute without_entity, to opt out of collector entity creation in SWO --- extension/solarwindsextension/README.md | 1 + extension/solarwindsextension/config_test.go | 1 + .../solarwindsextension/internal/config.go | 1 + .../solarwindsextension/internal/heartbeat.go | 7 ++ .../solarwindsextension/testdata/full.yaml | 1 + .../testdata/without_entity.yaml | 4 + internal/e2e/containers.go | 80 +++++++++++++- internal/e2e/signals_processing_test.go | 103 ++---------------- .../emitting_collector_without_entity.yaml | 28 +++++ internal/e2e/without_entity_test.go | 53 +++++++++ 10 files changed, 178 insertions(+), 101 deletions(-) create mode 100644 extension/solarwindsextension/testdata/without_entity.yaml create mode 100644 internal/e2e/testdata/emitting_collector_without_entity.yaml create mode 100644 internal/e2e/without_entity_test.go diff --git a/extension/solarwindsextension/README.md b/extension/solarwindsextension/README.md index f5e541f..85f6422 100644 --- a/extension/solarwindsextension/README.md +++ b/extension/solarwindsextension/README.md @@ -40,6 +40,7 @@ extensions: - `data_center` (mandatory) - Data center is the region you picked during the sign-up process. You can easily see in URLs after logging in to SolarWinds Observability SaaS - it's either `na-01`, `na-02` or `eu-01`. Please refer to the [documentation](https://documentation.solarwinds.com/en/success_center/observability/content/system_requirements/endpoints.htm#Find) for details. - `collector_name` (mandatory) - The collector name passed in the heartbeat metric (as `sw.otelcol.collector.name` resource attribute) to identify the collector. Doesn't have to be unique. - `resource` (optional) - You can specify additional attributes to be added to the `sw.otecol.uptime` metric. +- `without_entity` (optional) - You can disable Collector entity creation in SolarWinds Observability by setting it to `true`. If not configured, entity creation is enabled by default. ## Development - **Tests** can be executed with `make test`. - After changes to `metadata.yaml` generated files need to be re-generated with `make generate`. The [mdatagen](http://go.opentelemetry.io/collector/cmd/mdatagen) tool has to be in the `PATH`. diff --git a/extension/solarwindsextension/config_test.go b/extension/solarwindsextension/config_test.go index be04bd8..046ca26 100644 --- a/extension/solarwindsextension/config_test.go +++ b/extension/solarwindsextension/config_test.go @@ -47,6 +47,7 @@ func TestConfigUnmarshalFull(t *testing.T) { IngestionToken: "TOKEN", CollectorName: "test-collector", Resource: attributeMap, + WithoutEntity: true, }, cfg) } diff --git a/extension/solarwindsextension/internal/config.go b/extension/solarwindsextension/internal/config.go index a3202a8..72ef498 100644 --- a/extension/solarwindsextension/internal/config.go +++ b/extension/solarwindsextension/internal/config.go @@ -40,6 +40,7 @@ type Config struct { // EndpointURLOverride sets OTLP endpoint directly, it overrides the DataCenter configuration. EndpointURLOverride string `mapstructure:"endpoint_url_override"` Resource map[string]string `mapstructure:"resource"` + WithoutEntity bool `mapstructure:"without_entity"` } var ( diff --git a/extension/solarwindsextension/internal/heartbeat.go b/extension/solarwindsextension/internal/heartbeat.go index 6494687..b00d103 100644 --- a/extension/solarwindsextension/internal/heartbeat.go +++ b/extension/solarwindsextension/internal/heartbeat.go @@ -48,6 +48,7 @@ type Heartbeat struct { metric *UptimeMetric exporter MetricsExporter collectorName string + withoutEntity bool beatInterval time.Duration resource map[string]string @@ -78,6 +79,7 @@ func newHeartbeatWithExporter( exporter: exporter, beatInterval: defaultHeartbeatInterval, resource: cfg.Resource, + withoutEntity: cfg.WithoutEntity, } } @@ -165,6 +167,11 @@ func (h *Heartbeat) decorateResourceAttributes(resource pcommon.Resource) error if h.collectorName != "" { resource.Attributes().PutStr(CollectorNameAttribute, h.collectorName) } + if h.withoutEntity { + resource.Attributes().PutStr("sw.otelcol.collector.entity_creation", "off") + } else { + resource.Attributes().PutStr("sw.otelcol.collector.entity_creation", "on") + } resource.Attributes().PutStr("sw.otelcol.collector.version", version.Version) return nil } diff --git a/extension/solarwindsextension/testdata/full.yaml b/extension/solarwindsextension/testdata/full.yaml index 7eec15b..c465652 100644 --- a/extension/solarwindsextension/testdata/full.yaml +++ b/extension/solarwindsextension/testdata/full.yaml @@ -5,3 +5,4 @@ endpoint_url_override: "127.0.0.1:1234" resource: att1: "custom_attribute_value_1" att2: "custom_attribute_value_2" +without_entity: true \ No newline at end of file diff --git a/extension/solarwindsextension/testdata/without_entity.yaml b/extension/solarwindsextension/testdata/without_entity.yaml new file mode 100644 index 0000000..c7dd74f --- /dev/null +++ b/extension/solarwindsextension/testdata/without_entity.yaml @@ -0,0 +1,4 @@ +token: "YOUR-INGESTION-TOKEN" +data_center: "na-01" +collector_name: "test-collector" +without_entity: true \ No newline at end of file diff --git a/internal/e2e/containers.go b/internal/e2e/containers.go index 60315d8..d3741e2 100644 --- a/internal/e2e/containers.go +++ b/internal/e2e/containers.go @@ -19,13 +19,17 @@ package e2e import ( "context" "errors" - "log" - "path/filepath" - "time" - + "fmt" "github.com/mdelapenya/tlscert" + "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/network" "github.com/testcontainers/testcontainers-go/wait" + "log" + "path/filepath" + "strconv" + "testing" + "time" ) const ( @@ -74,8 +78,9 @@ func runTestedSolarWindsOTELCollector( ctx context.Context, certDir string, networkName string, + configName string, ) (testcontainers.Container, error) { - configPath, err := filepath.Abs(filepath.Join(".", "testdata", "emitting_collector.yaml")) + configPath, err := filepath.Abs(filepath.Join(".", "testdata", configName)) if err != nil { return nil, err } @@ -204,6 +209,50 @@ func runGeneratorContainer( return container, err } +// Starts the receiving collector, and the test collector. +// Test collector container is started with the supplied config file to be tested. +// Returns the receiving collector container instance, so tests can check the ingested data is as expected. +func startCollectorContainers( + t *testing.T, + ctx context.Context, + config string, + signalType SignalType, + waitTime time.Duration, +) testcontainers.Container { + + net, err := network.New(ctx) + require.NoError(t, err) + testcontainers.CleanupNetwork(t, net) + + certPath := t.TempDir() + _, err = generateCertificates(receivingContainer, certPath) + require.NoError(t, err) + + rContainer, err := runReceivingSolarWindsOTELCollector(ctx, certPath, net.Name) + require.NoError(t, err) + testcontainers.CleanupContainer(t, rContainer) + + eContainer, err := runTestedSolarWindsOTELCollector(ctx, certPath, net.Name, config) + require.NoError(t, err) + testcontainers.CleanupContainer(t, eContainer) + + cmd := []string{ + signalType.String(), + fmt.Sprintf("--%s", signalType), strconv.Itoa(samplesCount), + "--otlp-insecure", + "--otlp-endpoint", fmt.Sprintf("%s:%d", testedContainer, port), + "--otlp-attributes", fmt.Sprintf("%s=\"%s\"", resourceAttributeName, resourceAttributeValue), + } + + gContainer, err := runGeneratorContainer(ctx, net.Name, cmd) + require.NoError(t, err) + testcontainers.CleanupContainer(t, gContainer) + + <-time.After(waitTime) + + return rContainer +} + type logConsumer struct { Prefix string } @@ -211,3 +260,24 @@ type logConsumer struct { func (lc *logConsumer) Accept(l testcontainers.Log) { log.Printf("***%s: %s", lc.Prefix, string(l.Content)) } + +type SignalType int + +const ( + Logs SignalType = iota + Metrics + Traces +) + +func (s SignalType) String() string { + switch s { + case Logs: + return "logs" + case Metrics: + return "metrics" + case Traces: + return "traces" + default: + panic("unexpected signal type") + } +} diff --git a/internal/e2e/signals_processing_test.go b/internal/e2e/signals_processing_test.go index 4940a46..029f1fd 100644 --- a/internal/e2e/signals_processing_test.go +++ b/internal/e2e/signals_processing_test.go @@ -18,18 +18,14 @@ package e2e import ( "context" - "fmt" "io" "log" - "strconv" "strings" "testing" - "time" "github.com/solarwinds/solarwinds-otel-collector/pkg/version" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/network" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/pmetric" @@ -45,72 +41,13 @@ const ( func TestMetricStream(t *testing.T) { ctx := context.Background() - - net, err := network.New(ctx) - require.NoError(t, err) - testcontainers.CleanupNetwork(t, net) - - certPath := t.TempDir() - _, err = generateCertificates(receivingContainer, certPath) - require.NoError(t, err) - - rContainer, err := runReceivingSolarWindsOTELCollector(ctx, certPath, net.Name) - require.NoError(t, err) - testcontainers.CleanupContainer(t, rContainer) - - eContainer, err := runTestedSolarWindsOTELCollector(ctx, certPath, net.Name) - require.NoError(t, err) - testcontainers.CleanupContainer(t, eContainer) - - cmd := []string{ - "metrics", - "--metrics", strconv.Itoa(samplesCount), - "--otlp-insecure", - "--otlp-endpoint", fmt.Sprintf("%s:%d", testedContainer, port), - "--otlp-attributes", fmt.Sprintf("%s=\"%s\"", resourceAttributeName, resourceAttributeValue), - } - - gContainer, err := runGeneratorContainer(ctx, net.Name, cmd) - require.NoError(t, err) - testcontainers.CleanupContainer(t, gContainer) - - <-time.After(collectorRunningPeriod) - + rContainer := startCollectorContainers(t, ctx, "emitting_collector.yaml", Metrics, collectorRunningPeriod) evaluateMetricsStream(t, ctx, rContainer, samplesCount) } func TestTracesStream(t *testing.T) { ctx := context.Background() - - net, err := network.New(ctx) - require.NoError(t, err) - testcontainers.CleanupNetwork(t, net) - - certPath := t.TempDir() - _, err = generateCertificates(receivingContainer, certPath) - require.NoError(t, err) - - rContainer, err := runReceivingSolarWindsOTELCollector(ctx, certPath, net.Name) - require.NoError(t, err) - testcontainers.CleanupContainer(t, rContainer) - - eContainer, err := runTestedSolarWindsOTELCollector(ctx, certPath, net.Name) - require.NoError(t, err) - testcontainers.CleanupContainer(t, eContainer) - - cmd := []string{ - "traces", - "--traces", strconv.Itoa(samplesCount), - "--otlp-insecure", - "--otlp-endpoint", fmt.Sprintf("%s:%d", testedContainer, port), - "--otlp-attributes", fmt.Sprintf("%s=\"%s\"", resourceAttributeName, resourceAttributeValue), - } - - gContainer, err := runGeneratorContainer(ctx, net.Name, cmd) - require.NoError(t, err) - testcontainers.CleanupContainer(t, gContainer) - - <-time.After(collectorRunningPeriod) + rContainer := startCollectorContainers(t, ctx, "emitting_collector.yaml", Traces, collectorRunningPeriod) // Traces coming in couples. expectedTracesCount := samplesCount * 2 @@ -119,37 +56,7 @@ func TestTracesStream(t *testing.T) { func TestLogsStream(t *testing.T) { ctx := context.Background() - - net, err := network.New(ctx) - require.NoError(t, err) - testcontainers.CleanupNetwork(t, net) - - certPath := t.TempDir() - _, err = generateCertificates(receivingContainer, certPath) - require.NoError(t, err) - - rContainer, err := runReceivingSolarWindsOTELCollector(ctx, certPath, net.Name) - require.NoError(t, err) - testcontainers.CleanupContainer(t, rContainer) - - eContainer, err := runTestedSolarWindsOTELCollector(ctx, certPath, net.Name) - require.NoError(t, err) - testcontainers.CleanupContainer(t, eContainer) - - cmd := []string{ - "logs", - "--logs", strconv.Itoa(samplesCount), - "--otlp-insecure", - "--otlp-endpoint", fmt.Sprintf("%s:%d", testedContainer, port), - "--otlp-attributes", fmt.Sprintf("%s=\"%s\"", resourceAttributeName, resourceAttributeValue), - } - - gContainer, err := runGeneratorContainer(ctx, net.Name, cmd) - require.NoError(t, err) - testcontainers.CleanupContainer(t, gContainer) - - <-time.After(collectorRunningPeriod) - + rContainer := startCollectorContainers(t, ctx, "emitting_collector.yaml", Logs, collectorRunningPeriod) evaluateLogsStream(t, ctx, rContainer, samplesCount) } @@ -283,6 +190,10 @@ func evaluateHeartbeatMetric( v2, available2 := atts.Get("custom_attribute") require.True(t, available2, "custom_attribute resource attribute must be available") require.Equal(t, "custom_attribute_value", v2.AsString(), "attribute value must be the same") + + v3, available3 := atts.Get("sw.otelcol.collector.entity_creation") + require.True(t, available3, "sw.otelcol.collector.entity_creation resource attribute must be available") + require.Equal(t, "on", v3.AsString(), "attribute value must be the same") } func evaluateResourceAttributes( diff --git a/internal/e2e/testdata/emitting_collector_without_entity.yaml b/internal/e2e/testdata/emitting_collector_without_entity.yaml new file mode 100644 index 0000000..6ba8c01 --- /dev/null +++ b/internal/e2e/testdata/emitting_collector_without_entity.yaml @@ -0,0 +1,28 @@ +service: + extensions: [solarwinds] + pipelines: + metrics: + receivers: [otlp] + exporters: [solarwinds] + traces: + receivers: [otlp] + exporters: [solarwinds] + logs: + receivers: [otlp] + exporters: [solarwinds] + +receivers: + otlp: + protocols: + grpc: + endpoint: :17016 + +extensions: + solarwinds: + token: + collector_name: "testing_collector_name" + endpoint_url_override: receiver:17016 + without_entity: true + +exporters: + solarwinds: diff --git a/internal/e2e/without_entity_test.go b/internal/e2e/without_entity_test.go new file mode 100644 index 0000000..b2abf36 --- /dev/null +++ b/internal/e2e/without_entity_test.go @@ -0,0 +1,53 @@ +// Copyright 2025 SolarWinds Worldwide, LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build e2e + +package e2e + +import ( + "context" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pmetric" + "testing" +) + +func TestWithoutEntity(t *testing.T) { + ctx := context.Background() + rContainer := startCollectorContainers(t, ctx, "emitting_collector_without_entity.yaml", Metrics, collectorRunningPeriod) + ms := pmetric.NewMetrics() + mum := new(pmetric.JSONUnmarshaler) + lines, _ := loadResultFile(ctx, rContainer, "/tmp/result.json") + for _, line := range lines { + // Metrics to process. + m, err := mum.UnmarshalMetrics([]byte(line)) + if err == nil && m.ResourceMetrics().Len() != 0 { + m.ResourceMetrics().MoveAndAppendTo(ms.ResourceMetrics()) + continue + } + } + evaluateHeartbeatMetricHasEntityCreationAsOff(t, ms) +} + +func evaluateHeartbeatMetricHasEntityCreationAsOff( + t *testing.T, + ms pmetric.Metrics, +) { + require.GreaterOrEqual(t, ms.ResourceMetrics().Len(), 1, "there must be at least one metric") + atts := ms.ResourceMetrics().At(0).Resource().Attributes() + + v, available := atts.Get("sw.otelcol.collector.entity_creation") + require.True(t, available, "sw.otelcol.collector.entity_creation resource attribute must be available") + require.Equal(t, "off", v.AsString(), "attribute value must be the same") +}