From 4d57200692d5c5c39effad4ae3b29fef79209113 Mon Sep 17 00:00:00 2001 From: Justin Kolberg Date: Thu, 14 Nov 2024 12:06:23 -0800 Subject: [PATCH 1/5] fix(otelcol-config): update default remote config (#1701) * fix(otelcol-config): update default remote config Signed-off-by: Justin Kolberg * fix(otelcol-config): update remote tests Signed-off-by: Justin Kolberg --------- Signed-off-by: Justin Kolberg --- pkg/tools/otelcol-config/remote_control.go | 23 +++++++++++++++++- .../otelcol-config/remote_control_test.go | 24 +++++++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/pkg/tools/otelcol-config/remote_control.go b/pkg/tools/otelcol-config/remote_control.go index e81aee781f..e02f0750ad 100644 --- a/pkg/tools/otelcol-config/remote_control.go +++ b/pkg/tools/otelcol-config/remote_control.go @@ -53,11 +53,32 @@ func makeNewSumologicRemoteYAML(ctx *actionContext, conf ConfDir) error { var sumoRemoteConfig = map[string]any{ "extensions": map[string]any{ "opamp": map[string]any{ - "enabled": true, "remote_configuration_directory": remoteConfigDir, "endpoint": DefaultSumoLogicOpampEndpoint, }, }, + "receivers": map[string]any{ + "nop": map[string]any{}, + }, + "exporters": map[string]any{ + "nop": map[string]any{}, + }, + "service": map[string]any{ + "pipelines": map[string]any{ + "metrics/default": map[string]any{ + "receivers": []string{"nop"}, + "exporters": []string{"nop"}, + }, + "logs/default": map[string]any{ + "receivers": []string{"nop"}, + "exporters": []string{"nop"}, + }, + "traces/default": map[string]any{ + "receivers": []string{"nop"}, + "exporters": []string{"nop"}, + }, + }, + }, } buf := new(bytes.Buffer) diff --git a/pkg/tools/otelcol-config/remote_control_test.go b/pkg/tools/otelcol-config/remote_control_test.go index bee9ab3056..23c6c25e0d 100644 --- a/pkg/tools/otelcol-config/remote_control_test.go +++ b/pkg/tools/otelcol-config/remote_control_test.go @@ -76,11 +76,31 @@ func TestEnableRemoteControlConfigFileNotPresent(t *testing.T) { values := &flagValues{ EnableRemoteControl: true, } - const expData = `extensions: + const expData = `exporters: + nop: {} +extensions: opamp: - enabled: true endpoint: wss://opamp-events.sumologic.com/v1/opamp remote_configuration_directory: /etc/otelcol-sumo/opamp.d +receivers: + nop: {} +service: + pipelines: + logs/default: + exporters: + - nop + receivers: + - nop + metrics/default: + exporters: + - nop + receivers: + - nop + traces/default: + exporters: + - nop + receivers: + - nop ` slrWriter := newTestWriter([]byte(expData)) ctx := &actionContext{ From b1ea0a9a58b24648eea4b2fda6335700003e793a Mon Sep 17 00:00:00 2001 From: Francis Guimond Date: Thu, 14 Nov 2024 18:28:33 -0500 Subject: [PATCH 2/5] fix(packaging): cleanup build version number (#1703) Version numbers introduced by the new packaging implementation are not supported by the backend when sending collector information. When this new format is recognized clean it up to send a version that is compatible with the backend. --- pkg/extension/sumologicextension/extension.go | 20 ++++++++++- .../sumologicextension/extension_test.go | 34 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/pkg/extension/sumologicextension/extension.go b/pkg/extension/sumologicextension/extension.go index c66ec8a847..12211eb3c7 100644 --- a/pkg/extension/sumologicextension/extension.go +++ b/pkg/extension/sumologicextension/extension.go @@ -26,6 +26,7 @@ import ( "net/http" "net/url" "os" + "regexp" "strings" "sync" "time" @@ -798,7 +799,7 @@ func (se *SumologicExtension) updateMetadataWithHTTPClient(ctx context.Context, Environment: se.conf.CollectorEnvironment, }, CollectorDetails: api.OpenMetadataCollectorDetails{ - RunningVersion: se.buildVersion, + RunningVersion: cleanupBuildVersion(se.buildVersion), }, NetworkDetails: api.OpenMetadataNetworkDetails{ HostIpAddress: ip, @@ -1057,3 +1058,20 @@ func getHostname(logger *zap.Logger) (string, error) { return os.Hostname() } + +// cleanupBuildVersion adds a leading 'v' and removes the tailing build hash to make sure the +// backend understand the build number. Note that only version strings with the following format will be +// cleaned up. All other version formats will remain the same. +// Cleaned up format: 0.108.0-sumo-2-4d57200692d5c5c39effad4ae3b29fef79209113 +func cleanupBuildVersion(version string) string { + pattern := "(^[0-9]+\\.[0-9]+\\.[0-9]+-sumo-[0-9]+)-[0-9a-f]{40}$" + re := regexp.MustCompile(pattern) + + sm := re.FindAllStringSubmatch(version, 1) + if len(sm) == 1 { + ver := sm[0][1] + return "v" + ver + } + + return version +} diff --git a/pkg/extension/sumologicextension/extension_test.go b/pkg/extension/sumologicextension/extension_test.go index a3d733b156..bbe20d5a07 100644 --- a/pkg/extension/sumologicextension/extension_test.go +++ b/pkg/extension/sumologicextension/extension_test.go @@ -1593,3 +1593,37 @@ func TestUpdateMetadataRequestPayload(t *testing.T) { err = se.updateMetadataWithHTTPClient(context.TODO(), httpClient) require.NoError(t, err) } + +func Test_cleanupBuildVersion(t *testing.T) { + type args struct { + version string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "with hash", + args: args{version: "0.108.0-sumo-2-4d57200692d5c5c39effad4ae3b29fef79209113"}, + want: "v0.108.0-sumo-2", + }, { + name: "already ok", + args: args{version: "v0.108.0-sumo-2"}, + want: "v0.108.0-sumo-2", + }, { + name: "non v", + args: args{version: "0.108.0-sumo-2"}, + want: "0.108.0-sumo-2", + }, { + name: "nonsense", + args: args{version: "hfiwe-23rhc8eg.fhf"}, + want: "hfiwe-23rhc8eg.fhf", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, cleanupBuildVersion(tt.args.version), "cleanupBuildVersion(%v)", tt.args.version) + }) + } +} From 882f8971fef05a05f5854252dd67107ab5394a5a Mon Sep 17 00:00:00 2001 From: Justin Kolberg Date: Thu, 14 Nov 2024 15:48:02 -0800 Subject: [PATCH 3/5] fix(otelcol-config): add missing extension config (#1702) Signed-off-by: Justin Kolberg --- pkg/tools/otelcol-config/remote_control.go | 8 ++++++++ pkg/tools/otelcol-config/remote_control_test.go | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/pkg/tools/otelcol-config/remote_control.go b/pkg/tools/otelcol-config/remote_control.go index e02f0750ad..1b84258cea 100644 --- a/pkg/tools/otelcol-config/remote_control.go +++ b/pkg/tools/otelcol-config/remote_control.go @@ -56,6 +56,10 @@ func makeNewSumologicRemoteYAML(ctx *actionContext, conf ConfDir) error { "remote_configuration_directory": remoteConfigDir, "endpoint": DefaultSumoLogicOpampEndpoint, }, + "sumologic": map[string]any{ + "installation_token": "${SUMOLOGIC_INSTALLATION_TOKEN}", + "collector_credentials_directory": "/var/lib/otelcol-sumo/credentials", + }, }, "receivers": map[string]any{ "nop": map[string]any{}, @@ -64,6 +68,10 @@ func makeNewSumologicRemoteYAML(ctx *actionContext, conf ConfDir) error { "nop": map[string]any{}, }, "service": map[string]any{ + "extensions": []string{ + "sumologic", + "opamp", + }, "pipelines": map[string]any{ "metrics/default": map[string]any{ "receivers": []string{"nop"}, diff --git a/pkg/tools/otelcol-config/remote_control_test.go b/pkg/tools/otelcol-config/remote_control_test.go index 23c6c25e0d..59f6402a76 100644 --- a/pkg/tools/otelcol-config/remote_control_test.go +++ b/pkg/tools/otelcol-config/remote_control_test.go @@ -82,9 +82,15 @@ extensions: opamp: endpoint: wss://opamp-events.sumologic.com/v1/opamp remote_configuration_directory: /etc/otelcol-sumo/opamp.d + sumologic: + collector_credentials_directory: /var/lib/otelcol-sumo/credentials + installation_token: ${SUMOLOGIC_INSTALLATION_TOKEN} receivers: nop: {} service: + extensions: + - sumologic + - opamp pipelines: logs/default: exporters: From 6fa181e58b45fc7dfc3b8907ab2cc7c86864a5dc Mon Sep 17 00:00:00 2001 From: Cyril Cressent Date: Fri, 15 Nov 2024 11:02:27 -0800 Subject: [PATCH 4/5] Add nop components to OPAmp extension (#1704) --- pkg/extension/opampextension/components.go | 4 ++++ pkg/extension/opampextension/go.mod | 2 ++ pkg/extension/opampextension/go.sum | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/pkg/extension/opampextension/components.go b/pkg/extension/opampextension/components.go index 93810c7ba6..8113b5ca2d 100644 --- a/pkg/extension/opampextension/components.go +++ b/pkg/extension/opampextension/components.go @@ -3,6 +3,7 @@ package opampextension import ( "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/exporter/otlpexporter" + "go.opentelemetry.io/collector/exporter/nopexporter" "go.opentelemetry.io/collector/extension" "go.opentelemetry.io/collector/extension/ballastextension" @@ -12,6 +13,7 @@ import ( "go.opentelemetry.io/collector/processor/memorylimiterprocessor" "go.opentelemetry.io/collector/receiver" "go.opentelemetry.io/collector/receiver/otlpreceiver" + "go.opentelemetry.io/collector/receiver/nopreceiver" "go.uber.org/multierr" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sumologicexporter" @@ -59,6 +61,7 @@ func Components() ( errs = multierr.Append(errs, err) receivers, err := receiver.MakeFactoryMap( + nopreceiver.NewFactory(), apachereceiver.NewFactory(), filelogreceiver.NewFactory(), hostmetricsreceiver.NewFactory(), @@ -79,6 +82,7 @@ func Components() ( errs = multierr.Append(errs, err) exporters, err := exporter.MakeFactoryMap( + nopexporter.NewFactory(), otlpexporter.NewFactory(), sumologicexporter.NewFactory(), syslogexporter.NewFactory(), diff --git a/pkg/extension/opampextension/go.mod b/pkg/extension/opampextension/go.mod index 03410f027c..c82c7929fc 100644 --- a/pkg/extension/opampextension/go.mod +++ b/pkg/extension/opampextension/go.mod @@ -45,6 +45,7 @@ require ( go.opentelemetry.io/collector/confmap/provider/fileprovider v0.108.1 go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.108.1 go.opentelemetry.io/collector/exporter v0.108.1 + go.opentelemetry.io/collector/exporter/nopexporter v0.108.0 go.opentelemetry.io/collector/exporter/otlpexporter v0.108.0 go.opentelemetry.io/collector/extension v0.108.1 go.opentelemetry.io/collector/extension/ballastextension v0.108.0 @@ -54,6 +55,7 @@ require ( go.opentelemetry.io/collector/processor/batchprocessor v0.108.0 go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.108.0 go.opentelemetry.io/collector/receiver v0.108.1 + go.opentelemetry.io/collector/receiver/nopreceiver v0.108.0 go.opentelemetry.io/collector/receiver/otlpreceiver v0.108.0 go.opentelemetry.io/collector/semconv v0.108.1 go.uber.org/multierr v1.11.0 diff --git a/pkg/extension/opampextension/go.sum b/pkg/extension/opampextension/go.sum index 620356b449..564c517df6 100644 --- a/pkg/extension/opampextension/go.sum +++ b/pkg/extension/opampextension/go.sum @@ -765,6 +765,8 @@ go.opentelemetry.io/collector/consumer/consumertest v0.108.1 h1:WLvi27Vu5nkltjhm go.opentelemetry.io/collector/consumer/consumertest v0.108.1/go.mod h1:tRqOtUukG76iBlPTAUwFSU87dDO+x33Gyn2x9mecfko= go.opentelemetry.io/collector/exporter v0.108.1 h1:VjtbIwwR2B1GMf69FxVvwcKYIpbkC7v2wBxLt5vjYKI= go.opentelemetry.io/collector/exporter v0.108.1/go.mod h1:nD32bomG/yYyqwTZQJFIhRhP++bbk+3oqlx+/RZZ6XY= +go.opentelemetry.io/collector/exporter/nopexporter v0.108.0 h1:zgiD7XdMDPjGFOoSRnqgDX3yk+UCi1y8ahceufyQadM= +go.opentelemetry.io/collector/exporter/nopexporter v0.108.0/go.mod h1:7lS/dsgHJIwp7qHvn5O5zOh6btfNl82yFUrJ1zb5olE= go.opentelemetry.io/collector/exporter/otlpexporter v0.108.0 h1:QnfkiAZz9SyAxuzpKur+hrCNsfUjJBejWBQzvp8lejI= go.opentelemetry.io/collector/exporter/otlpexporter v0.108.0/go.mod h1:JbP/ysqzLTFI4i0BAo+ykSdhDbq0LAbTYjGSpbruJSc= go.opentelemetry.io/collector/extension v0.108.1 h1:XNQ9bOegD38gktyLJXlGN2Wb6AbiBi2nAuiOIdg+zA0= @@ -799,6 +801,8 @@ go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.108.0 h1:4eIMg go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.108.0/go.mod h1:bVGW/AR9/qV72YoSgaxVCSM8bJt1QII9Ua2ZGyBNHeU= go.opentelemetry.io/collector/receiver v0.108.1 h1:YQgDv69v3fgd6uoiGZ+vUdUPdNzoodbLzjB7XfdQvxs= go.opentelemetry.io/collector/receiver v0.108.1/go.mod h1:eKe/VJgdvHr8JsBDma/PF3DlaheTRC2X6AmCUByJCNU= +go.opentelemetry.io/collector/receiver/nopreceiver v0.108.0 h1:t+cdmQoXIzRKjxTpyneV5Ok+RHrh1P28QYHjBEOcK6Y= +go.opentelemetry.io/collector/receiver/nopreceiver v0.108.0/go.mod h1:luQPHxdk59Icw1pJotYdcxKyZDj/TYh+pzA7+rkv280= go.opentelemetry.io/collector/receiver/otlpreceiver v0.108.0 h1:SHQM1MsxshDStIF8WIh76B3fsVkw3TpAW+gWxefegzc= go.opentelemetry.io/collector/receiver/otlpreceiver v0.108.0/go.mod h1:VHom6BQVWOnCdR+iZRAkRbEspGqBOdfMcnC78HKMVAo= go.opentelemetry.io/collector/semconv v0.108.1 h1:Txk9tauUnamZaxS5vlf1O0uZ4VD6nioRBR0nX8L/fU4= From 74a9060c41224a13f8018f8ab511a167d8589bdf Mon Sep 17 00:00:00 2001 From: Justin Kolberg Date: Fri, 15 Nov 2024 16:27:33 -0800 Subject: [PATCH 5/5] fix(opamp): multiple fixes for opamp (#1705) * Uses new sumologicextension from our fork of opentelemetry-collector-contrib. This is necessary for now as the changes are not available on upstream and upstream is using a newer version of the collector. * Updated the OpAmp agent to output the filename (includes the source template ID) when processing source templates. * Added the `file_storage` and `health_check` extensions to the generated `sumologic-remote.yaml` config when using otelcol-config. Signed-off-by: Justin Kolberg --- otelcolbuilder/.otelcol-builder.yaml | 2 +- pkg/extension/opampextension/opamp_agent.go | 24 ++++++++++++------- .../opampextension/opamp_agent_test.go | 2 +- pkg/tools/otelcol-config/remote_control.go | 12 ++++++++++ .../otelcol-config/remote_control_test.go | 9 +++++++ 5 files changed, 38 insertions(+), 11 deletions(-) diff --git a/otelcolbuilder/.otelcol-builder.yaml b/otelcolbuilder/.otelcol-builder.yaml index 13c2f54be4..fe579a9f2b 100644 --- a/otelcolbuilder/.otelcol-builder.yaml +++ b/otelcolbuilder/.otelcol-builder.yaml @@ -227,7 +227,6 @@ providers: - gomod: go.opentelemetry.io/collector/confmap/provider/httpsprovider v0.108.0 - gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.108.0 - excludes: - github.com/knadh/koanf v1.5.0 @@ -243,5 +242,6 @@ replaces: # version which gets then translated into a replace in go.mod file. # This does not replace the version that sumologicexporter depends on. - github.com/SumoLogic/sumologic-otel-collector/pkg/extension/opampextension => ../../pkg/extension/opampextension + - github.com/open-telemetry/opentelemetry-collector-contrib/extension/sumologicextension v0.108.0 => github.com/SumoLogic/opentelemetry-collector-contrib/extension/sumologicextension bf012a09b9651cddda4a907ad4ff17834a0b81ef - github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/globprovider => ../../pkg/configprovider/globprovider - github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/opampprovider => ../../pkg/configprovider/opampprovider diff --git a/pkg/extension/opampextension/opamp_agent.go b/pkg/extension/opampextension/opamp_agent.go index 31e2f180b6..9d474ae8e6 100644 --- a/pkg/extension/opampextension/opamp_agent.go +++ b/pkg/extension/opampextension/opamp_agent.go @@ -406,7 +406,17 @@ func (o *opampAgent) saveEffectiveConfig(dir string) error { } } + o.logger.Debug("Loading Component Factories...") + factories, err := Components() + if err != nil { + return fmt.Errorf("cannot get the list of factories: %v", err) + } + for k, v := range o.effectiveConfig { + logger := o.logger.With( + zap.String("filename", k), + ) + p := filepath.Join(dir, k) // OpenFile the same way os.Create does it, but with 0600 perms @@ -420,12 +430,8 @@ func (o *opampAgent) saveEffectiveConfig(dir string) error { if err != nil { return err } - o.logger.Debug("Loading Component Factories...") - factories, err := Components() - if err != nil { - return fmt.Errorf("cannot get the list of factories: %v", err) - } - o.logger.Info("Loading Configuration to Validate...") + + logger.Info("Loading Configuration to Validate...") _, errValidate := loadConfigAndValidateWithSettings(factories, otelcol.ConfigProviderSettings{ ResolverSettings: confmap.ResolverSettings{ @@ -438,10 +444,10 @@ func (o *opampAgent) saveEffectiveConfig(dir string) error { }, }) if errValidate != nil { - o.logger.Error("Validation Failed... %v", zap.Error(errValidate)) - return fmt.Errorf("cannot validate config named %v", errValidate) + logger.Error("Validation Failed", zap.Error(errValidate)) + return fmt.Errorf("cannot validate config: %v", errValidate) } - o.logger.Info("Config Validation Successful...") + logger.Info("Config Validation Successful") } return nil diff --git a/pkg/extension/opampextension/opamp_agent_test.go b/pkg/extension/opampextension/opamp_agent_test.go index 5e70d7154a..9a2e8b4ac7 100644 --- a/pkg/extension/opampextension/opamp_agent_test.go +++ b/pkg/extension/opampextension/opamp_agent_test.go @@ -34,7 +34,7 @@ import ( const ( errMsgRemoteConfigNotAccepted = "OpAMP agent does not accept remote configuration" - errMsgInvalidConfigName = "cannot validate config named " + + errMsgInvalidConfigName = "cannot validate config: " + "service::pipelines::logs/localfilesource/0aa79379-c764-4d3d-9d66-03f6df029a07: " + "references processor \"batch\" which is not configured" errMsgInvalidInterval = "'max_elapsed_time' must be non-negative" diff --git a/pkg/tools/otelcol-config/remote_control.go b/pkg/tools/otelcol-config/remote_control.go index 1b84258cea..357822516b 100644 --- a/pkg/tools/otelcol-config/remote_control.go +++ b/pkg/tools/otelcol-config/remote_control.go @@ -52,6 +52,16 @@ func makeNewSumologicRemoteYAML(ctx *actionContext, conf ConfDir) error { var sumoRemoteConfig = map[string]any{ "extensions": map[string]any{ + "file_storage": map[string]any{ + "compaction": map[string]any{ + "directory": "/var/lib/otelcol-sumo/file_storage", + "on_rebound": true, + }, + "directory": "/var/lib/otelcol-sumo/file_storage", + }, + "health_check": map[string]any{ + "endpoint": "localhost:13133", + }, "opamp": map[string]any{ "remote_configuration_directory": remoteConfigDir, "endpoint": DefaultSumoLogicOpampEndpoint, @@ -70,6 +80,8 @@ func makeNewSumologicRemoteYAML(ctx *actionContext, conf ConfDir) error { "service": map[string]any{ "extensions": []string{ "sumologic", + "health_check", + "file_storage", "opamp", }, "pipelines": map[string]any{ diff --git a/pkg/tools/otelcol-config/remote_control_test.go b/pkg/tools/otelcol-config/remote_control_test.go index 59f6402a76..bae5ef07ac 100644 --- a/pkg/tools/otelcol-config/remote_control_test.go +++ b/pkg/tools/otelcol-config/remote_control_test.go @@ -79,6 +79,13 @@ func TestEnableRemoteControlConfigFileNotPresent(t *testing.T) { const expData = `exporters: nop: {} extensions: + file_storage: + compaction: + directory: /var/lib/otelcol-sumo/file_storage + on_rebound: true + directory: /var/lib/otelcol-sumo/file_storage + health_check: + endpoint: localhost:13133 opamp: endpoint: wss://opamp-events.sumologic.com/v1/opamp remote_configuration_directory: /etc/otelcol-sumo/opamp.d @@ -90,6 +97,8 @@ receivers: service: extensions: - sumologic + - health_check + - file_storage - opamp pipelines: logs/default: