chore: Replace dependency on unexported otelcol functions (#2148)

* Replace dependency on unexported otelcol functions

* Remove unused test code

* Make the endpoint_params output consistent with previous behavior

* Remove the unnecessary conversions to raw types in static config

go.mod

@@ -920,18 +920,6 @@ replace (
 	github.com/prometheus/node_exporter => github.com/grafana/node_exporter v0.18.1-grafana-r01.0.20231004161416-702318429731
 )
 
-// Replacing for an internal fork which allows us to observe metrics produced by the Collector.
-// This is a temporary solution while a new configuration design is discussed for the collector. Related issues:
-// https://github.com/open-telemetry/opentelemetry-collector/issues/7532
-// https://github.com/open-telemetry/opentelemetry-collector/pull/7644
-// https://github.com/open-telemetry/opentelemetry-collector/pull/7696
-// https://github.com/open-telemetry/opentelemetry-collector/issues/4970
-replace (
-	go.opentelemetry.io/collector/otelcol => github.com/grafana/opentelemetry-collector/otelcol v0.0.0-20241104164848-8ea9d0a3e17a
-	go.opentelemetry.io/collector/processor/batchprocessor => github.com/grafana/opentelemetry-collector/processor/batchprocessor v0.0.0-20241104164848-8ea9d0a3e17a
-	go.opentelemetry.io/collector/service => github.com/grafana/opentelemetry-collector/service v0.0.0-20241104164848-8ea9d0a3e17a
-)
-
 replace github.com/github/smimesign => github.com/grafana/smimesign v0.2.1-0.20220408144937-2a5adf3481d3
 
 // Submodules.

go.sum

@@ -1875,12 +1875,6 @@ github.com/grafana/opentelemetry-collector-contrib/receiver/prometheusreceiver v
 github.com/grafana/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20240326165551-1ae1b9218b1b/go.mod h1:XVaL+FyxEWEfHkDI/X6FJgFYDA0080A1/qoOHQ43dxM=
 github.com/grafana/opentelemetry-collector/featuregate v0.0.0-20240325174506-2fd1623b2ca0 h1:i/Ne0XwoRokYj52ZcSmnvuyID3h/uA91n0Ycg/grHU8=
 github.com/grafana/opentelemetry-collector/featuregate v0.0.0-20240325174506-2fd1623b2ca0/go.mod h1:mm8+xyQfgDmqhyegZRNIQmoKsNnDTwWKFLsdMoXAb7A=
-github.com/grafana/opentelemetry-collector/otelcol v0.0.0-20241104164848-8ea9d0a3e17a h1:6TnkhPwEjuleHfhKSw+IyC/I2cQORuEsImZZMhoodIE=
-github.com/grafana/opentelemetry-collector/otelcol v0.0.0-20241104164848-8ea9d0a3e17a/go.mod h1:H/HurP0qCcwcWdDRgvTJ/FRrVLnt++agxzHvgLQn/Ew=
-github.com/grafana/opentelemetry-collector/processor/batchprocessor v0.0.0-20241104164848-8ea9d0a3e17a h1:WrDY3pU4EkHsCIepUAYWt7lIfsMEPdugmYB8k6RYjMo=
-github.com/grafana/opentelemetry-collector/processor/batchprocessor v0.0.0-20241104164848-8ea9d0a3e17a/go.mod h1:QLQ31rGjPuMc/nGw4rL4HzQI9F0jVAPEmC342chxoqA=
-github.com/grafana/opentelemetry-collector/service v0.0.0-20241104164848-8ea9d0a3e17a h1:ZycgUSrrwtB2x1fMdLD88J2k8886/HvX1tYHlaOH/hg=
-github.com/grafana/opentelemetry-collector/service v0.0.0-20241104164848-8ea9d0a3e17a/go.mod h1:VTLnax+DjHal3q7WKQO0ITjWdfPTq2txaoNRcVXYzgE=
 github.com/grafana/postgres_exporter v0.15.1-0.20241105053755-e0a51174f168 h1:I7FyVTtge/3G5YHVOMDG0l4If6W+kXbFDqtzj5gCSGs=
 github.com/grafana/postgres_exporter v0.15.1-0.20241105053755-e0a51174f168/go.mod h1:dMrETGkSetWByp2XGsm8g6pRVh/ibnrDxKsN4BqnGNg=
 github.com/grafana/prometheus v1.8.2-0.20240514135907-13889ba362e6 h1:kih3d3M3dxAmrpFLvnIxFzWx8KMQyKxQwKgWP67C/Fg=
@@ -3351,6 +3345,8 @@ go.opentelemetry.io/collector/filter v0.112.0 h1:xX0MIfXOkbcWzCcmNqvtpnBDnSZAJmN
 go.opentelemetry.io/collector/filter v0.112.0/go.mod h1:ZcPbD9CLxqcQJ5D2dV2Ma0Hm2IKMOYggTLW8dDdZQSQ=
 go.opentelemetry.io/collector/internal/memorylimiter v0.112.0 h1:u1hUa48x1qEONUSOtz8dx/c8oz74RpIHyWnGFJ7t0CE=
 go.opentelemetry.io/collector/internal/memorylimiter v0.112.0/go.mod h1:BtHruDt40QTW8klZVQCqsVfhVsOkh6hDg5w1cPvLpeU=
+go.opentelemetry.io/collector/otelcol v0.112.0 h1:xOq7z5WK5jS1Qg5w+l99H1EiQRq9rHHDv7EIiLryldw=
+go.opentelemetry.io/collector/otelcol v0.112.0/go.mod h1:H/HurP0qCcwcWdDRgvTJ/FRrVLnt++agxzHvgLQn/Ew=
 go.opentelemetry.io/collector/otelcol/otelcoltest v0.112.0 h1:AdjoVnYl7RxoOvhWZcJb0SWY1VvlRT1cdlCwHBpn9vs=
 go.opentelemetry.io/collector/otelcol/otelcoltest v0.112.0/go.mod h1:VSbEYgmiSM5K6p501XD35QuhxbDpkxrfS2Wf5OKnHPs=
 go.opentelemetry.io/collector/pdata v1.18.0 h1:/yg2rO2dxqDM2p6GutsMCxXN6sKlXwyIz/ZYyUPONBg=
@@ -3365,6 +3361,8 @@ go.opentelemetry.io/collector/pipeline/pipelineprofiles v0.112.0 h1:opXGNrlJAjYR
 go.opentelemetry.io/collector/pipeline/pipelineprofiles v0.112.0/go.mod h1:c9yn4x+vY3G10eLCRuUu/oH7Y8YdE/BsgmLWmfHkaNY=
 go.opentelemetry.io/collector/processor v0.112.0 h1:nMv9DOBYR9MB78ddUgY3A3ytwAwk3t4HQMNIu+w8o0g=
 go.opentelemetry.io/collector/processor v0.112.0/go.mod h1:AJ8EHq8Z/ev90f4gU6G5ULUncdpWmBRATYk8ioR3pvw=
+go.opentelemetry.io/collector/processor/batchprocessor v0.112.0 h1:Dq/RpdClawI8HrnSi177LziPjfHo733BWOCgRTbWrfY=
+go.opentelemetry.io/collector/processor/batchprocessor v0.112.0/go.mod h1:QLQ31rGjPuMc/nGw4rL4HzQI9F0jVAPEmC342chxoqA=
 go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.112.0 h1:+V+4OUcg1s3CrZpttT4dA+Uuv7VWpOIPQpOkcsrMBIo=
 go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.112.0/go.mod h1:f/eEZ3JMbRNLsRzNwATtTjuulDrLvhYMvXinLrmHtTU=
 go.opentelemetry.io/collector/processor/processorprofiles v0.112.0 h1:Aef68SAbmBbhbsZZPuZb0ECwkV05vIcHIizGOGbWsbM=
@@ -3379,6 +3377,8 @@ go.opentelemetry.io/collector/receiver/receiverprofiles v0.112.0 h1:SShkZsWRsFss
 go.opentelemetry.io/collector/receiver/receiverprofiles v0.112.0/go.mod h1:615smszDXiz4YWwXslxlAjX7FzOVDU7Bk6xARFk+zpk=
 go.opentelemetry.io/collector/semconv v0.112.0 h1:JPQyvZhlNLVSuVI+FScONaiFygB7h7NTZceUEKIQUEc=
 go.opentelemetry.io/collector/semconv v0.112.0/go.mod h1:zCJ5njhWpejR+A40kiEoeFm1xq1uzyZwMnRNX6/D82A=
+go.opentelemetry.io/collector/service v0.112.0 h1:SI5bwPrRHLRn/kR9AoSSDX/8vaKFe+NMYloAcXfWMSE=
+go.opentelemetry.io/collector/service v0.112.0/go.mod h1:VTLnax+DjHal3q7WKQO0ITjWdfPTq2txaoNRcVXYzgE=
 go.opentelemetry.io/contrib/config v0.10.0 h1:2JknAzMaYjxrHkTnZh3eOme/Y2P5eHE2SWfhfV6Xd6c=
 go.opentelemetry.io/contrib/config v0.10.0/go.mod h1:aND2M6/KfNkntI5cyvHriR/zvZgPf8j9yETdSmvpfmc=
 go.opentelemetry.io/contrib/detectors/aws/ec2 v1.28.0 h1:d+y/wygENfwEbVpo7c3A9GfnMhoTiepQcthQSh+Mc9g=

internal/static/traces/config.go

@@ -1,6 +1,8 @@
 package traces
 
 import (
+	"bytes"
+	"context"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -24,7 +26,9 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	prom_config "github.com/prometheus/common/config"
 	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/config/configopaque"
 	"go.opentelemetry.io/collector/confmap"
+	"go.opentelemetry.io/collector/confmap/provider/yamlprovider"
 	otelexporter "go.opentelemetry.io/collector/exporter"
 	"go.opentelemetry.io/collector/exporter/otlpexporter"
 	"go.opentelemetry.io/collector/exporter/otlphttpexporter"
@@ -277,25 +281,6 @@ type TLSClientSetting struct {
 	ServerNameOverride       string        `yaml:"server_name_override,omitempty"`
 }
 
-func (c TLSClientSetting) toOtelConfig() map[string]interface{} {
-	m := make(map[string]interface{}, 0)
-	m["ca_file"] = c.CAFile
-	m["ca_pem"] = c.CAPem
-	m["include_system_ca_certs_pool"] = c.IncludeSystemCACertsPool
-	m["cert_file"] = c.CertFile
-	m["cert_pem"] = c.CertPem
-	m["key_file"] = c.KeyFile
-	m["key_pem"] = c.KeyPem
-	m["min_version"] = c.MinVersion
-	m["max_version"] = c.MaxVersion
-	m["cipher_suites"] = c.CipherSuites
-	m["reload_interval"] = c.ReloadInterval
-	m["insecure"] = c.Insecure
-	m["insecure_skip_verify"] = c.InsecureSkipVerify
-	m["server_name_override"] = c.ServerNameOverride
-	return m
-}
-
 // OAuth2Config configures the oauth2client extension for a remote_write exporter
 // compatible with oauth2clientauthextension.Config
 type OAuth2Config struct {
@@ -308,21 +293,6 @@ type OAuth2Config struct {
 	Timeout        time.Duration    `yaml:"timeout,omitempty"`
 }
 
-// Agent uses standard YAML unmarshalling, while the oauth2clientauthextension relies on
-// mapstructure without providing YAML labels. `toOtelConfig` marshals `Oauth2Config` to configuration type expected by
-// the oauth2clientauthextension Extension Factory
-func (c OAuth2Config) toOtelConfig() map[string]interface{} {
-	m := make(map[string]interface{}, 0)
-	m["client_id"] = c.ClientID
-	m["client_secret"] = c.ClientSecret
-	m["endpoint_params"] = c.EndpointParams
-	m["token_url"] = c.TokenURL
-	m["scopes"] = c.Scopes
-	m["tls"] = c.TLS.toOtelConfig()
-	m["timeout"] = c.Timeout
-	return m
-}
-
 // RemoteWriteConfig controls the configuration of an exporter
 type RemoteWriteConfig struct {
 	Endpoint    string `yaml:"endpoint,omitempty"`
@@ -571,8 +541,7 @@ func (c *InstanceConfig) extensions() (map[string]interface{}, error) {
 		if err != nil {
 			return nil, err
 		}
-		oauthConfig := remoteWriteConfig.Oauth2.toOtelConfig()
-		extensions[getAuthExtensionName(exporterName)] = oauthConfig
+		extensions[getAuthExtensionName(exporterName)] = remoteWriteConfig.Oauth2
 	}
 	if c.JaegerRemoteSampling != nil {
 		if len(c.JaegerRemoteSampling) == 0 {
@@ -1002,26 +971,46 @@ func orderProcessors(processors []string, splitPipelines bool) [][]string {
 }
 
 func otelcolConfigFromStringMap(otelMapStructure map[string]interface{}, factories *otelcol.Factories) (*otelcol.Config, error) {
-	configMap := confmap.NewFromStringMap(otelMapStructure)
-	otelCfg, err := otelcol.Unmarshal(configMap, *factories)
+	var b bytes.Buffer
+	enc := yaml.NewEncoder(&b)
+
+	enc.SetHook(func(in interface{}) (ok bool, out interface{}, err error) {
+		switch v := in.(type) {
+		case SecretString:
+			return true, string(v), nil
+		case configopaque.String:
+			return true, string(v), nil
+		default:
+			return false, nil, nil
+		}
+	})
+
+	if err := enc.Encode(otelMapStructure); err != nil {
+		return nil, err
+	}
+	cp, err := otelcol.NewConfigProvider(otelcol.ConfigProviderSettings{
+		ResolverSettings: confmap.ResolverSettings{
+			URIs: []string{"yaml:" + string(b.Bytes())},
+			ProviderFactories: []confmap.ProviderFactory{
+				yamlprovider.NewFactory(),
+			},
+		},
+	})
+
 	if err != nil {
-		return nil, fmt.Errorf("failed to load OTel config: %w", err)
+		return nil, fmt.Errorf("failed to create config provider: %w", err)
 	}
 
-	res := otelcol.Config{
-		Receivers:  otelCfg.Receivers.Configs(),
-		Processors: otelCfg.Processors.Configs(),
-		Exporters:  otelCfg.Exporters.Configs(),
-		Connectors: otelCfg.Connectors.Configs(),
-		Extensions: otelCfg.Extensions.Configs(),
-		Service:    otelCfg.Service,
+	otelCfg, err := cp.Get(context.Background(), *factories)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load OTel config: %w", err)
 	}
 
-	if err := res.Validate(); err != nil {
+	if err := otelCfg.Validate(); err != nil {
 		return nil, err
 	}
 
-	return &res, nil
+	return otelCfg, nil
 }
 
 // Code taken from OTel's service/configcheck.go

internal/static/traces/config_test.go

@@ -1160,8 +1160,6 @@ remote_write:
       scopes: ["api.metrics"]
       timeout: 2s
 `,
-			// The tls.ciphersuites would appear that it should output nothing or nil but during the conversion otelcolConfigFromStringMap the otelcol.Unmarshal converts the nil array to a blank one []. This is specifically used
-			// in confmap.go that transforms nil arrays into [] on the zeroSliceHookFunc.
 			expectedConfig: `
 receivers:
   push_receiver: {}
@@ -1175,8 +1173,6 @@ extensions:
     token_url: https://example.com/oauth2/default/v1/token
     scopes: ["api.metrics"]
     timeout: 2s
-    tls:
-      cipher_suites: []
 exporters:
   otlphttp/0:
     endpoint: example.com:12345
@@ -1245,7 +1241,6 @@ extensions:
       key_file: keyfile
       min_version: "1.3"
       reload_interval: 1h
-      cipher_suites: []
 exporters:
   otlphttp/0:
     endpoint: example.com:12345
@@ -1314,7 +1309,6 @@ extensions:
       key_pem: test_secret_key_pem_string
       max_version: "1.2"
       reload_interval: 1h
-      cipher_suites: []
 exporters:
   otlphttp/0:
     endpoint: example.com:12345
@@ -1371,16 +1365,12 @@ extensions:
    token_url: https://example.com/oauth2/default/v1/token
    scopes: ["api.metrics"]
    timeout: 2s
-   tls:
-     cipher_suites: []
  oauth2client/otlp1:
    client_id: anotherclientid
    client_secret: anotherclientsecret
    token_url: https://example.com/oauth2/default/v1/token
    scopes: ["api.metrics"]
    timeout: 2s
-   tls:
-     cipher_suites: []
 exporters:
   otlphttp/0:
     endpoint: example.com:12345
@@ -1441,7 +1431,6 @@ extensions:
     timeout: 2s
     tls:
       insecure: true
-      cipher_suites: []
 exporters:
   otlphttp/0:
     endpoint: http://example.com:12345
@@ -1545,7 +1534,7 @@ service:
 			sortService(actualConfig)
 			sortService(expectedConfig)
 
-			assert.Equal(t, *expectedConfig, *actualConfig)
+			assert.Equal(t, expectedConfig, actualConfig)
 		})
 	}
 }