Capture configuration files & runtime config in support bundle (#2094)

* Capture configuration files & runtime config in support bundle * Update docs/sources/troubleshoot/support_bundle.md Co-authored-by: Clayton Cornell <[email protected]> * Implement POC of printing redacted secrets * Print quoted secret placeholder * Ensure we get the parsed AST for remotecfg, and ensure existing printers aren't affected * Add tests for printer redaction * Fix test value --------- Co-authored-by: Clayton Cornell <[email protected]>
grafana · Nov 22, 2024 · f24aeca · f24aeca
1 parent 3b57159
commit f24aeca
Show file tree

Hide file tree

Showing 15 changed files with 325 additions and 70 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,6 +24,8 @@ Main (unreleased)
 
 - Add second metrics sample to the support bundle to provide delta information (@dehaansa)
 
+- Add all raw configuration files & a copy of the latest remote config to the support bundle (@dehaansa)
+
 - Add relevant golang environment variables to the support bundle (@dehaansa)
 
 ### Bugfixes

diff --git a/docs/sources/troubleshoot/support_bundle.md b/docs/sources/troubleshoot/support_bundle.md
@@ -46,8 +46,11 @@ A support bundle contains the following data:
 * `alloy-runtime-flags.txt` contains the values of the runtime flags available in {{< param "PRODUCT_NAME" >}}.
 * The `pprof/` directory contains Go runtime profiling data (CPU, heap, goroutine, mutex, block profiles) as exported by the pprof package.
 Refer to the [profile][profile] documentation for more details on how to use this information.
+* The `sources/` directory contains copies of the local configuration files used to configure {{< param "PRODUCT_NAME" >}}.
+* `sources/remote-config/remote.alloy` contains a copy of the last received [remote configuration][remotecfg].
 
 [profile]: ../profile
 [components]: ../../get-started/components/
 [alloy-repo]: https://github.com/grafana/alloy/issues
-[backward-compatibility]: ../../introduction/backward-compatibility
+[backward-compatibility]: ../../introduction/backward-compatibility
+[remotecfg]: ../../reference/config-blocks/remotecfg/
diff --git a/internal/alloycli/cmd_run.go b/internal/alloycli/cmd_run.go
@@ -356,6 +356,7 @@ func (fr *alloyRun) Run(cmd *cobra.Command, configPath string) error {
 		if err != nil {
 			return nil, fmt.Errorf("reading config path %q: %w", configPath, err)
 		}
+		httpService.SetSources(alloySource.SourceFiles())
 		if err := f.LoadSource(alloySource, nil, configPath); err != nil {
 			return alloySource, fmt.Errorf("error during the initial load: %w", err)
 		}

diff --git a/internal/runtime/alloy_services.go b/internal/runtime/alloy_services.go
@@ -6,6 +6,7 @@ import (
 	"github.com/grafana/alloy/internal/runtime/internal/controller"
 	"github.com/grafana/alloy/internal/runtime/internal/dag"
 	"github.com/grafana/alloy/internal/service"
+	"github.com/grafana/alloy/syntax/ast"
 )
 
 // GetServiceConsumers implements [service.Host]. It returns a slice of
@@ -93,12 +94,12 @@ type ServiceController struct {
 }
 
 func (sc ServiceController) Run(ctx context.Context) { sc.f.Run(ctx) }
-func (sc ServiceController) LoadSource(b []byte, args map[string]any, configPath string) error {
+func (sc ServiceController) LoadSource(b []byte, args map[string]any, configPath string) (*ast.File, error) {
 	source, err := ParseSource("", b)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	return sc.f.LoadSource(source, args, configPath)
+	return source.SourceFiles()[""], sc.f.LoadSource(source, args, configPath)
 }
 func (sc ServiceController) Ready() bool { return sc.f.Ready() }
 

diff --git a/internal/runtime/source.go b/internal/runtime/source.go
@@ -15,6 +15,7 @@ import (
 // A Source holds the contents of a parsed Alloy configuration source module.
 type Source struct {
 	sourceMap map[string][]byte // Map that links parsed Alloy source's name with its content.
+	fileMap   map[string]*ast.File
 	hash      [sha256.Size]byte // Hash of all files in sourceMap sorted by name.
 
 	// Components holds the list of raw Alloy AST blocks describing components.
@@ -42,6 +43,7 @@ func ParseSource(name string, bb []byte) (*Source, error) {
 		return nil, err
 	}
 	source.sourceMap = map[string][]byte{name: bb}
+	source.fileMap = map[string]*ast.File{name: node}
 	source.hash = sha256.Sum256(bb)
 	return source, nil
 }
@@ -107,8 +109,12 @@ type namedSource struct {
 // Source. sources must not be modified after calling ParseSources.
 func ParseSources(sources map[string][]byte) (*Source, error) {
 	var (
-		mergedSource = &Source{sourceMap: sources} // Combined source from all the input content.
-		hash         = sha256.New()                // Combined hash of all the sources.
+		// Combined source from all the input content.
+		mergedSource = &Source{
+			sourceMap: sources,
+			fileMap:   make(map[string]*ast.File, len(sources)),
+		}
+		hash = sha256.New() // Combined hash of all the sources.
 	)
 
 	// Sorted slice so ParseSources always does the same thing.
@@ -132,6 +138,8 @@ func ParseSources(sources map[string][]byte) (*Source, error) {
 			return nil, err
 		}
 
+		mergedSource.fileMap[namedSource.Name] = sourceFragment.fileMap[namedSource.Name]
+
 		mergedSource.components = append(mergedSource.components, sourceFragment.components...)
 		mergedSource.configBlocks = append(mergedSource.configBlocks, sourceFragment.configBlocks...)
 		mergedSource.declareBlocks = append(mergedSource.declareBlocks, sourceFragment.declareBlocks...)
@@ -150,6 +158,15 @@ func (s *Source) RawConfigs() map[string][]byte {
 	return s.sourceMap
 }
 
+// SourceFiles returns the parsed source content used to create Source.
+// Do not modify the returned map.
+func (s *Source) SourceFiles() map[string]*ast.File {
+	if s == nil {
+		return nil
+	}
+	return s.fileMap
+}
+
 // SHA256 returns the sha256 checksum of the source.
 // Do not modify the returned byte array.
 func (s *Source) SHA256() [sha256.Size]byte {

diff --git a/internal/service/http/http.go b/internal/service/http/http.go
@@ -6,6 +6,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	_ "net/http/pprof" // Register pprof handlers
@@ -27,6 +28,8 @@ import (
 	"github.com/grafana/alloy/internal/service"
 	"github.com/grafana/alloy/internal/service/remotecfg"
 	"github.com/grafana/alloy/internal/static/server"
+	"github.com/grafana/alloy/syntax/ast"
+	"github.com/grafana/alloy/syntax/printer"
 	"github.com/grafana/ckit/memconn"
 	_ "github.com/grafana/pyroscope-go/godeltaprof/http/pprof" // Register godeltaprof handler
 	"github.com/prometheus/client_golang/prometheus"
@@ -78,6 +81,9 @@ type Service struct {
 	// Used to enforce single-flight requests to supportHandler
 	supportBundleMut sync.Mutex
 
+	// Track the raw config for use with the support bundle
+	sources map[string]*ast.File
+
 	// publicLis and tcpLis are used to lazily enable TLS, since TLS is
 	// optionally configurable at runtime.
 	//
@@ -225,7 +231,7 @@ func (s *Service) Run(ctx context.Context, host service.Host) error {
 	}
 
 	// Wire in support bundle generator
-	r.HandleFunc("/-/support", s.supportHandler).Methods("GET")
+	r.HandleFunc("/-/support", s.generateSupportBundleHandler(host)).Methods("GET")
 
 	// Wire custom service handlers for services which depend on the http
 	// service.
@@ -259,60 +265,80 @@ func (s *Service) Run(ctx context.Context, host service.Host) error {
 	return nil
 }
 
-func (s *Service) supportHandler(rw http.ResponseWriter, r *http.Request) {
-	s.supportBundleMut.Lock()
-	defer s.supportBundleMut.Unlock()
+func (s *Service) generateSupportBundleHandler(host service.Host) func(rw http.ResponseWriter, r *http.Request) {
+	return func(rw http.ResponseWriter, r *http.Request) {
+		s.supportBundleMut.Lock()
+		defer s.supportBundleMut.Unlock()
 
-	// TODO(dehaansa) remove this check once the support bundle is generally available
-	if !s.opts.MinStability.Permits(featuregate.StabilityPublicPreview) {
-		rw.WriteHeader(http.StatusForbidden)
-		_, _ = rw.Write([]byte("support bundle generation is only available in public preview. Use" +
-			" --stability.level command-line flag to enable public-preview features"))
-		return
-	}
+		// TODO(dehaansa) remove this check once the support bundle is generally available
+		if !s.opts.MinStability.Permits(featuregate.StabilityPublicPreview) {
+			rw.WriteHeader(http.StatusForbidden)
+			_, _ = rw.Write([]byte("support bundle generation is only available in public preview. Use" +
+				" --stability.level command-line flag to enable public-preview features"))
+			return
+		}
 
-	if s.opts.BundleContext.DisableSupportBundle {
-		rw.WriteHeader(http.StatusForbidden)
-		_, _ = rw.Write([]byte("support bundle generation is disabled; it can be re-enabled by removing the --disable-support-bundle flag"))
-		return
-	}
+		if s.opts.BundleContext.DisableSupportBundle {
+			rw.WriteHeader(http.StatusForbidden)
+			_, _ = rw.Write([]byte("support bundle generation is disabled; it can be re-enabled by removing the --disable-support-bundle flag"))
+			return
+		}
 
-	duration := getServerWriteTimeout(r)
-	if r.URL.Query().Has("duration") {
-		d, err := strconv.Atoi(r.URL.Query().Get("duration"))
+		duration := getServerWriteTimeout(r)
+		if r.URL.Query().Has("duration") {
+			d, err := strconv.Atoi(r.URL.Query().Get("duration"))
+			if err != nil {
+				http.Error(rw, fmt.Sprintf("duration value (in seconds) should be a positive integer: %s", err), http.StatusBadRequest)
+				return
+			}
+			if d < 1 {
+				http.Error(rw, "duration value (in seconds) should be larger than 1", http.StatusBadRequest)
+				return
+			}
+			if float64(d) > duration.Seconds() {
+				http.Error(rw, "duration value exceeds the server's write timeout", http.StatusBadRequest)
+				return
+			}
+			duration = time.Duration(d) * time.Second
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), duration)
+		defer cancel()
+
+		var logsBuffer bytes.Buffer
+		syncBuff := log.NewSyncWriter(&logsBuffer)
+		s.globalLogger.SetTemporaryWriter(syncBuff)
+		defer func() {
+			s.globalLogger.RemoveTemporaryWriter()
+		}()
+
+		// Get and redact the cached remote config.
+		cachedConfig, err := remoteCfgRedactedCachedConfig(host)
 		if err != nil {
-			http.Error(rw, fmt.Sprintf("duration value (in seconds) should be a positive integer: %s", err), http.StatusBadRequest)
-			return
+			level.Debug(s.log).Log("msg", "failed to get cached remote config", "err", err)
 		}
-		if d < 1 {
-			http.Error(rw, "duration value (in seconds) should be larger than 1", http.StatusBadRequest)
+
+		// Ensure the sources are written using the printer as it will handle
+		// secret redaction.
+		sources := redactedSources(s.sources)
+
+		bundle, err := ExportSupportBundle(ctx, s.opts.BundleContext.RuntimeFlags, s.opts.HTTPListenAddr, sources, cachedConfig, s.Data().(Data).DialFunc)
+		if err != nil {
+			http.Error(rw, err.Error(), http.StatusInternalServerError)
 			return
 		}
-		if float64(d) > duration.Seconds() {
-			http.Error(rw, "duration value exceeds the server's write timeout", http.StatusBadRequest)
+		if err := ServeSupportBundle(rw, bundle, &logsBuffer); err != nil {
+			http.Error(rw, err.Error(), http.StatusInternalServerError)
 			return
 		}
-		duration = time.Duration(d) * time.Second
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), duration)
-	defer cancel()
-
-	var logsBuffer bytes.Buffer
-	syncBuff := log.NewSyncWriter(&logsBuffer)
-	s.globalLogger.SetTemporaryWriter(syncBuff)
-	defer func() {
-		s.globalLogger.RemoveTemporaryWriter()
-	}()
+}
 
-	bundle, err := ExportSupportBundle(ctx, s.opts.BundleContext.RuntimeFlags, s.opts.HTTPListenAddr, s.Data().(Data).DialFunc)
-	if err != nil {
-		http.Error(rw, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	if err := ServeSupportBundle(rw, bundle, &logsBuffer); err != nil {
-		http.Error(rw, err.Error(), http.StatusInternalServerError)
-		return
-	}
+// SetSources sets the sources on reload to be delivered
+// with the support bundle.
+func (s *Service) SetSources(sources map[string]*ast.File) {
+	s.supportBundleMut.Lock()
+	defer s.supportBundleMut.Unlock()
+	s.sources = sources
 }
 
 func getServerWriteTimeout(r *http.Request) time.Duration {
@@ -582,6 +608,45 @@ func (lis *lazyListener) Addr() net.Addr {
 	return lis.inner.Addr()
 }
 
+func redactedSources(sources map[string]*ast.File) map[string][]byte {
+	if sources == nil {
+		return nil
+	}
+	printedSources := make(map[string][]byte, len(sources))
+
+	for k, v := range sources {
+		b, err := printFileRedacted(v)
+		if err != nil {
+			printedSources[k] = []byte(fmt.Errorf("failed to print source: %w", err).Error())
+			continue
+		}
+		printedSources[k] = b
+	}
+	return printedSources
+}
+
+func remoteCfgRedactedCachedConfig(host service.Host) ([]byte, error) {
+	svc, ok := host.GetService(remotecfg.ServiceName)
+	if !ok {
+		return nil, fmt.Errorf("failed to get the remotecfg service")
+	}
+
+	return printFileRedacted(svc.(*remotecfg.Service).GetCachedAstFile())
+}
+
+func printFileRedacted(f *ast.File) ([]byte, error) {
+	c := printer.Config{
+		RedactSecrets: true,
+	}
+
+	var buf bytes.Buffer
+	w := io.Writer(&buf)
+	if err := c.Fprint(w, f); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
 func remoteCfgHostProvider(host service.Host) func() (service.Host, error) {
 	return func() (service.Host, error) {
 		svc, ok := host.GetService(remotecfg.ServiceName)

diff --git a/internal/service/http/supportbundle.go b/internal/service/http/supportbundle.go
@@ -36,6 +36,8 @@ type Bundle struct {
 	peers                []byte
 	runtimeFlags         []byte
 	environmentVariables []byte
+	sources              map[string][]byte
+	remoteCfg            []byte
 	heapBuf              *bytes.Buffer
 	goroutineBuf         *bytes.Buffer
 	blockBuf             *bytes.Buffer
@@ -52,7 +54,7 @@ type Metadata struct {
 }
 
 // ExportSupportBundle gathers the information required for the support bundle.
-func ExportSupportBundle(ctx context.Context, runtimeFlags []string, srvAddress string, dialContext server.DialContextFunc) (*Bundle, error) {
+func ExportSupportBundle(ctx context.Context, runtimeFlags []string, srvAddress string, sources map[string][]byte, remoteCfg []byte, dialContext server.DialContextFunc) (*Bundle, error) {
 	var httpClient http.Client
 	httpClient.Transport = &http.Transport{DialContext: dialContext}
 
@@ -148,6 +150,8 @@ func ExportSupportBundle(ctx context.Context, runtimeFlags []string, srvAddress
 		alloyMetricsEnd:      alloyMetricsEnd,
 		components:           components,
 		peers:                peers,
+		sources:              sources,
+		remoteCfg:            remoteCfg,
 		runtimeFlags:         []byte(strings.Join(runtimeFlags, "\n")),
 		environmentVariables: []byte(strings.Join(retrieveEnvironmentVariables(), "\n")),
 		heapBuf:              &heapBuf,
@@ -208,19 +212,24 @@ func ServeSupportBundle(rw http.ResponseWriter, b *Bundle, logsBuf *bytes.Buffer
 	rw.Header().Set("Content-Disposition", "attachment; filename=\"alloy-support-bundle.zip\"")
 
 	zipStructure := map[string][]byte{
-		"alloy-metadata.yaml":            b.meta,
-		"alloy-components.json":          b.components,
-		"alloy-peers.json":               b.peers,
-		"alloy-metrics-sample-start.txt": b.alloyMetricsStart,
-		"alloy-metrics-sample-end.txt":   b.alloyMetricsEnd,
-		"alloy-runtime-flags.txt":        b.runtimeFlags,
-		"alloy-environment.txt":          b.environmentVariables,
-		"alloy-logs.txt":                 logsBuf.Bytes(),
-		"pprof/cpu.pprof":                b.cpuBuf.Bytes(),
-		"pprof/heap.pprof":               b.heapBuf.Bytes(),
-		"pprof/goroutine.pprof":          b.goroutineBuf.Bytes(),
-		"pprof/mutex.pprof":              b.mutexBuf.Bytes(),
-		"pprof/block.pprof":              b.blockBuf.Bytes(),
+		"alloy-metadata.yaml":                b.meta,
+		"alloy-components.json":              b.components,
+		"alloy-peers.json":                   b.peers,
+		"alloy-metrics-sample-start.txt":     b.alloyMetricsStart,
+		"alloy-metrics-sample-end.txt":       b.alloyMetricsEnd,
+		"alloy-runtime-flags.txt":            b.runtimeFlags,
+		"alloy-environment.txt":              b.environmentVariables,
+		"alloy-logs.txt":                     logsBuf.Bytes(),
+		"sources/remote-config/remote.alloy": b.remoteCfg,
+		"pprof/cpu.pprof":                    b.cpuBuf.Bytes(),
+		"pprof/heap.pprof":                   b.heapBuf.Bytes(),
+		"pprof/goroutine.pprof":              b.goroutineBuf.Bytes(),
+		"pprof/mutex.pprof":                  b.mutexBuf.Bytes(),
+		"pprof/block.pprof":                  b.blockBuf.Bytes(),
+	}
+
+	for p, s := range b.sources {
+		zipStructure[filepath.Join("sources", filepath.Base(p))] = s
 	}
 
 	for fn, b := range zipStructure {