From f24aecab5fe813ed93edc53e843d182df7e87749 Mon Sep 17 00:00:00 2001 From: Sam DeHaan Date: Fri, 22 Nov 2024 15:01:47 -0500 Subject: [PATCH] Capture configuration files & runtime config in support bundle (#2094) * Capture configuration files & runtime config in support bundle * Update docs/sources/troubleshoot/support_bundle.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Implement POC of printing redacted secrets * Print quoted secret placeholder * Ensure we get the parsed AST for remotecfg, and ensure existing printers aren't affected * Add tests for printer redaction * Fix test value --------- Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- CHANGELOG.md | 2 + docs/sources/troubleshoot/support_bundle.md | 5 +- internal/alloycli/cmd_run.go | 1 + internal/runtime/alloy_services.go | 7 +- internal/runtime/source.go | 21 ++- internal/service/http/http.go | 153 +++++++++++++------ internal/service/http/supportbundle.go | 37 +++-- internal/service/remotecfg/remotecfg.go | 21 ++- internal/service/remotecfg/remotecfg_test.go | 7 +- internal/service/service.go | 3 +- syntax/ast/ast.go | 44 ++++++ syntax/printer/printer.go | 11 +- syntax/printer/printer_test.go | 29 ++++ syntax/vm/struct_decoder.go | 7 + syntax/vm/vm_block_test.go | 47 ++++++ 15 files changed, 325 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4aae44cf88..1c325e6043 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ Main (unreleased) - Add second metrics sample to the support bundle to provide delta information (@dehaansa) +- Add all raw configuration files & a copy of the latest remote config to the support bundle (@dehaansa) + - Add relevant golang environment variables to the support bundle (@dehaansa) ### Bugfixes diff --git a/docs/sources/troubleshoot/support_bundle.md b/docs/sources/troubleshoot/support_bundle.md index cbd910a0f3..e027330a4b 100644 --- a/docs/sources/troubleshoot/support_bundle.md +++ b/docs/sources/troubleshoot/support_bundle.md @@ -46,8 +46,11 @@ A support bundle contains the following data: * `alloy-runtime-flags.txt` contains the values of the runtime flags available in {{< param "PRODUCT_NAME" >}}. * The `pprof/` directory contains Go runtime profiling data (CPU, heap, goroutine, mutex, block profiles) as exported by the pprof package. Refer to the [profile][profile] documentation for more details on how to use this information. +* The `sources/` directory contains copies of the local configuration files used to configure {{< param "PRODUCT_NAME" >}}. +* `sources/remote-config/remote.alloy` contains a copy of the last received [remote configuration][remotecfg]. [profile]: ../profile [components]: ../../get-started/components/ [alloy-repo]: https://github.com/grafana/alloy/issues -[backward-compatibility]: ../../introduction/backward-compatibility \ No newline at end of file +[backward-compatibility]: ../../introduction/backward-compatibility +[remotecfg]: ../../reference/config-blocks/remotecfg/ \ No newline at end of file diff --git a/internal/alloycli/cmd_run.go b/internal/alloycli/cmd_run.go index 306a030c55..43f6768255 100644 --- a/internal/alloycli/cmd_run.go +++ b/internal/alloycli/cmd_run.go @@ -356,6 +356,7 @@ func (fr *alloyRun) Run(cmd *cobra.Command, configPath string) error { if err != nil { return nil, fmt.Errorf("reading config path %q: %w", configPath, err) } + httpService.SetSources(alloySource.SourceFiles()) if err := f.LoadSource(alloySource, nil, configPath); err != nil { return alloySource, fmt.Errorf("error during the initial load: %w", err) } diff --git a/internal/runtime/alloy_services.go b/internal/runtime/alloy_services.go index 6c6171a7a6..95601c31a2 100644 --- a/internal/runtime/alloy_services.go +++ b/internal/runtime/alloy_services.go @@ -6,6 +6,7 @@ import ( "github.com/grafana/alloy/internal/runtime/internal/controller" "github.com/grafana/alloy/internal/runtime/internal/dag" "github.com/grafana/alloy/internal/service" + "github.com/grafana/alloy/syntax/ast" ) // GetServiceConsumers implements [service.Host]. It returns a slice of @@ -93,12 +94,12 @@ type ServiceController struct { } func (sc ServiceController) Run(ctx context.Context) { sc.f.Run(ctx) } -func (sc ServiceController) LoadSource(b []byte, args map[string]any, configPath string) error { +func (sc ServiceController) LoadSource(b []byte, args map[string]any, configPath string) (*ast.File, error) { source, err := ParseSource("", b) if err != nil { - return err + return nil, err } - return sc.f.LoadSource(source, args, configPath) + return source.SourceFiles()[""], sc.f.LoadSource(source, args, configPath) } func (sc ServiceController) Ready() bool { return sc.f.Ready() } diff --git a/internal/runtime/source.go b/internal/runtime/source.go index f02e71c6e4..a32cdd3f5a 100644 --- a/internal/runtime/source.go +++ b/internal/runtime/source.go @@ -15,6 +15,7 @@ import ( // A Source holds the contents of a parsed Alloy configuration source module. type Source struct { sourceMap map[string][]byte // Map that links parsed Alloy source's name with its content. + fileMap map[string]*ast.File hash [sha256.Size]byte // Hash of all files in sourceMap sorted by name. // Components holds the list of raw Alloy AST blocks describing components. @@ -42,6 +43,7 @@ func ParseSource(name string, bb []byte) (*Source, error) { return nil, err } source.sourceMap = map[string][]byte{name: bb} + source.fileMap = map[string]*ast.File{name: node} source.hash = sha256.Sum256(bb) return source, nil } @@ -107,8 +109,12 @@ type namedSource struct { // Source. sources must not be modified after calling ParseSources. func ParseSources(sources map[string][]byte) (*Source, error) { var ( - mergedSource = &Source{sourceMap: sources} // Combined source from all the input content. - hash = sha256.New() // Combined hash of all the sources. + // Combined source from all the input content. + mergedSource = &Source{ + sourceMap: sources, + fileMap: make(map[string]*ast.File, len(sources)), + } + hash = sha256.New() // Combined hash of all the sources. ) // Sorted slice so ParseSources always does the same thing. @@ -132,6 +138,8 @@ func ParseSources(sources map[string][]byte) (*Source, error) { return nil, err } + mergedSource.fileMap[namedSource.Name] = sourceFragment.fileMap[namedSource.Name] + mergedSource.components = append(mergedSource.components, sourceFragment.components...) mergedSource.configBlocks = append(mergedSource.configBlocks, sourceFragment.configBlocks...) mergedSource.declareBlocks = append(mergedSource.declareBlocks, sourceFragment.declareBlocks...) @@ -150,6 +158,15 @@ func (s *Source) RawConfigs() map[string][]byte { return s.sourceMap } +// SourceFiles returns the parsed source content used to create Source. +// Do not modify the returned map. +func (s *Source) SourceFiles() map[string]*ast.File { + if s == nil { + return nil + } + return s.fileMap +} + // SHA256 returns the sha256 checksum of the source. // Do not modify the returned byte array. func (s *Source) SHA256() [sha256.Size]byte { diff --git a/internal/service/http/http.go b/internal/service/http/http.go index 590802b9b4..61425033ec 100644 --- a/internal/service/http/http.go +++ b/internal/service/http/http.go @@ -6,6 +6,7 @@ import ( "context" "crypto/tls" "fmt" + "io" "net" "net/http" _ "net/http/pprof" // Register pprof handlers @@ -27,6 +28,8 @@ import ( "github.com/grafana/alloy/internal/service" "github.com/grafana/alloy/internal/service/remotecfg" "github.com/grafana/alloy/internal/static/server" + "github.com/grafana/alloy/syntax/ast" + "github.com/grafana/alloy/syntax/printer" "github.com/grafana/ckit/memconn" _ "github.com/grafana/pyroscope-go/godeltaprof/http/pprof" // Register godeltaprof handler "github.com/prometheus/client_golang/prometheus" @@ -78,6 +81,9 @@ type Service struct { // Used to enforce single-flight requests to supportHandler supportBundleMut sync.Mutex + // Track the raw config for use with the support bundle + sources map[string]*ast.File + // publicLis and tcpLis are used to lazily enable TLS, since TLS is // optionally configurable at runtime. // @@ -225,7 +231,7 @@ func (s *Service) Run(ctx context.Context, host service.Host) error { } // Wire in support bundle generator - r.HandleFunc("/-/support", s.supportHandler).Methods("GET") + r.HandleFunc("/-/support", s.generateSupportBundleHandler(host)).Methods("GET") // Wire custom service handlers for services which depend on the http // service. @@ -259,60 +265,80 @@ func (s *Service) Run(ctx context.Context, host service.Host) error { return nil } -func (s *Service) supportHandler(rw http.ResponseWriter, r *http.Request) { - s.supportBundleMut.Lock() - defer s.supportBundleMut.Unlock() +func (s *Service) generateSupportBundleHandler(host service.Host) func(rw http.ResponseWriter, r *http.Request) { + return func(rw http.ResponseWriter, r *http.Request) { + s.supportBundleMut.Lock() + defer s.supportBundleMut.Unlock() - // TODO(dehaansa) remove this check once the support bundle is generally available - if !s.opts.MinStability.Permits(featuregate.StabilityPublicPreview) { - rw.WriteHeader(http.StatusForbidden) - _, _ = rw.Write([]byte("support bundle generation is only available in public preview. Use" + - " --stability.level command-line flag to enable public-preview features")) - return - } + // TODO(dehaansa) remove this check once the support bundle is generally available + if !s.opts.MinStability.Permits(featuregate.StabilityPublicPreview) { + rw.WriteHeader(http.StatusForbidden) + _, _ = rw.Write([]byte("support bundle generation is only available in public preview. Use" + + " --stability.level command-line flag to enable public-preview features")) + return + } - if s.opts.BundleContext.DisableSupportBundle { - rw.WriteHeader(http.StatusForbidden) - _, _ = rw.Write([]byte("support bundle generation is disabled; it can be re-enabled by removing the --disable-support-bundle flag")) - return - } + if s.opts.BundleContext.DisableSupportBundle { + rw.WriteHeader(http.StatusForbidden) + _, _ = rw.Write([]byte("support bundle generation is disabled; it can be re-enabled by removing the --disable-support-bundle flag")) + return + } - duration := getServerWriteTimeout(r) - if r.URL.Query().Has("duration") { - d, err := strconv.Atoi(r.URL.Query().Get("duration")) + duration := getServerWriteTimeout(r) + if r.URL.Query().Has("duration") { + d, err := strconv.Atoi(r.URL.Query().Get("duration")) + if err != nil { + http.Error(rw, fmt.Sprintf("duration value (in seconds) should be a positive integer: %s", err), http.StatusBadRequest) + return + } + if d < 1 { + http.Error(rw, "duration value (in seconds) should be larger than 1", http.StatusBadRequest) + return + } + if float64(d) > duration.Seconds() { + http.Error(rw, "duration value exceeds the server's write timeout", http.StatusBadRequest) + return + } + duration = time.Duration(d) * time.Second + } + ctx, cancel := context.WithTimeout(context.Background(), duration) + defer cancel() + + var logsBuffer bytes.Buffer + syncBuff := log.NewSyncWriter(&logsBuffer) + s.globalLogger.SetTemporaryWriter(syncBuff) + defer func() { + s.globalLogger.RemoveTemporaryWriter() + }() + + // Get and redact the cached remote config. + cachedConfig, err := remoteCfgRedactedCachedConfig(host) if err != nil { - http.Error(rw, fmt.Sprintf("duration value (in seconds) should be a positive integer: %s", err), http.StatusBadRequest) - return + level.Debug(s.log).Log("msg", "failed to get cached remote config", "err", err) } - if d < 1 { - http.Error(rw, "duration value (in seconds) should be larger than 1", http.StatusBadRequest) + + // Ensure the sources are written using the printer as it will handle + // secret redaction. + sources := redactedSources(s.sources) + + bundle, err := ExportSupportBundle(ctx, s.opts.BundleContext.RuntimeFlags, s.opts.HTTPListenAddr, sources, cachedConfig, s.Data().(Data).DialFunc) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) return } - if float64(d) > duration.Seconds() { - http.Error(rw, "duration value exceeds the server's write timeout", http.StatusBadRequest) + if err := ServeSupportBundle(rw, bundle, &logsBuffer); err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) return } - duration = time.Duration(d) * time.Second } - ctx, cancel := context.WithTimeout(context.Background(), duration) - defer cancel() - - var logsBuffer bytes.Buffer - syncBuff := log.NewSyncWriter(&logsBuffer) - s.globalLogger.SetTemporaryWriter(syncBuff) - defer func() { - s.globalLogger.RemoveTemporaryWriter() - }() +} - bundle, err := ExportSupportBundle(ctx, s.opts.BundleContext.RuntimeFlags, s.opts.HTTPListenAddr, s.Data().(Data).DialFunc) - if err != nil { - http.Error(rw, err.Error(), http.StatusInternalServerError) - return - } - if err := ServeSupportBundle(rw, bundle, &logsBuffer); err != nil { - http.Error(rw, err.Error(), http.StatusInternalServerError) - return - } +// SetSources sets the sources on reload to be delivered +// with the support bundle. +func (s *Service) SetSources(sources map[string]*ast.File) { + s.supportBundleMut.Lock() + defer s.supportBundleMut.Unlock() + s.sources = sources } func getServerWriteTimeout(r *http.Request) time.Duration { @@ -582,6 +608,45 @@ func (lis *lazyListener) Addr() net.Addr { return lis.inner.Addr() } +func redactedSources(sources map[string]*ast.File) map[string][]byte { + if sources == nil { + return nil + } + printedSources := make(map[string][]byte, len(sources)) + + for k, v := range sources { + b, err := printFileRedacted(v) + if err != nil { + printedSources[k] = []byte(fmt.Errorf("failed to print source: %w", err).Error()) + continue + } + printedSources[k] = b + } + return printedSources +} + +func remoteCfgRedactedCachedConfig(host service.Host) ([]byte, error) { + svc, ok := host.GetService(remotecfg.ServiceName) + if !ok { + return nil, fmt.Errorf("failed to get the remotecfg service") + } + + return printFileRedacted(svc.(*remotecfg.Service).GetCachedAstFile()) +} + +func printFileRedacted(f *ast.File) ([]byte, error) { + c := printer.Config{ + RedactSecrets: true, + } + + var buf bytes.Buffer + w := io.Writer(&buf) + if err := c.Fprint(w, f); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + func remoteCfgHostProvider(host service.Host) func() (service.Host, error) { return func() (service.Host, error) { svc, ok := host.GetService(remotecfg.ServiceName) diff --git a/internal/service/http/supportbundle.go b/internal/service/http/supportbundle.go index f60fa18d08..a025cea6c2 100644 --- a/internal/service/http/supportbundle.go +++ b/internal/service/http/supportbundle.go @@ -36,6 +36,8 @@ type Bundle struct { peers []byte runtimeFlags []byte environmentVariables []byte + sources map[string][]byte + remoteCfg []byte heapBuf *bytes.Buffer goroutineBuf *bytes.Buffer blockBuf *bytes.Buffer @@ -52,7 +54,7 @@ type Metadata struct { } // ExportSupportBundle gathers the information required for the support bundle. -func ExportSupportBundle(ctx context.Context, runtimeFlags []string, srvAddress string, dialContext server.DialContextFunc) (*Bundle, error) { +func ExportSupportBundle(ctx context.Context, runtimeFlags []string, srvAddress string, sources map[string][]byte, remoteCfg []byte, dialContext server.DialContextFunc) (*Bundle, error) { var httpClient http.Client httpClient.Transport = &http.Transport{DialContext: dialContext} @@ -148,6 +150,8 @@ func ExportSupportBundle(ctx context.Context, runtimeFlags []string, srvAddress alloyMetricsEnd: alloyMetricsEnd, components: components, peers: peers, + sources: sources, + remoteCfg: remoteCfg, runtimeFlags: []byte(strings.Join(runtimeFlags, "\n")), environmentVariables: []byte(strings.Join(retrieveEnvironmentVariables(), "\n")), heapBuf: &heapBuf, @@ -208,19 +212,24 @@ func ServeSupportBundle(rw http.ResponseWriter, b *Bundle, logsBuf *bytes.Buffer rw.Header().Set("Content-Disposition", "attachment; filename=\"alloy-support-bundle.zip\"") zipStructure := map[string][]byte{ - "alloy-metadata.yaml": b.meta, - "alloy-components.json": b.components, - "alloy-peers.json": b.peers, - "alloy-metrics-sample-start.txt": b.alloyMetricsStart, - "alloy-metrics-sample-end.txt": b.alloyMetricsEnd, - "alloy-runtime-flags.txt": b.runtimeFlags, - "alloy-environment.txt": b.environmentVariables, - "alloy-logs.txt": logsBuf.Bytes(), - "pprof/cpu.pprof": b.cpuBuf.Bytes(), - "pprof/heap.pprof": b.heapBuf.Bytes(), - "pprof/goroutine.pprof": b.goroutineBuf.Bytes(), - "pprof/mutex.pprof": b.mutexBuf.Bytes(), - "pprof/block.pprof": b.blockBuf.Bytes(), + "alloy-metadata.yaml": b.meta, + "alloy-components.json": b.components, + "alloy-peers.json": b.peers, + "alloy-metrics-sample-start.txt": b.alloyMetricsStart, + "alloy-metrics-sample-end.txt": b.alloyMetricsEnd, + "alloy-runtime-flags.txt": b.runtimeFlags, + "alloy-environment.txt": b.environmentVariables, + "alloy-logs.txt": logsBuf.Bytes(), + "sources/remote-config/remote.alloy": b.remoteCfg, + "pprof/cpu.pprof": b.cpuBuf.Bytes(), + "pprof/heap.pprof": b.heapBuf.Bytes(), + "pprof/goroutine.pprof": b.goroutineBuf.Bytes(), + "pprof/mutex.pprof": b.mutexBuf.Bytes(), + "pprof/block.pprof": b.blockBuf.Bytes(), + } + + for p, s := range b.sources { + zipStructure[filepath.Join("sources", filepath.Base(p))] = s } for fn, b := range zipStructure { diff --git a/internal/service/remotecfg/remotecfg.go b/internal/service/remotecfg/remotecfg.go index 50ef83aee0..f3c21a4424 100644 --- a/internal/service/remotecfg/remotecfg.go +++ b/internal/service/remotecfg/remotecfg.go @@ -28,6 +28,7 @@ import ( "github.com/grafana/alloy/internal/service" "github.com/grafana/alloy/internal/util/jitter" "github.com/grafana/alloy/syntax" + "github.com/grafana/alloy/syntax/ast" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" commonconfig "github.com/prometheus/common/config" @@ -67,6 +68,10 @@ type Service struct { // This is the hash received from the API. It is used to determine if // the configuration has changed since the last fetch remoteHash string + + // This is the AST file parsed from the configuration. This is used + // for the support bundle + astFile *ast.File } type metrics struct { @@ -331,6 +336,13 @@ func (s *Service) Update(newConfig any) error { return nil } +// GetCachedAstFile returns the AST file that was parsed from the configuration. +func (s *Service) GetCachedAstFile() *ast.File { + s.mut.RLock() + defer s.mut.RUnlock() + return s.astFile +} + // fetch attempts to read configuration from the API and the local cache // and then parse/load their contents in order of preference. func (s *Service) fetch() { @@ -468,11 +480,12 @@ func (s *Service) parseAndLoad(b []byte) error { return nil } - err := ctrl.LoadSource(b, nil, s.opts.ConfigPath) + file, err := ctrl.LoadSource(b, nil, s.opts.ConfigPath) if err != nil { return err } + s.setAstFile(file) s.setCfgHash(getHash(b)) return nil } @@ -484,6 +497,12 @@ func (s *Service) getCfgHash() string { return s.currentConfigHash } +func (s *Service) setAstFile(f *ast.File) { + s.mut.Lock() + defer s.mut.Unlock() + s.astFile = f +} + func (s *Service) setCfgHash(h string) { s.mut.Lock() defer s.mut.Unlock() diff --git a/internal/service/remotecfg/remotecfg_test.go b/internal/service/remotecfg/remotecfg_test.go index 5fe015fbb8..fd634341f5 100644 --- a/internal/service/remotecfg/remotecfg_test.go +++ b/internal/service/remotecfg/remotecfg_test.go @@ -22,6 +22,7 @@ import ( "github.com/grafana/alloy/internal/service/livedebugging" "github.com/grafana/alloy/internal/util" "github.com/grafana/alloy/syntax" + "github.com/grafana/alloy/syntax/ast" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -290,11 +291,11 @@ type serviceController struct { } func (sc serviceController) Run(ctx context.Context) { sc.f.Run(ctx) } -func (sc serviceController) LoadSource(b []byte, args map[string]any, configPath string) error { +func (sc serviceController) LoadSource(b []byte, args map[string]any, configPath string) (*ast.File, error) { source, err := alloy_runtime.ParseSource("", b) if err != nil { - return err + return nil, err } - return sc.f.LoadSource(source, args, configPath) + return source.SourceFiles()[""], sc.f.LoadSource(source, args, configPath) } func (sc serviceController) Ready() bool { return sc.f.Ready() } diff --git a/internal/service/service.go b/internal/service/service.go index b6fc24675f..5a049ce586 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/alloy/internal/component" "github.com/grafana/alloy/internal/featuregate" + "github.com/grafana/alloy/syntax/ast" ) // Definition describes an individual service. Services have unique names @@ -75,7 +76,7 @@ type Host interface { // Controller is implemented by alloy.Alloy. type Controller interface { Run(ctx context.Context) - LoadSource(source []byte, args map[string]any, configPath string) error + LoadSource(source []byte, args map[string]any, configPath string) (*ast.File, error) Ready() bool } diff --git a/syntax/ast/ast.go b/syntax/ast/ast.go index 82a191c688..7c89b14b30 100644 --- a/syntax/ast/ast.go +++ b/syntax/ast/ast.go @@ -27,6 +27,8 @@ type Stmt interface { type Expr interface { Node astExpr() + IsSecret() bool // Used when printing, annotated in struct decoding + SetSecret(bool) // Used when printing, annotated in struct decoding } // File is a parsed file. @@ -81,6 +83,8 @@ type Ident struct { // IdentifierExpr refers to a named value. type IdentifierExpr struct { Ident *Ident + + Secret bool } // LiteralExpr is a constant value of a specific token kind. @@ -92,18 +96,24 @@ type LiteralExpr struct { // token.STRING, then Value would be wrapped in the original quotes (e.g., // `"foobar"`). Value string + + Secret bool } // ArrayExpr is an array of values. type ArrayExpr struct { Elements []Expr LBrackPos, RBrackPos token.Pos + + Secret bool } // ObjectExpr declares an object of key-value pairs. type ObjectExpr struct { Fields []*ObjectField LCurlyPos, RCurlyPos token.Pos + + Secret bool } // ObjectField defines an individual key-value pair within an object. @@ -118,12 +128,16 @@ type ObjectField struct { type AccessExpr struct { Value Expr Name *Ident + + Secret bool } // IndexExpr accesses an index in an array value. type IndexExpr struct { Value, Index Expr LBrackPos, RBrackPos token.Pos + + Secret bool } // CallExpr invokes a function value with a set of arguments. @@ -132,6 +146,8 @@ type CallExpr struct { Args []Expr LParenPos, RParenPos token.Pos + + Secret bool } // UnaryExpr performs a unary operation on a single value. @@ -139,6 +155,8 @@ type UnaryExpr struct { Kind token.Token KindPos token.Pos Value Expr + + Secret bool } // BinaryExpr performs a binary operation against two values. @@ -146,12 +164,16 @@ type BinaryExpr struct { Kind token.Token KindPos token.Pos Left, Right Expr + + Secret bool } // ParenExpr represents an expression wrapped in parentheses. type ParenExpr struct { Inner Expr LParenPos, RParenPos token.Pos + + Secret bool } // Type assertions @@ -220,6 +242,28 @@ func (n *UnaryExpr) astExpr() {} func (n *BinaryExpr) astExpr() {} func (n *ParenExpr) astExpr() {} +func (n *IdentifierExpr) IsSecret() bool { return n.Secret } +func (n *LiteralExpr) IsSecret() bool { return n.Secret } +func (n *ArrayExpr) IsSecret() bool { return n.Secret } +func (n *ObjectExpr) IsSecret() bool { return n.Secret } +func (n *AccessExpr) IsSecret() bool { return n.Secret } +func (n *IndexExpr) IsSecret() bool { return n.Secret } +func (n *CallExpr) IsSecret() bool { return n.Secret } +func (n *UnaryExpr) IsSecret() bool { return n.Secret } +func (n *BinaryExpr) IsSecret() bool { return n.Secret } +func (n *ParenExpr) IsSecret() bool { return n.Secret } + +func (n *IdentifierExpr) SetSecret(s bool) { n.Secret = s } +func (n *LiteralExpr) SetSecret(s bool) { n.Secret = s } +func (n *ArrayExpr) SetSecret(s bool) { n.Secret = s } +func (n *ObjectExpr) SetSecret(s bool) { n.Secret = s } +func (n *AccessExpr) SetSecret(s bool) { n.Secret = s } +func (n *IndexExpr) SetSecret(s bool) { n.Secret = s } +func (n *CallExpr) SetSecret(s bool) { n.Secret = s } +func (n *UnaryExpr) SetSecret(s bool) { n.Secret = s } +func (n *BinaryExpr) SetSecret(s bool) { n.Secret = s } +func (n *ParenExpr) SetSecret(s bool) { n.Secret = s } + // StartPos returns the position of the first character belonging to a Node. func StartPos(n Node) token.Pos { if n == nil || reflect.ValueOf(n).IsZero() { diff --git a/syntax/printer/printer.go b/syntax/printer/printer.go index cba9d3047d..23f1d2f0bf 100644 --- a/syntax/printer/printer.go +++ b/syntax/printer/printer.go @@ -13,7 +13,8 @@ import ( // Config configures behavior of the printer. type Config struct { - Indent int // Indentation to apply to all emitted code. Default 0. + Indent int // Indentation to apply to all emitted code. Default 0. + RedactSecrets bool // Should secrets be redacted. Default false. } // Fprint pretty-prints the specified node to w. The Node type must be an @@ -237,6 +238,14 @@ func (p *printer) Write(args ...interface{}) { panic(fmt.Sprintf("printer: unsupported argument %v (%T)\n", arg, arg)) } + if p.cfg.RedactSecrets { + if v, ok := arg.(ast.Expr); ok { + if v.IsSecret() { + data = "\"(secret)\"" + } + } + } + next := p.pos p.flush(next, p.lastTok) diff --git a/syntax/printer/printer_test.go b/syntax/printer/printer_test.go index 9ffad8a03d..aff5df154c 100644 --- a/syntax/printer/printer_test.go +++ b/syntax/printer/printer_test.go @@ -9,6 +9,7 @@ import ( "testing" "unicode" + "github.com/grafana/alloy/syntax/ast" "github.com/grafana/alloy/syntax/parser" "github.com/grafana/alloy/syntax/printer" "github.com/stretchr/testify/require" @@ -37,6 +38,34 @@ func TestPrinter(t *testing.T) { }) } +func TestSecretRedaction(t *testing.T) { + input := `password = "my_password" +string = "normal string"` + + f, err := parser.ParseFile("", []byte(input)) + require.NoError(t, err) + + pw := f.Body[0].(*ast.AttributeStmt) + require.Equal(t, "\"my_password\"", pw.Value.(*ast.LiteralExpr).Value) + + unredactedOutput := `password = "my_password" +string = "normal string"` + var buf bytes.Buffer + require.NoError(t, printer.Fprint(&buf, f)) + require.Equal(t, unredactedOutput, buf.String()) + + redactedOutput := `password = "(secret)" +string = "normal string"` + + pw.Value.SetSecret(true) + c := printer.Config{ + RedactSecrets: true, + } + buf.Reset() + require.NoError(t, c.Fprint(&buf, f)) + require.Equal(t, redactedOutput, buf.String()) +} + func testPrinter(t *testing.T, inputFile string, expectFile string, expectErrorFile string) { inputBB, err := os.ReadFile(inputFile) require.NoError(t, err) diff --git a/syntax/vm/struct_decoder.go b/syntax/vm/struct_decoder.go index 249e371268..d53a7de891 100644 --- a/syntax/vm/struct_decoder.go +++ b/syntax/vm/struct_decoder.go @@ -5,6 +5,7 @@ import ( "reflect" "strings" + "github.com/grafana/alloy/syntax/alloytypes" "github.com/grafana/alloy/syntax/ast" "github.com/grafana/alloy/syntax/diag" "github.com/grafana/alloy/syntax/internal/reflectutil" @@ -183,6 +184,12 @@ func (st *structDecoder) decodeAttr(attr *ast.AttributeStmt, rv reflect.Value, s return err } + // This annotation on the AST is used when printing the value. + switch field.Addr().Interface().(type) { + case *alloytypes.Secret, *alloytypes.OptionalSecret: + attr.Value.SetSecret(true) + } + return nil } diff --git a/syntax/vm/vm_block_test.go b/syntax/vm/vm_block_test.go index d911f3a3f1..3374ccbf34 100644 --- a/syntax/vm/vm_block_test.go +++ b/syntax/vm/vm_block_test.go @@ -1,13 +1,18 @@ package vm_test import ( + "bytes" "fmt" + "io" "math" + "os" "reflect" "testing" + "github.com/grafana/alloy/syntax/alloytypes" "github.com/grafana/alloy/syntax/ast" "github.com/grafana/alloy/syntax/parser" + "github.com/grafana/alloy/syntax/printer" "github.com/grafana/alloy/syntax/vm" "github.com/stretchr/testify/require" ) @@ -764,6 +769,48 @@ func TestVM_Block_UnmarshalToAny(t *testing.T) { require.Equal(t, expect, actual.Settings) } +func TestVM_AnnotatesSecrets(t *testing.T) { + type block struct { + OptionalPassword alloytypes.OptionalSecret `alloy:"optional_password,attr,optional"` + Password alloytypes.Secret `alloy:"password,attr"` + } + + os.Setenv("SECRET", "my_password") + defer os.Setenv("SECRET", "") + + input := ` + password = "my_password" + optional_password = sys.env("SECRET") + ` + + expect := block{ + Password: "my_password", + OptionalPassword: alloytypes.OptionalSecret{ + Value: "my_password", + }, + } + + res, err := parser.ParseFile(t.Name(), []byte(input)) + require.NoError(t, err) + + eval := vm.New(res) + + var actual block + require.NoError(t, eval.Evaluate(nil, &actual)) + require.Equal(t, expect, actual) + + // Ensure that the secrets are redacted. + c := printer.Config{ + RedactSecrets: true, + } + var buf bytes.Buffer + w := io.Writer(&buf) + require.NoError(t, c.Fprint(w, res)) + + require.NotContains(t, buf.String(), "my_password") + require.Contains(t, buf.String(), "(secret)") +} + type Setting struct { FieldA string `alloy:"field_a,attr"` FieldB string `alloy:"field_b,attr"`