From a2a2d597b64f141d8f0724929707189efbe028b9 Mon Sep 17 00:00:00 2001 From: sg Date: Mon, 3 Jun 2024 11:58:05 +0100 Subject: [PATCH 1/3] add structured dracon logging --- pkg/log/dracon-logger.go | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 pkg/log/dracon-logger.go diff --git a/pkg/log/dracon-logger.go b/pkg/log/dracon-logger.go new file mode 100644 index 000000000..63c55f0e9 --- /dev/null +++ b/pkg/log/dracon-logger.go @@ -0,0 +1,11 @@ +package draconLogger + +import ( + "log/slog" + "os" +) + +func SetDefault(logLevel slog.Level) { + logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel})) + slog.SetDefault(logger) +} From 0f15346e9b9f91355a84180422d6c3b42c2ff584 Mon Sep 17 00:00:00 2001 From: sg Date: Mon, 3 Jun 2024 12:06:32 +0100 Subject: [PATCH 2/3] add logging to base producers and consumers --- cmd/draconctl/main.go | 5 +++-- components/consumers/consumer.go | 19 ++++++++++++++++--- components/producers/producer.go | 10 ++++++++++ pkg/log/dracon-logger.go | 12 ++++++++++-- 4 files changed, 39 insertions(+), 7 deletions(-) diff --git a/cmd/draconctl/main.go b/cmd/draconctl/main.go index cc238b92a..8cb5cb71e 100644 --- a/cmd/draconctl/main.go +++ b/cmd/draconctl/main.go @@ -1,6 +1,7 @@ package main import ( + "log/slog" "os" "github.com/spf13/cobra" @@ -8,6 +9,7 @@ import ( "github.com/ocurity/dracon/cmd/draconctl/components" "github.com/ocurity/dracon/cmd/draconctl/migrations" "github.com/ocurity/dracon/cmd/draconctl/pipelines" + draconLogger "github.com/ocurity/dracon/pkg/log" ) var rootCmd = &cobra.Command{ @@ -20,11 +22,10 @@ func main() { ID: "top-level", Title: "Top-level Commands:", }) - pipelines.RegisterPipelinesSubcommands(rootCmd) migrations.RegisterMigrationsSubcommands(rootCmd) components.RegisterComponentsSubcommands(rootCmd) - + draconLogger.SetDefault(slog.LevelInfo, "", false) err := rootCmd.Execute() if err != nil { os.Exit(1) diff --git a/components/consumers/consumer.go b/components/consumers/consumer.go index 7cfd8dd2d..6440d791c 100644 --- a/components/consumers/consumer.go +++ b/components/consumers/consumer.go @@ -5,9 +5,12 @@ package consumers import ( "flag" "fmt" + "log/slog" + "os" - v1 "github.com/ocurity/dracon/api/proto/v1" + apiv1 "github.com/ocurity/dracon/api/proto/v1" + draconLogger "github.com/ocurity/dracon/pkg/log" "github.com/ocurity/dracon/pkg/putil" ) @@ -24,16 +27,26 @@ var ( inResults string // Raw represents if the non-enriched results should be used. Raw bool + // Debug flag initializes the logger with a debug level + Debug bool ) func init() { flag.StringVar(&inResults, "in", "", "the directory where dracon producer/enricher outputs are") flag.BoolVar(&Raw, "raw", false, "if the non-enriched results should be used") + flag.BoolVar(&Debug, "debug", false, "turn on debug logging") + } // ParseFlags will parse the input flags for the consumer and perform simple validation. func ParseFlags() error { flag.Parse() + if Debug { + draconLogger.SetDefault(slog.LevelDebug, os.Getenv(EnvDraconScanID), true) + } else { + draconLogger.SetDefault(slog.LevelInfo, os.Getenv(EnvDraconScanID), true) + } + if len(inResults) < 1 { return fmt.Errorf("in is undefined") } @@ -41,11 +54,11 @@ func ParseFlags() error { } // LoadToolResponse loads raw results from producers. -func LoadToolResponse() ([]*v1.LaunchToolResponse, error) { +func LoadToolResponse() ([]*apiv1.LaunchToolResponse, error) { return putil.LoadToolResponse(inResults) } // LoadEnrichedToolResponse loads enriched results from the enricher. -func LoadEnrichedToolResponse() ([]*v1.EnrichedLaunchToolResponse, error) { +func LoadEnrichedToolResponse() ([]*apiv1.EnrichedLaunchToolResponse, error) { return putil.LoadEnrichedToolResponse(inResults) } diff --git a/components/producers/producer.go b/components/producers/producer.go index e179cf2f4..261f1313a 100644 --- a/components/producers/producer.go +++ b/components/producers/producer.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "log" + "log/slog" "os" "path/filepath" "strings" @@ -18,6 +19,7 @@ import ( v1 "github.com/ocurity/dracon/api/proto/v1" + draconLogger "github.com/ocurity/dracon/pkg/log" "github.com/ocurity/dracon/pkg/putil" ) @@ -28,6 +30,8 @@ var ( OutFile string // Append flag will append to the outfile instead of overwriting, useful when there's multiple inresults. Append bool + // Debug flag initializes the logger with a debug level + Debug bool ) const ( @@ -45,9 +49,15 @@ const ( func ParseFlags() error { flag.StringVar(&InResults, "in", "", "") flag.StringVar(&OutFile, "out", "", "") + flag.BoolVar(&Debug, "debug", false, "turn on debug logging") flag.BoolVar(&Append, "append", false, "Append to output file instead of overwriting it") flag.Parse() + if Debug { + draconLogger.SetDefault(slog.LevelDebug, os.Getenv(EnvDraconScanID), true) + } else { + draconLogger.SetDefault(slog.LevelInfo, os.Getenv(EnvDraconScanID), true) + } if InResults == "" { return fmt.Errorf("in is undefined") } diff --git a/pkg/log/dracon-logger.go b/pkg/log/dracon-logger.go index 63c55f0e9..3477c912c 100644 --- a/pkg/log/dracon-logger.go +++ b/pkg/log/dracon-logger.go @@ -5,7 +5,15 @@ import ( "os" ) -func SetDefault(logLevel slog.Level) { - logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel})) +func SetDefault(logLevel slog.Level, scanID string, jsonLogging bool) { + var logger *slog.Logger + if jsonLogging { + logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel})) + } else { + logger = slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel})) + } + if scanID != "" { + logger = logger.With("scanID", scanID) + } slog.SetDefault(logger) } From e50b5198a56d2c0f3dd403d496300fd28b4dd773 Mon Sep 17 00:00:00 2001 From: sg Date: Mon, 3 Jun 2024 13:38:56 +0100 Subject: [PATCH 3/3] move constants to base component --- cmd/draconctl/main.go | 3 +- components/component.go | 10 +++++++ components/consumers/consumer.go | 34 ++++++++------------- components/consumers/consumer_test.go | 12 ++++---- components/producers/producer.go | 43 +++++++++++---------------- components/producers/producer_test.go | 9 +++--- pkg/log/dracon-logger.go | 19 ------------ 7 files changed, 52 insertions(+), 78 deletions(-) create mode 100644 components/component.go delete mode 100644 pkg/log/dracon-logger.go diff --git a/cmd/draconctl/main.go b/cmd/draconctl/main.go index 8cb5cb71e..448435df4 100644 --- a/cmd/draconctl/main.go +++ b/cmd/draconctl/main.go @@ -9,7 +9,6 @@ import ( "github.com/ocurity/dracon/cmd/draconctl/components" "github.com/ocurity/dracon/cmd/draconctl/migrations" "github.com/ocurity/dracon/cmd/draconctl/pipelines" - draconLogger "github.com/ocurity/dracon/pkg/log" ) var rootCmd = &cobra.Command{ @@ -25,7 +24,7 @@ func main() { pipelines.RegisterPipelinesSubcommands(rootCmd) migrations.RegisterMigrationsSubcommands(rootCmd) components.RegisterComponentsSubcommands(rootCmd) - draconLogger.SetDefault(slog.LevelInfo, "", false) + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{}))) err := rootCmd.Execute() if err != nil { os.Exit(1) diff --git a/components/component.go b/components/component.go new file mode 100644 index 000000000..8486be2a2 --- /dev/null +++ b/components/component.go @@ -0,0 +1,10 @@ +package components + +const ( + // EnvDraconStartTime Start Time of Dracon Scan in RFC3339. + EnvDraconStartTime = "DRACON_SCAN_TIME" + // EnvDraconScanID the ID of the dracon scan. + EnvDraconScanID = "DRACON_SCAN_ID" + // EnvDraconScanTags the tags of the dracon scan. + EnvDraconScanTags = "DRACON_SCAN_TAGS" +) diff --git a/components/consumers/consumer.go b/components/consumers/consumer.go index 6440d791c..67586d829 100644 --- a/components/consumers/consumer.go +++ b/components/consumers/consumer.go @@ -8,45 +8,35 @@ import ( "log/slog" "os" - apiv1 "github.com/ocurity/dracon/api/proto/v1" - - draconLogger "github.com/ocurity/dracon/pkg/log" + draconapiv1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components" "github.com/ocurity/dracon/pkg/putil" ) -const ( - // EnvDraconStartTime Start Time of Dracon Scan in RFC3339. - EnvDraconStartTime = "DRACON_SCAN_TIME" - // EnvDraconScanID the ID of the dracon scan. - EnvDraconScanID = "DRACON_SCAN_ID" - // EnvDraconScanTags the tags of the dracon scan. - EnvDraconScanTags = "DRACON_SCAN_TAGS" -) - var ( inResults string // Raw represents if the non-enriched results should be used. Raw bool - // Debug flag initializes the logger with a debug level - Debug bool + // debug flag initializes the logger with a debug level + debug bool ) func init() { flag.StringVar(&inResults, "in", "", "the directory where dracon producer/enricher outputs are") flag.BoolVar(&Raw, "raw", false, "if the non-enriched results should be used") - flag.BoolVar(&Debug, "debug", false, "turn on debug logging") - + flag.BoolVar(&debug, "debug", false, "turn on debug logging") } // ParseFlags will parse the input flags for the consumer and perform simple validation. func ParseFlags() error { flag.Parse() - if Debug { - draconLogger.SetDefault(slog.LevelDebug, os.Getenv(EnvDraconScanID), true) - } else { - draconLogger.SetDefault(slog.LevelInfo, os.Getenv(EnvDraconScanID), true) + + logLevel := slog.LevelInfo + if debug { + logLevel = slog.LevelDebug } + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel})).With("scanID", os.Getenv(components.EnvDraconScanID))) if len(inResults) < 1 { return fmt.Errorf("in is undefined") } @@ -54,11 +44,11 @@ func ParseFlags() error { } // LoadToolResponse loads raw results from producers. -func LoadToolResponse() ([]*apiv1.LaunchToolResponse, error) { +func LoadToolResponse() ([]*draconapiv1.LaunchToolResponse, error) { return putil.LoadToolResponse(inResults) } // LoadEnrichedToolResponse loads enriched results from the enricher. -func LoadEnrichedToolResponse() ([]*apiv1.EnrichedLaunchToolResponse, error) { +func LoadEnrichedToolResponse() ([]*draconapiv1.EnrichedLaunchToolResponse, error) { return putil.LoadEnrichedToolResponse(inResults) } diff --git a/components/consumers/consumer_test.go b/components/consumers/consumer_test.go index c10fbd3f3..834f1f7ec 100644 --- a/components/consumers/consumer_test.go +++ b/components/consumers/consumer_test.go @@ -6,11 +6,11 @@ import ( "testing" "time" - v1 "github.com/ocurity/dracon/api/proto/v1" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + draconapiv1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components" "github.com/ocurity/dracon/pkg/putil" ) @@ -23,7 +23,7 @@ func TestLoadToolResponse(t *testing.T) { defer require.NoError(t, os.Remove(tmpFile.Name())) - issues := []*v1.Issue{ + issues := []*draconapiv1.Issue{ { Target: "/dracon/source/foobar", Title: "/dracon/source/barfoo", @@ -39,9 +39,9 @@ func TestLoadToolResponse(t *testing.T) { scanTags, err := json.Marshal(tags) assert.NoError(t, err) - require.NoError(t, os.Setenv(EnvDraconStartTime, timestamp)) - require.NoError(t, os.Setenv(EnvDraconScanID, scanID)) - require.NoError(t, os.Setenv(EnvDraconScanTags, string(scanTags))) + require.NoError(t, os.Setenv(components.EnvDraconStartTime, timestamp)) + require.NoError(t, os.Setenv(components.EnvDraconScanID, scanID)) + require.NoError(t, os.Setenv(components.EnvDraconScanTags, string(scanTags))) resultTempDir := tmpFile.Name() resultFile := "test-tool" diff --git a/components/producers/producer.go b/components/producers/producer.go index 261f1313a..b2442d2f8 100644 --- a/components/producers/producer.go +++ b/components/producers/producer.go @@ -10,16 +10,15 @@ import ( "flag" "fmt" "io" - "log" "log/slog" "os" "path/filepath" "strings" "time" - v1 "github.com/ocurity/dracon/api/proto/v1" + draconapiv1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components" - draconLogger "github.com/ocurity/dracon/pkg/log" "github.com/ocurity/dracon/pkg/putil" ) @@ -30,34 +29,28 @@ var ( OutFile string // Append flag will append to the outfile instead of overwriting, useful when there's multiple inresults. Append bool - // Debug flag initializes the logger with a debug level - Debug bool + // debug flag initializes the logger with a debug level + debug bool ) const ( sourceDir = "/workspace/output" - - // EnvDraconStartTime Start Time of Dracon Scan in RFC3339. - EnvDraconStartTime = "DRACON_SCAN_TIME" - // EnvDraconScanID the ID of the dracon scan. - EnvDraconScanID = "DRACON_SCAN_ID" - // EnvDraconScanTags the tags of the dracon scan. - EnvDraconScanTags = "DRACON_SCAN_TAGS" ) // ParseFlags will parse the input flags for the producer and perform simple validation. func ParseFlags() error { flag.StringVar(&InResults, "in", "", "") flag.StringVar(&OutFile, "out", "", "") - flag.BoolVar(&Debug, "debug", false, "turn on debug logging") + flag.BoolVar(&debug, "debug", false, "turn on debug logging") flag.BoolVar(&Append, "append", false, "Append to output file instead of overwriting it") flag.Parse() - if Debug { - draconLogger.SetDefault(slog.LevelDebug, os.Getenv(EnvDraconScanID), true) - } else { - draconLogger.SetDefault(slog.LevelInfo, os.Getenv(EnvDraconScanID), true) + logLevel := slog.LevelInfo + if debug { + logLevel = slog.LevelDebug } + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel})).With("scanID", os.Getenv(components.EnvDraconScanID))) + if InResults == "" { return fmt.Errorf("in is undefined") } @@ -123,28 +116,28 @@ func ParseMultiJSONMessages(in []byte) ([]interface{}, error) { // WriteDraconOut provides a generic method to write the resulting protobuf to the output file. func WriteDraconOut( toolName string, - issues []*v1.Issue, + issues []*draconapiv1.Issue, ) error { source := getSource() - cleanIssues := []*v1.Issue{} + cleanIssues := []*draconapiv1.Issue{} for _, iss := range issues { iss.Description = strings.ReplaceAll(iss.Description, sourceDir, ".") iss.Title = strings.ReplaceAll(iss.Title, sourceDir, ".") iss.Target = strings.ReplaceAll(iss.Target, sourceDir, ".") iss.Source = source cleanIssues = append(cleanIssues, iss) - log.Printf("found issue: %+v\n", iss) + slog.Info(fmt.Sprintf("found issue: %+v\n", iss)) } - scanStartTime := strings.TrimSpace(os.Getenv(EnvDraconStartTime)) + scanStartTime := strings.TrimSpace(os.Getenv(components.EnvDraconStartTime)) if scanStartTime == "" { scanStartTime = time.Now().UTC().Format(time.RFC3339) } - scanUUUID := strings.TrimSpace(os.Getenv(EnvDraconScanID)) - scanTagsStr := strings.TrimSpace(os.Getenv(EnvDraconScanTags)) + scanUUUID := strings.TrimSpace(os.Getenv(components.EnvDraconScanID)) + scanTagsStr := strings.TrimSpace(os.Getenv(components.EnvDraconScanTags)) scanTags := map[string]string{} err := json.Unmarshal([]byte(scanTagsStr), &scanTags) if err != nil { - log.Printf("scan with uuid %s does not have any tags, err:%s", scanUUUID, err) + slog.Error(fmt.Sprintf("scan does not have any tags, err:%s", err)) } stat, err := os.Stat(OutFile) @@ -163,7 +156,7 @@ func getSource() string { dat, err := os.ReadFile(sourceMetaPath) if err != nil { - log.Println(err) + slog.Error(err.Error()) } return strings.TrimSpace(string(dat)) } diff --git a/components/producers/producer_test.go b/components/producers/producer_test.go index 7d384c401..5316b53ce 100644 --- a/components/producers/producer_test.go +++ b/components/producers/producer_test.go @@ -7,6 +7,7 @@ import ( "time" v1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components" "github.com/mitchellh/mapstructure" "github.com/stretchr/testify/assert" @@ -25,8 +26,8 @@ func TestWriteDraconOut(t *testing.T) { baseTime := time.Now().UTC() timestamp := baseTime.Format(time.RFC3339) - require.NoError(t, os.Setenv(EnvDraconStartTime, timestamp)) - require.NoError(t, os.Setenv(EnvDraconScanID, "ab3d3290-cd9f-482c-97dc-ec48bdfcc4de")) + require.NoError(t, os.Setenv(components.EnvDraconStartTime, timestamp)) + require.NoError(t, os.Setenv(components.EnvDraconScanID, "ab3d3290-cd9f-482c-97dc-ec48bdfcc4de")) OutFile = tmpFile.Name() Append = false @@ -66,8 +67,8 @@ func TestWriteDraconOutAppend(t *testing.T) { baseTime := time.Now().UTC() timestamp := baseTime.Format(time.RFC3339) - require.NoError(t, os.Setenv(EnvDraconStartTime, timestamp)) - require.NoError(t, os.Setenv(EnvDraconScanID, "ab3d3290-cd9f-482c-97dc-ec48bdfcc4de")) + require.NoError(t, os.Setenv(components.EnvDraconStartTime, timestamp)) + require.NoError(t, os.Setenv(components.EnvDraconScanID, "ab3d3290-cd9f-482c-97dc-ec48bdfcc4de")) OutFile = tmpFile.Name() Append = true diff --git a/pkg/log/dracon-logger.go b/pkg/log/dracon-logger.go deleted file mode 100644 index 3477c912c..000000000 --- a/pkg/log/dracon-logger.go +++ /dev/null @@ -1,19 +0,0 @@ -package draconLogger - -import ( - "log/slog" - "os" -) - -func SetDefault(logLevel slog.Level, scanID string, jsonLogging bool) { - var logger *slog.Logger - if jsonLogging { - logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel})) - } else { - logger = slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel})) - } - if scanID != "" { - logger = logger.With("scanID", scanID) - } - slog.SetDefault(logger) -}