From 31a5de6481b6bb09707610e71298df0e50278275 Mon Sep 17 00:00:00 2001 From: sg Date: Mon, 3 Jun 2024 13:38:13 +0100 Subject: [PATCH] create base enricher and make it follow conventions --- components/consumers/consumer.go | 3 +- components/enrichers/aggregator/main.go | 47 ++++---- components/enrichers/codeowners/main.go | 73 ++++-------- components/enrichers/deduplication/main.go | 123 +++++++-------------- components/enrichers/depsdev/main.go | 64 +++-------- components/enrichers/depsdev/main_test.go | 22 ++-- components/enrichers/enricher.go | 94 ++++++++++++++++ components/enrichers/policy/main.go | 66 +++-------- components/enrichers/policy/main_test.go | 6 +- 9 files changed, 223 insertions(+), 275 deletions(-) create mode 100644 components/enrichers/enricher.go diff --git a/components/consumers/consumer.go b/components/consumers/consumer.go index 0eefacfbf..3bc31811b 100644 --- a/components/consumers/consumer.go +++ b/components/consumers/consumer.go @@ -36,7 +36,8 @@ func ParseFlags() error { if debug { logLevel = slog.LevelDebug } - draconLogger.SetDefault(logLevel, os.Getenv(components.EnvDraconScanID), true) if len(inResults) < 1 { + draconLogger.SetDefault(logLevel, os.Getenv(components.EnvDraconScanID), true) + if len(inResults) < 1 { return fmt.Errorf("in is undefined") } return nil diff --git a/components/enrichers/aggregator/main.go b/components/enrichers/aggregator/main.go index c7a43089a..673d78376 100644 --- a/components/enrichers/aggregator/main.go +++ b/components/enrichers/aggregator/main.go @@ -6,12 +6,13 @@ import ( "flag" "fmt" "log" - "os" "path/filepath" "golang.org/x/crypto/nacl/sign" + apiv1 "github.com/ocurity/dracon/api/proto/v1" v1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components/enrichers" "github.com/ocurity/dracon/pkg/putil" ) @@ -24,17 +25,10 @@ var ( keyBytes []byte ) -func lookupEnvOrString(key string, defaultVal string) string { - if val, ok := os.LookupEnv(key); ok { - return val - } - return defaultVal -} - // Aggregation Rules: // all k,v annotations get merged // if there's a key conflict keep the value of the one you saw first. -func aggregateIssue(i *v1.EnrichedIssue, issues map[string]*v1.EnrichedIssue) map[string]*v1.EnrichedIssue { +func aggregateIssue(i *apiv1.EnrichedIssue, issues map[string]*apiv1.EnrichedIssue) map[string]*apiv1.EnrichedIssue { if _, ok := issues[i.RawIssue.Uuid]; ok { // do we already know about this Uuid? for k, v := range i.Annotations { // if yes, merge known Uuid annotations with new annotations if val, found := issues[i.RawIssue.Uuid].Annotations[k]; found { @@ -74,13 +68,13 @@ func aggregateIssue(i *v1.EnrichedIssue, issues map[string]*v1.EnrichedIssue) ma } // signMessage uses Nacl.Sign to append a Base64 encoded signature of the whole message to the annotation named: "JSON-Message-Signature". -func signMessage(i *v1.EnrichedIssue) (*v1.EnrichedIssue, error) { +func signMessage(i *apiv1.EnrichedIssue) (*apiv1.EnrichedIssue, error) { // if you have been instructed to sign results, then add the signature annotation log.Println("signing message " + i.RawIssue.Title) msg, err := json.Marshal(i) if err != nil { log.Printf("Error: could not serialise the message for signing") - return &v1.EnrichedIssue{}, nil + return &apiv1.EnrichedIssue{}, nil } if i.Annotations == nil { @@ -90,33 +84,33 @@ func signMessage(i *v1.EnrichedIssue) (*v1.EnrichedIssue, error) { return i, nil } -func aggregateToolResponse(response *v1.EnrichedLaunchToolResponse, issues map[string]*v1.EnrichedIssue) map[string]*v1.EnrichedIssue { +func aggregateToolResponse(response *apiv1.EnrichedLaunchToolResponse, issues map[string]*apiv1.EnrichedIssue) map[string]*apiv1.EnrichedIssue { for _, i := range response.GetIssues() { issues = aggregateIssue(i, issues) } return issues } -func run() { +func run() error { results, err := putil.LoadEnrichedNonAggregatedToolResponse(readPath) if err != nil { - log.Fatalf("could not load tool response from path %s , error:%v", readPath, err) + return fmt.Errorf("could not load tool response from path %s , error:%v", readPath, err) } if len(signKey) > 0 { keyBytes, err = base64.StdEncoding.DecodeString(signKey) if err != nil { - log.Fatalf("could not decode private key for signing") + return fmt.Errorf("could not decode private key for signing") } } log.Printf("loaded %d, enriched but not aggregated tool responses\n", len(results)) - issues := make(map[string]map[string]*v1.EnrichedIssue) - originalResponses := make(map[string]*v1.LaunchToolResponse) + issues := make(map[string]map[string]*apiv1.EnrichedIssue) + originalResponses := make(map[string]*apiv1.LaunchToolResponse) for _, r := range results { toolName := r.GetOriginalResults().GetToolName() originalResponses[toolName] = r.GetOriginalResults() if _, ok := issues[toolName]; !ok { - issues[toolName] = make(map[string]*v1.EnrichedIssue) + issues[toolName] = make(map[string]*apiv1.EnrichedIssue) } issues[toolName] = aggregateToolResponse(r, issues[toolName]) } @@ -136,17 +130,18 @@ func run() { if err := putil.WriteEnrichedResults(originalResponses[toolName], result, filepath.Join(writePath, fmt.Sprintf("%s.enriched.aggregated.pb", toolName)), ); err != nil { - log.Fatal(err) + return err } } + return nil } func main() { - flag.StringVar(&readPath, "read_path", lookupEnvOrString("READ_PATH", ""), "where to find producer results") - flag.StringVar(&writePath, "write_path", lookupEnvOrString("WRITE_PATH", ""), "where to put tagged results") - - flag.StringVar(&signKey, "signature_key", lookupEnvOrString("B64_SIGNATURE_KEY", ""), "where to put tagged results") - - flag.Parse() - run() + flag.StringVar(&signKey, "signature_key", enrichers.LookupEnvOrString("B64_SIGNATURE_KEY", ""), "where to put tagged results") + if err := enrichers.ParseFlags(); err != nil { + log.Fatal(err) + } + if err := run(); err != nil { + log.Fatal(err) + } } diff --git a/components/enrichers/codeowners/main.go b/components/enrichers/codeowners/main.go index 16554dfae..335ad1e7b 100644 --- a/components/enrichers/codeowners/main.go +++ b/components/enrichers/codeowners/main.go @@ -9,35 +9,25 @@ import ( "flag" "fmt" "log" - "os" + "log/slog" "path/filepath" "strings" - "time" owners "github.com/hairyhenderson/go-codeowners" - v1 "github.com/ocurity/dracon/api/proto/v1" - "github.com/ocurity/dracon/pkg/putil" + apiv1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components/enrichers" ) const defaultAnnotation = "Owner" var ( - readPath string - writePath string repoBasePath string annotation string ) -func lookupEnvOrString(key string, defaultVal string) string { - if val, ok := os.LookupEnv(key); ok { - return val - } - return defaultVal -} - -func enrichIssue(i *v1.Issue) (*v1.EnrichedIssue, error) { - enrichedIssue := v1.EnrichedIssue{} +func enrichIssue(i *apiv1.Issue) (*apiv1.EnrichedIssue, error) { + enrichedIssue := apiv1.EnrichedIssue{} annotations := map[string]string{} targets := []string{} if i.GetCycloneDXSBOM() != "" { @@ -64,8 +54,7 @@ func enrichIssue(i *v1.Issue) (*v1.EnrichedIssue, error) { annotations[fmt.Sprintf("Owner-%d", len(annotations))] = owner } } - - enrichedIssue = v1.EnrichedIssue{ + enrichedIssue = apiv1.EnrichedIssue{ RawIssue: i, Annotations: annotations, } @@ -73,55 +62,37 @@ func enrichIssue(i *v1.Issue) (*v1.EnrichedIssue, error) { return &enrichedIssue, nil } -func run() { - res, err := putil.LoadTaggedToolResponse(readPath) +func run() error { + res, err := enrichers.LoadData() if err != nil { - log.Fatalf("could not load tool response from path %s , error:%v", readPath, err) + return err } if annotation == "" { annotation = defaultAnnotation } for _, r := range res { - enrichedIssues := []*v1.EnrichedIssue{} + enrichedIssues := []*apiv1.EnrichedIssue{} for _, i := range r.GetIssues() { eI, err := enrichIssue(i) if err != nil { - log.Println(err) + slog.Error(err.Error()) continue } enrichedIssues = append(enrichedIssues, eI) } - if len(enrichedIssues) > 0 { - if err := putil.WriteEnrichedResults(r, enrichedIssues, - filepath.Join(writePath, fmt.Sprintf("%s.depsdev.enriched.pb", r.GetToolName())), - ); err != nil { - log.Fatal(err) - } - } else { - log.Println("no enriched issues were created for", r.GetToolName()) - } - if len(r.GetIssues()) > 0 { - scanStartTime := r.GetScanInfo().GetScanStartTime().AsTime() - if err := putil.WriteResults( - r.GetToolName(), - r.GetIssues(), - filepath.Join(writePath, fmt.Sprintf("%s.raw.pb", r.GetToolName())), - r.GetScanInfo().GetScanUuid(), - scanStartTime.Format(time.RFC3339), - r.GetScanInfo().GetScanTags(), - ); err != nil { - log.Fatalf("could not write results: %s", err) - } - } - + return enrichers.WriteData(enrichedIssues, r, "codeowners") } + return nil } func main() { - flag.StringVar(&readPath, "read_path", lookupEnvOrString("READ_PATH", ""), "where to find producer results") - flag.StringVar(&writePath, "write_path", lookupEnvOrString("WRITE_PATH", ""), "where to put enriched results") - flag.StringVar(&annotation, "annotation", lookupEnvOrString("ANNOTATION", defaultAnnotation), "what is the annotation this enricher will add to the issues, by default `Enriched Licenses`") - flag.StringVar(&repoBasePath, "repoBasePath", lookupEnvOrString("REPO_BASE_PATH", ""), `the base path of the repository, this is most likely an internally set variable`) - flag.Parse() - run() + flag.StringVar(&annotation, "annotation", enrichers.LookupEnvOrString("ANNOTATION", defaultAnnotation), "what is the annotation this enricher will add to the issues, by default `Enriched Licenses`") + flag.StringVar(&repoBasePath, "repoBasePath", enrichers.LookupEnvOrString("REPO_BASE_PATH", ""), `the base path of the repository, this is most likely an internally set variable`) + + if err := enrichers.ParseFlags(); err != nil { + log.Fatal(err) + } + if err := run(); err != nil { + log.Fatal(err) + } } diff --git a/components/enrichers/deduplication/main.go b/components/enrichers/deduplication/main.go index 00a2fa69d..c2f76727f 100644 --- a/components/enrichers/deduplication/main.go +++ b/components/enrichers/deduplication/main.go @@ -1,108 +1,59 @@ package main import ( + "flag" "fmt" "log" - "path/filepath" - "time" + "log/slog" v1 "github.com/ocurity/dracon/api/proto/v1" - - "github.com/spf13/cobra" - "github.com/spf13/viper" + "github.com/ocurity/dracon/components/enrichers" "github.com/ocurity/dracon/pkg/db" "github.com/ocurity/dracon/pkg/enrichment" - "github.com/ocurity/dracon/pkg/putil" ) var ( - connStr string - readPath string - writePath string + connStr string ) -var rootCmd = &cobra.Command{ - Use: "enricher", - Short: "enricher", - Long: "tool to enrich issues against a database", - RunE: func(cmd *cobra.Command, args []string) error { - connStr = viper.GetString("db_connection") - dbUrl, err := db.ParseConnectionStr(connStr) - if err != nil { - return err - } - - conn, err := dbUrl.Connect() - if err != nil { - return err - } - - readPath = viper.GetString("read_path") - res, err := putil.LoadTaggedToolResponse(readPath) - if err != nil { - log.Fatal(err) - } - - log.Printf("Loaded %d tagged tool responses\n", len(res)) - writePath = viper.GetString("write_path") - for _, r := range res { - enrichedIssues := []*v1.EnrichedIssue{} - log.Printf("enriching %d issues", len(r.GetIssues())) - for _, i := range r.GetIssues() { - eI, err := enrichment.EnrichIssue(conn, i) - if err != nil { - log.Printf("error enriching issue %s, err: %#v\n", i.Uuid, err) - continue - } - enrichedIssues = append(enrichedIssues, eI) - log.Printf("enriched issue '%s'", eI.GetRawIssue().GetUuid()) - } - if len(enrichedIssues) > 0 { - if err := putil.WriteEnrichedResults(r, enrichedIssues, - filepath.Join(writePath, fmt.Sprintf("%s.enriched.pb", r.GetToolName())), - ); err != nil { - return err - } - } - if len(r.GetIssues()) > 0 { - scanStartTime := r.GetScanInfo().GetScanStartTime().AsTime() - if err := putil.WriteResults( - r.GetToolName(), - r.GetIssues(), - filepath.Join(writePath, fmt.Sprintf("%s.raw.pb", r.GetToolName())), - r.GetScanInfo().GetScanUuid(), - scanStartTime.Format(time.RFC3339), - r.GetScanInfo().GetScanTags(), - ); err != nil { - log.Fatalf("could not write results: %s", err) - } - } - } - - return nil - }, +func main() { + flag.StringVar(&connStr, "db_connection", enrichers.LookupEnvOrString("ENRICHER_DB_CONNECTION", ""), "the database connection DSN") + if err := enrichers.ParseFlags(); err != nil { + log.Fatal(err) + } + if err := run(); err != nil { + log.Fatal(err) + } } -func init() { - rootCmd.Flags().StringVar(&connStr, "db_connection", "", "the database connection DSN") - rootCmd.Flags().StringVar(&readPath, "read_path", "", "the path to read LaunchToolResponses from") - rootCmd.Flags().StringVar(&writePath, "write_path", "", "the path to write enriched results to") - if err := viper.BindPFlag("db_connection", rootCmd.Flags().Lookup("db_connection")); err != nil { - log.Fatalf("could not bind db_connection flag: %s", err) +func run() error { + dbURL, err := db.ParseConnectionStr(connStr) + if err != nil { + return err } - if err := viper.BindPFlag("read_path", rootCmd.Flags().Lookup("read_path")); err != nil { - log.Fatalf("could not bind read_path flag: %s", err) + conn, err := dbURL.Connect() + if err != nil { + return err } - if err := viper.BindPFlag("write_path", rootCmd.Flags().Lookup("write_path")); err != nil { - log.Fatalf("could not bind write_path flag: %s", err) + res, err := enrichers.LoadData() + if err != nil { + return err } - viper.SetEnvPrefix("enricher") - viper.AutomaticEnv() -} - -func main() { - if err := rootCmd.Execute(); err != nil { - log.Fatal(err) + log.Printf("Loaded %d tagged tool responses\n", len(res)) + for _, r := range res { + enrichedIssues := []*v1.EnrichedIssue{} + log.Printf("enriching %d issues", len(r.GetIssues())) + for _, i := range r.GetIssues() { + eI, err := enrichment.EnrichIssue(conn, i) + if err != nil { + slog.Error(fmt.Sprintf("error enriching issue %s, err: %#v\n", i.Uuid, err)) + continue + } + enrichedIssues = append(enrichedIssues, eI) + log.Printf("enriched issue '%s'", eI.GetRawIssue().GetUuid()) + } + return enrichers.WriteData(enrichedIssues, r, "deduplication") } + return nil } diff --git a/components/enrichers/depsdev/main.go b/components/enrichers/depsdev/main.go index 85ecf3f3b..aae98899c 100644 --- a/components/enrichers/depsdev/main.go +++ b/components/enrichers/depsdev/main.go @@ -6,25 +6,21 @@ import ( "flag" "fmt" "log" + "log/slog" "net/http" "net/url" - "os" - "path/filepath" - "time" cdx "github.com/CycloneDX/cyclonedx-go" packageurl "github.com/package-url/packageurl-go" v1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components/enrichers" "github.com/ocurity/dracon/pkg/cyclonedx" - "github.com/ocurity/dracon/pkg/putil" ) const defaultAnnotation = "Enriched Licenses" var ( - readPath string - writePath string depsdevBaseURL = "https://deps.dev" licensesInEvidence string scoreCardInfo string @@ -101,13 +97,6 @@ type Response struct { DefaultVersion string `json:"defaultVersion,omitempty"` } -func lookupEnvOrString(key string, defaultVal string) string { - if val, ok := os.LookupEnv(key); ok { - return val - } - return defaultVal -} - func makeURL(component cdx.Component, api bool) (string, error) { instance, err := packageurl.FromString(component.PackageURL) if err != nil { @@ -276,10 +265,10 @@ func enrichIssue(i *v1.Issue) (*v1.EnrichedIssue, error) { return &enrichedIssue, nil } -func run() { - res, err := putil.LoadTaggedToolResponse(readPath) +func run() error { + res, err := enrichers.LoadData() if err != nil { - log.Fatalf("could not load tool response from path %s , error:%v", readPath, err) + return err } if annotation == "" { annotation = defaultAnnotation @@ -289,48 +278,29 @@ func run() { for _, i := range r.GetIssues() { eI, err := enrichIssue(i) if err != nil { - log.Println(err) + slog.Error(err.Error()) continue } enrichedIssues = append(enrichedIssues, eI) } - if len(enrichedIssues) > 0 { - if err := putil.WriteEnrichedResults(r, enrichedIssues, - filepath.Join(writePath, fmt.Sprintf("%s.depsdev.enriched.pb", r.GetToolName())), - ); err != nil { - log.Fatal(err) - } - } else { - log.Println("no enriched issues were created for", r.GetToolName()) - } - if len(r.GetIssues()) > 0 { - scanStartTime := r.GetScanInfo().GetScanStartTime().AsTime() - if err := putil.WriteResults( - r.GetToolName(), - r.GetIssues(), - filepath.Join(writePath, fmt.Sprintf("%s.raw.pb", r.GetToolName())), - r.GetScanInfo().GetScanUuid(), - scanStartTime.Format(time.RFC3339), - r.GetScanInfo().GetScanTags(), - ); err != nil { - log.Fatalf("could not write results: %s", err) - } - } - + return enrichers.WriteData(enrichedIssues, r, "deps-dev") } + return nil } func main() { - flag.StringVar(&readPath, "read_path", lookupEnvOrString("READ_PATH", ""), "where to find producer results") - flag.StringVar(&writePath, "write_path", lookupEnvOrString("WRITE_PATH", ""), "where to put enriched results") - flag.StringVar(&annotation, "annotation", lookupEnvOrString("ANNOTATION", defaultAnnotation), "what is the annotation this enricher will add to the issues, by default `Enriched Licenses`") - flag.StringVar(&scoreCardInfo, "scoreCardInfo", lookupEnvOrString("SCORECARD_INFO", "false"), "add security score card scan results from deps.dev to the components of the SBOM as properties") - flag.StringVar(&licensesInEvidence, "licensesInEvidence", lookupEnvOrString("LICENSES_IN_EVIDENCE", ""), + flag.StringVar(&annotation, "annotation", enrichers.LookupEnvOrString("ANNOTATION", defaultAnnotation), "what is the annotation this enricher will add to the issues, by default `Enriched Licenses`") + flag.StringVar(&scoreCardInfo, "scoreCardInfo", enrichers.LookupEnvOrString("SCORECARD_INFO", "false"), "add security score card scan results from deps.dev to the components of the SBOM as properties") + flag.StringVar(&licensesInEvidence, "licensesInEvidence", enrichers.LookupEnvOrString("LICENSES_IN_EVIDENCE", ""), `If this flag is provided and set to "true", the enricher will populate the 'evidence' CycloneDX field with license information instead of the license field. This means that the result conforms to the CycloneDX intention of providing accurate information when licensing information cannot be guaranteed to be accurate. However, no tools currently support reading license information from evidence. This is because deps.dev does not guarantee accurate licensing information for Go. Enable this switch if you need to provide SBOM information for regulatory reasons.`) - flag.Parse() - run() + if err := enrichers.ParseFlags(); err != nil { + log.Fatal(err) + } + if err := run(); err != nil { + log.Fatal(err) + } } diff --git a/components/enrichers/depsdev/main_test.go b/components/enrichers/depsdev/main_test.go index 0aa24952c..869e625f7 100644 --- a/components/enrichers/depsdev/main_test.go +++ b/components/enrichers/depsdev/main_test.go @@ -17,6 +17,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" v1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components/enrichers" "github.com/ocurity/dracon/pkg/cyclonedx" ) @@ -66,9 +67,8 @@ func prepareIssue(t *testing.T) string { // write sample raw issues in mktemp out, _ := proto.Marshal(&orig) require.NoError(t, os.WriteFile(dir+"/depsdevSAT.tagged.pb", out, 0o600)) - - readPath = dir - writePath = dir + enrichers.SetReadPathForTests(dir) + enrichers.SetWritePathForTests(dir) return dir } @@ -112,10 +112,10 @@ func TestParseIssuesDepsDevScoreCardInfoWritten(t *testing.T) { scoreCardInfo = "true" // run enricher run() - assert.FileExists(t, dir+"/depsdevSAT.depsdev.enriched.pb", "file was not created") + assert.FileExists(t, dir+"/depsdevSAT.deps-dev.enriched.pb", "file was not created") // load *enriched.pb - pbBytes, err := os.ReadFile(dir + "/depsdevSAT.depsdev.enriched.pb") + pbBytes, err := os.ReadFile(dir + "/depsdevSAT.deps-dev.enriched.pb") require.NoError(t, err, "could not read enriched file") res := v1.EnrichedLaunchToolResponse{} @@ -152,10 +152,10 @@ func TestParseIssuesDepsDevExternalReferenceLinksWritten(t *testing.T) { // run enricher run() - assert.FileExists(t, dir+"/depsdevSAT.depsdev.enriched.pb", "file was not created") + assert.FileExists(t, dir+"/depsdevSAT.deps-dev.enriched.pb", "file was not created") // load *enriched.pb - pbBytes, err := os.ReadFile(dir + "/depsdevSAT.depsdev.enriched.pb") + pbBytes, err := os.ReadFile(dir + "/depsdevSAT.deps-dev.enriched.pb") require.NoError(t, err, "could not read enriched file") res := v1.EnrichedLaunchToolResponse{} @@ -195,10 +195,10 @@ func TestParseIssuesLicensesWritten(t *testing.T) { // run enricher run() - assert.FileExists(t, dir+"/depsdevSAT.depsdev.enriched.pb", "file was not created") + assert.FileExists(t, dir+"/depsdevSAT.deps-dev.enriched.pb", "file was not created") // load *enriched.pb - pbBytes, err := os.ReadFile(dir + "/depsdevSAT.depsdev.enriched.pb") + pbBytes, err := os.ReadFile(dir + "/depsdevSAT.deps-dev.enriched.pb") require.NoError(t, err, "could not read enriched file") res := v1.EnrichedLaunchToolResponse{} require.NoError(t, proto.Unmarshal(pbBytes, &res)) @@ -225,10 +225,10 @@ func TestParseIssuesLicensesWrittenACcurateLicenses(t *testing.T) { run() - assert.FileExists(t, dir+"/depsdevSAT.depsdev.enriched.pb", "file was not created") + assert.FileExists(t, dir+"/depsdevSAT.deps-dev.enriched.pb", "file was not created") // load *enriched.pb - pbBytes, err := os.ReadFile(dir + "/depsdevSAT.depsdev.enriched.pb") + pbBytes, err := os.ReadFile(dir + "/depsdevSAT.deps-dev.enriched.pb") require.NoError(t, err, "could not read enriched file") res := v1.EnrichedLaunchToolResponse{} diff --git a/components/enrichers/enricher.go b/components/enrichers/enricher.go new file mode 100644 index 000000000..4f5fbd2e7 --- /dev/null +++ b/components/enrichers/enricher.go @@ -0,0 +1,94 @@ +// Package enrichers provides helper functions for writing Dracon compatible enrichers that enrich dracon outputs. +package enrichers + +import ( + "flag" + "fmt" + "log" + "log/slog" + "os" + "path/filepath" + "time" + + "github.com/go-errors/errors" + + v1 "github.com/ocurity/dracon/api/proto/v1" + component "github.com/ocurity/dracon/components" + + draconLogger "github.com/ocurity/dracon/pkg/log" + "github.com/ocurity/dracon/pkg/putil" +) + +var ( + readPath string + writePath string + // Debug flag initializes the logger with a debug level + Debug bool +) + +func LookupEnvOrString(key string, defaultVal string) string { + if val, ok := os.LookupEnv(key); ok { + return val + } + return defaultVal +} + +// ParseFlags will parse the input flags for the producer and perform simple validation. +func ParseFlags() error { + flag.StringVar(&readPath, "read_path", LookupEnvOrString("READ_PATH", ""), "where to find producer results") + flag.StringVar(&writePath, "write_path", LookupEnvOrString("WRITE_PATH", ""), "where to put enriched results") + flag.BoolVar(&Debug, "debug", false, "turn on debug logging") + + flag.Parse() + if Debug { + draconLogger.SetDefault(slog.LevelDebug, os.Getenv(component.EnvDraconScanID), true) + } else { + draconLogger.SetDefault(slog.LevelInfo, os.Getenv(component.EnvDraconScanID), true) + } + if readPath == "" { + return fmt.Errorf("read_path is undefined") + } + if writePath == "" { + return fmt.Errorf("write_path is undefined") + } + return nil +} + +// LoadData returns the LaunchToolResponses meant for this enricher. +func LoadData() ([]*v1.LaunchToolResponse, error) { + return putil.LoadTaggedToolResponse(readPath) +} + +func WriteData(enrichedIssues []*v1.EnrichedIssue, originalResults *v1.LaunchToolResponse, enricherName string) error { + if len(enrichedIssues) > 0 { + if err := putil.WriteEnrichedResults(originalResults, enrichedIssues, + filepath.Join(writePath, fmt.Sprintf("%s.%s.enriched.pb", originalResults.GetToolName(), enricherName)), + ); err != nil { + return err + } + } else { + log.Println("no enriched issues were created for", originalResults.GetToolName()) + } + if len(originalResults.GetIssues()) > 0 { + scanStartTime := originalResults.GetScanInfo().GetScanStartTime().AsTime() + if err := putil.WriteResults( + originalResults.GetToolName(), + originalResults.GetIssues(), + filepath.Join(writePath, fmt.Sprintf("%s.raw.pb", originalResults.GetToolName())), + originalResults.GetScanInfo().GetScanUuid(), + scanStartTime.Format(time.RFC3339), + originalResults.GetScanInfo().GetScanTags(), + ); err != nil { + return errors.Errorf("could not write results: %s", err) + } + } + return nil +} + +func SetReadPathForTests(readFromPath string) { + readPath = readFromPath +} + +func SetWritePathForTests(writeToPath string) { + writePath = writeToPath +} diff --git a/components/enrichers/policy/main.go b/components/enrichers/policy/main.go index 2e5426bb2..656b44c6d 100644 --- a/components/enrichers/policy/main.go +++ b/components/enrichers/policy/main.go @@ -5,23 +5,18 @@ import ( "flag" "fmt" "log" - "os" - "path/filepath" "strconv" - "time" "google.golang.org/protobuf/encoding/protojson" v1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components/enrichers" opaclient "github.com/ocurity/dracon/components/enrichers/policy/opaClient" - "github.com/ocurity/dracon/pkg/putil" ) var ( policy string regoServer string - readPath string - writePath string annotation string ) @@ -31,20 +26,12 @@ type opaIssue struct { Input map[string]interface{} } -func lookupEnvOrString(key string, defaultVal string) string { - if val, ok := os.LookupEnv(key); ok { - return val - } - return defaultVal -} - func enrichIssue(i *v1.Issue, client opaclient.Client) (*v1.EnrichedIssue, error) { var strFinding map[string]interface{} if err := json.Unmarshal([]byte(protojson.Format(i)), &strFinding); err != nil { log.Println("Could not marshal proto to json err:", err) } - opaIssue := opaIssue{ Input: strFinding, } @@ -65,10 +52,10 @@ func enrichIssue(i *v1.Issue, client opaclient.Client) (*v1.EnrichedIssue, error return issue, nil } -func run() { - res, err := putil.LoadTaggedToolResponse(readPath) +func run() error { + res, err := enrichers.LoadData() if err != nil { - log.Fatalf("could not load tool response from path %s , error:%v", readPath, err) + return err } if annotation == "" { annotation = defaultAnnotation @@ -78,11 +65,9 @@ func run() { Policy: policy, } if err := client.Bootstrap(); err != nil { - log.Fatalf("Could not bootstrap OPA policies, err: %v", err) - return + return fmt.Errorf("could not bootstrap OPA policies, err: %v", err) } log.Printf("successfully bootstrapped policy %s", client.PolicyPath) - for _, r := range res { enrichedIssues := []*v1.EnrichedIssue{} for _, i := range r.GetIssues() { @@ -93,38 +78,19 @@ func run() { } enrichedIssues = append(enrichedIssues, eI) } - if len(enrichedIssues) > 0 { - if err := putil.WriteEnrichedResults(r, enrichedIssues, - filepath.Join(writePath, fmt.Sprintf("%s.policy.enriched.pb", r.GetToolName())), - ); err != nil { - log.Fatal(err) - } - } else { - log.Println("no enriched issues were created for", r.GetToolName()) - } - if len(r.GetIssues()) > 0 { - scanStartTime := r.GetScanInfo().GetScanStartTime().AsTime() - if err := putil.WriteResults( - r.GetToolName(), - r.GetIssues(), - filepath.Join(writePath, fmt.Sprintf("%s.raw.pb", r.GetToolName())), - r.GetScanInfo().GetScanUuid(), - scanStartTime.Format(time.RFC3339), - r.GetScanInfo().GetScanTags(), - ); err != nil { - log.Fatalf("could not write results: %s", err) - } - } - + return enrichers.WriteData(enrichedIssues, r, "policy") } + return nil } func main() { - flag.StringVar(&policy, "policy", lookupEnvOrString("POLICY", ""), "base64 encoded policy") - flag.StringVar(&annotation, "annotation", lookupEnvOrString("ANNOTATION", defaultAnnotation), "How to label the issues this binary will enrich by default `Policy Pass: `") - flag.StringVar(®oServer, "opa_server", lookupEnvOrString("OPA_SERVER", "http://127.0.0.1:8181"), "where to find the rego server") - flag.StringVar(&readPath, "read_path", lookupEnvOrString("READ_PATH", ""), "where to find producer results") - flag.StringVar(&writePath, "write_path", lookupEnvOrString("WRITE_PATH", ""), "where to put enriched results") - flag.Parse() - run() + flag.StringVar(&policy, "policy", enrichers.LookupEnvOrString("POLICY", ""), "base64 encoded policy") + flag.StringVar(&annotation, "annotation", enrichers.LookupEnvOrString("ANNOTATION", defaultAnnotation), "How to label the issues this binary will enrich by default `Policy Pass: `") + flag.StringVar(®oServer, "opa_server", enrichers.LookupEnvOrString("OPA_SERVER", "http://127.0.0.1:8181"), "where to find the rego server") + if err := enrichers.ParseFlags(); err != nil { + log.Fatal(err) + } + if err := run(); err != nil { + log.Fatal(err) + } } diff --git a/components/enrichers/policy/main_test.go b/components/enrichers/policy/main_test.go index 70d5886b8..a4d739e85 100644 --- a/components/enrichers/policy/main_test.go +++ b/components/enrichers/policy/main_test.go @@ -17,6 +17,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" v1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components/enrichers" ) const ( @@ -71,9 +72,8 @@ func TestParseIssues(t *testing.T) { out, err := proto.Marshal(&orig) require.NoError(t, err) require.NoError(t, os.WriteFile(dir+"/policySat.tagged.pb", out, 0o600)) - - readPath = dir - writePath = dir + enrichers.SetReadPathForTests(dir) + enrichers.SetWritePathForTests(dir) policy = "cGFja2FnZSBleGFtcGxlLnBvbGljeVNhdAoKZGVmYXVsdCBhbGxvdyA6PSBmYWxzZQoKYWxsb3cgPXRydWUgewogICAgcHJpbnQoaW5wdXQpCiAgICBjaGVja19zZXZlcml0eQp9CgpjaGVja19zZXZlcml0eSB7CiAgICBpbnB1dC5zZXZlcml0eSA9PSAiU0VWRVJJVFlfTE9XIgp9CgpjaGVja19zZXZlcml0eSB7CiAgICBpbnB1dC5zZXZlcml0eSA9PSAiU0VWRVJJVFlfSElHSCIKfQpjaGVja19zZXZlcml0eSB7CiAgICBpbnB1dC5zZXZlcml0eSA9PSAiU0VWRVJJVFlfTUVESVVNIgp9Cg==" // setup server