Merge branch 'main' into codeowners-enricher

components/enrichers/depsdev/kustomization.yaml

@@ -25,12 +25,17 @@ patches:
             params:
               - name: enricher-depsdev-licenses-in-evidence
                 value: $(params.enricher-depsdev-licenses-in-evidence)
+              - name: enricher-depsdev-scorecard-info
+                value: $(params.enricher-depsdev-scorecard-info)
               - name: enricher-depsdev-annotation
                 value: $(params.enricher-depsdev-annotation)
         params:
           - name: enricher-depsdev-licenses-in-evidence
             type: string
             default: "false"
+          - name: enricher-depsdev-scorecard-info
+            type: string
+            default: "true"
           - name: enricher-depsdev-annotation
             type: string
             default: ""

components/enrichers/depsdev/main.go

@@ -30,6 +30,50 @@ var (
 	annotation         string
 )
 
+// Check is a deps.dev ScoreCardV2 check
+type Check struct {
+	Name          string `json:"name,omitempty"`
+	Documentation struct {
+		Short string `json:"short,omitempty"`
+		URL   string `json:"url,omitempty"`
+	} `json:"documentation,omitempty"`
+	Score   int           `json:"score,omitempty"`
+	Reason  string        `json:"reason,omitempty"`
+	Details []interface{} `json:"details,omitempty"`
+}
+
+// ScorecardV2 is a deps.dev ScoreCardV2 result
+type ScorecardV2 struct {
+	Date string `json:"date,omitempty"`
+	Repo struct {
+		Name   string `json:"name,omitempty"`
+		Commit string `json:"commit,omitempty"`
+	} `json:"repo,omitempty"`
+	Scorecard struct {
+		Version string `json:"version,omitempty"`
+		Commit  string `json:"commit,omitempty"`
+	} `json:"scorecard,omitempty"`
+	Check    []Check       `json:"check,omitempty"`
+	Metadata []interface{} `json:"metadata,omitempty"`
+	Score    float64       `json:"score,omitempty"`
+}
+
+// Project is a deps.dev project
+type Project struct {
+	Type        string      `json:"type,omitempty"`
+	Name        string      `json:"name,omitempty"`
+	ObservedAt  int         `json:"observedAt,omitempty"`
+	Issues      int         `json:"issues,omitempty"`
+	Forks       int         `json:"forks,omitempty"`
+	Stars       int         `json:"stars,omitempty"`
+	Description string      `json:"description,omitempty"`
+	License     string      `json:"license,omitempty"`
+	DisplayName string      `json:"displayName,omitempty"`
+	Link        string      `json:"link,omitempty"`
+	ScorecardV2 ScorecardV2 `json:"scorecardV2,omitempty"`
+}
+
+// Version is a deps.dev version, main object in the response
 type Version struct {
 	Version                string        `json:"version,omitempty"`
 	SymbolicVersions       []interface{} `json:"symbolicVersions,omitempty"`
@@ -42,41 +86,7 @@ type Version struct {
 	Links                  struct {
 		Origins []string `json:"origins,omitempty"`
 	} `json:"links,omitempty"`
-	Projects []struct {
-		Type        string `json:"type,omitempty"`
-		Name        string `json:"name,omitempty"`
-		ObservedAt  int    `json:"observedAt,omitempty"`
-		Issues      int    `json:"issues,omitempty"`
-		Forks       int    `json:"forks,omitempty"`
-		Stars       int    `json:"stars,omitempty"`
-		Description string `json:"description,omitempty"`
-		License     string `json:"license,omitempty"`
-		DisplayName string `json:"displayName,omitempty"`
-		Link        string `json:"link,omitempty"`
-		ScorecardV2 struct {
-			Date string `json:"date,omitempty"`
-			Repo struct {
-				Name   string `json:"name,omitempty"`
-				Commit string `json:"commit,omitempty"`
-			} `json:"repo,omitempty"`
-			Scorecard struct {
-				Version string `json:"version,omitempty"`
-				Commit  string `json:"commit,omitempty"`
-			} `json:"scorecard,omitempty"`
-			Check []struct {
-				Name          string `json:"name,omitempty"`
-				Documentation struct {
-					Short string `json:"short,omitempty"`
-					URL   string `json:"url,omitempty"`
-				} `json:"documentation,omitempty"`
-				Score   int           `json:"score,omitempty"`
-				Reason  string        `json:"reason,omitempty"`
-				Details []interface{} `json:"details,omitempty"`
-			} `json:"check,omitempty"`
-			Metadata []interface{} `json:"metadata,omitempty"`
-			Score    float64       `json:"score,omitempty"`
-		} `json:"scorecardV2,omitempty"`
-	} `json:"projects,omitempty"`
+	Projects        []Project     `json:"projects,omitempty"`
 	Advisories      []interface{} `json:"advisories,omitempty"`
 	RelatedPackages struct{}      `json:"relatedPackages,omitempty"`
 }
@@ -125,7 +135,7 @@ func makeURL(component cdx.Component, api bool) (string, error) {
 	if api {
 		resultURL = fmt.Sprintf("%s/_/s%s/p/%s/v/%s", depsdevBaseURL, ecosystem, url.QueryEscape(component.Name), version)
 	} else {
-		resultURL = fmt.Sprintf("%s/%s/p/%s/v/%s", depsdevBaseURL, ecosystem, url.QueryEscape(component.Name), version)
+		resultURL = fmt.Sprintf("%s%s/p/%s/v/%s", depsdevBaseURL, ecosystem, url.QueryEscape(component.Name), version)
 	}
 	return resultURL, nil
 }
@@ -135,8 +145,6 @@ func addDepsDevLink(component cdx.Component) (cdx.Component, error) {
 	if err != nil {
 		return component, err
 	}
-	log.Println("url is", url)
-
 	depsDevRef := cdx.ExternalReference{
 		Type: cdx.ERTypeOther,
 		URL:  url,
@@ -152,7 +160,6 @@ func addDepsDevLink(component cdx.Component) (cdx.Component, error) {
 
 	return component, nil
 }
-
 func addDepsDevInfo(component cdx.Component, annotations map[string]string) (cdx.Component, map[string]string, error) {
 	var depsResp Response
 	licenses := cdx.Licenses{}
@@ -161,7 +168,6 @@ func addDepsDevInfo(component cdx.Component, annotations map[string]string) (cdx
 		return component, annotations, err
 	}
 	resp, err := http.Get(url) // nolint: gosec, url get constructed above with a hardcoded domain and relatively trusted data
-	log.Println("url is", url)
 	if err != nil {
 		return component, annotations, err
 	}
@@ -181,6 +187,7 @@ func addDepsDevInfo(component cdx.Component, annotations map[string]string) (cdx
 		log.Println("found license", lic, "for component", component.Name)
 	}
 	if scoreCardInfo == "true" {
+		log.Println("adding scorecard info")
 		for _, project := range depsResp.Version.Projects {
 			if project.ScorecardV2.Date != "" && len(project.ScorecardV2.Check) != 0 && project.ScorecardV2.Score >= 0 {
 				scoreCardInfo, err := json.MarshalIndent(project.ScorecardV2, "", "\t")
@@ -204,6 +211,7 @@ func addDepsDevInfo(component cdx.Component, annotations map[string]string) (cdx
 		}
 	}
 	if licensesInEvidence == "true" {
+		log.Println("adding Licenses in the 'Evidence' field")
 		evid := cdx.Evidence{
 			Licenses: &licenses,
 		}
@@ -215,7 +223,6 @@ func addDepsDevInfo(component cdx.Component, annotations map[string]string) (cdx
 	} else {
 		component.Licenses = &licenses
 	}
-
 	annotations[annotation] = "True"
 	return component, annotations, nil
 }
@@ -230,24 +237,27 @@ func enrichIssue(i *v1.Issue) (*v1.EnrichedIssue, error) {
 	if bom == nil || *bom.Components == nil {
 		return &enrichedIssue, errors.New("bom does not have components")
 	}
-	for index, component := range *bom.Components {
+	newComponents := (*bom.Components)[:0]
+	for _, component := range *bom.Components {
+		newComp := component
 		if component.Type == cdx.ComponentTypeLibrary {
 			if component.Licenses == nil {
-				(*bom.Components)[index], annotations, err = addDepsDevInfo(component, annotations)
-				if err != nil {
-					log.Println(err)
-					continue
-				}
-				(*bom.Components)[index], err = addDepsDevLink(component)
+				newComp, annotations, err = addDepsDevInfo(component, annotations)
 				if err != nil {
 					log.Println(err)
 					continue
 				}
 			}
+			newComp, err = addDepsDevLink(newComp)
+			if err != nil {
+				log.Println(err)
+				continue
+			}
 			// TODO(): enrich with vulnerability info whenever a consumer supports showing arbitrary properties in components
 		}
+		newComponents = append(newComponents, newComp)
 	}
-
+	bom.Components = &newComponents
 	marshalled, err := json.Marshal(bom)
 	if err != nil {
 		return &enrichedIssue, err
@@ -289,7 +299,7 @@ func run() {
 				log.Fatal(err)
 			}
 		} else {
-			log.Println("no enriched issues were created")
+			log.Println("no enriched issues were created for", r.GetToolName())
 		}
 		if len(r.GetIssues()) > 0 {
 			scanStartTime := r.GetScanInfo().GetScanStartTime().AsTime()

components/enrichers/depsdev/main_test.go

@@ -2,13 +2,15 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 	"time"
 
+	cdx "github.com/CycloneDX/cyclonedx-go"
 	"github.com/google/uuid"
 	v1 "github.com/ocurity/dracon/api/proto/v1"
 	"github.com/ocurity/dracon/pkg/cyclonedx"
@@ -70,23 +72,125 @@ func prepareIssue() string {
 func MockServer(t *testing.T) {
 }
 
-func TestParseIssuesLicensesWritten(t *testing.T) {
+// TODO make this be the common setup method
+// todo add test for deps dev and scorecard stuff
+func setup(t *testing.T) (string, *httptest.Server) {
 	dir := prepareIssue()
 
 	// setup server
 	response := Response{
 		Version: Version{
 			Licenses: []string{license},
+			Projects: []Project{
+				{
+					ScorecardV2: ScorecardV2{
+						Date:  "irrelevant",
+						Score: 5.5,
+						Check: []Check{
+							{
+								Name:   "foo",
+								Score:  2,
+								Reason: "bar",
+							},
+						},
+					},
+				},
+			},
 		},
 	}
-	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Contains(t, r.URL.String(), "/_/s/go/p/")
 		json.NewEncoder(w).Encode(response)
 	}))
-	defer svr.Close()
-	depsdevBaseURL = svr.URL
+	depsdevBaseURL = srv.URL
+	return dir, srv
+}
+
+func TestParseIssuesDepsDevScoreCardInfoWritten(t *testing.T) {
+	dir, srv := setup(t)
+	defer srv.Close()
+	scoreCardInfo = "true"
+	// run enricher
 	run()
+	assert.FileExists(t, dir+"/depsdevSAT.depsdev.enriched.pb", "file was not created")
+
+	// load *enriched.pb
+	pbBytes, err := ioutil.ReadFile(dir + "/depsdevSAT.depsdev.enriched.pb")
+	assert.NoError(t, err, "could not read enriched file")
+	res := v1.EnrichedLaunchToolResponse{}
+	proto.Unmarshal(pbBytes, &res)
+	expectedProperties := []cdx.Property{
+		{Name: "aquasecurity:trivy:PkgType", Value: "gomod"},
+		{Name: "ScorecardScore", Value: "5.500000"},
+		{Name: "ScorecardInfo", Value: "{\n\t\"date\": \"irrelevant\",\n\t\"repo\": {},\n\t\"scorecard\": {},\n\t\"check\": [\n\t\t{\n\t\t\t\"name\": \"foo\",\n\t\t\t\"documentation\": {},\n\t\t\t\"score\": 2,\n\t\t\t\"reason\": \"bar\"\n\t\t}\n\t],\n\t\"score\": 5.5\n}"},
+		{Name: "aquasecurity:trivy:PkgType", Value: "gomod"},
+		{Name: "ScorecardScore", Value: "5.500000"},
+		{Name: "ScorecardInfo", Value: "{\n\t\"date\": \"irrelevant\",\n\t\"repo\": {},\n\t\"scorecard\": {},\n\t\"check\": [\n\t\t{\n\t\t\t\"name\": \"foo\",\n\t\t\t\"documentation\": {},\n\t\t\t\"score\": 2,\n\t\t\t\"reason\": \"bar\"\n\t\t}\n\t],\n\t\"score\": 5.5\n}"},
+		{Name: "aquasecurity:trivy:PkgType", Value: "gomod"},
+		{Name: "ScorecardScore", Value: "5.500000"},
+		{Name: "ScorecardInfo", Value: "{\n\t\"date\": \"irrelevant\",\n\t\"repo\": {},\n\t\"scorecard\": {},\n\t\"check\": [\n\t\t{\n\t\t\t\"name\": \"foo\",\n\t\t\t\"documentation\": {},\n\t\t\t\"score\": 2,\n\t\t\t\"reason\": \"bar\"\n\t\t}\n\t],\n\t\"score\": 5.5\n}"},
+	}
+	//  ensure every component has a license attached to it
+	for _, finding := range res.Issues {
+		bom, err := cyclonedx.FromDracon(finding.RawIssue)
+		assert.NoError(t, err, "Could not read enriched cyclone dx info")
 
+		properties := []cdx.Property{}
+
+		for _, component := range *bom.Components {
+			properties = append(properties, *component.Properties...)
+		}
+		assert.Equal(t, properties, expectedProperties)
+	}
+}
+
+func TestParseIssuesDepsDevExternalReferenceLinksWritten(t *testing.T) {
+	dir, srv := setup(t)
+	defer srv.Close()
+
+	// run enricher
+	run()
+	assert.FileExists(t, dir+"/depsdevSAT.depsdev.enriched.pb", "file was not created")
+
+	// load *enriched.pb
+	pbBytes, err := ioutil.ReadFile(dir + "/depsdevSAT.depsdev.enriched.pb")
+	assert.NoError(t, err, "could not read enriched file")
+	res := v1.EnrichedLaunchToolResponse{}
+	proto.Unmarshal(pbBytes, &res)
+	expectedExternalReferences := []cdx.ExternalReference{
+		{
+			URL:  fmt.Sprintf("%s/go/p/cloud.google.com%%2Fgo%%2Fcompute/v/v1.14.0", srv.URL),
+			Type: "other",
+		}, {
+			URL:  fmt.Sprintf("%s/go/p/cloud.google.com%%2Fgo%%2Fcompute%%2Fmetadata/v/v0.2.3", srv.URL),
+			Type: "other",
+		}, {
+			URL:  fmt.Sprintf("%s/go/p/github.com%%2FAzure%%2Fazure-pipeline-go/v/v0.2.3", srv.URL),
+			Type: "other",
+		},
+	}
+	//  ensure every component has a license attached to it
+	for _, finding := range res.Issues {
+		bom, err := cyclonedx.FromDracon(finding.RawIssue)
+		assert.NoError(t, err, "Could not read enriched cyclone dx info")
+
+		externalReferences := []cdx.ExternalReference{}
+
+		for _, component := range *bom.Components {
+			externalReferences = append(externalReferences, *component.ExternalReferences...)
+		}
+		assert.Equal(t, externalReferences, expectedExternalReferences)
+	}
+}
+
+func TestParseIssuesLicensesWritten(t *testing.T) {
+	dir, srv := setup(t)
+	defer srv.Close()
+
+	licensesInEvidence = "false"
+
+	// run enricher
+	run()
 	assert.FileExists(t, dir+"/depsdevSAT.depsdev.enriched.pb", "file was not created")
 
 	// load *enriched.pb
@@ -112,20 +216,8 @@ func TestParseIssuesLicensesWritten(t *testing.T) {
 }
 
 func TestParseIssuesLicensesWrittenACcurateLicenses(t *testing.T) {
-	dir := prepareIssue()
-
-	// setup server
-	response := Response{
-		Version: Version{
-			Licenses: []string{license},
-		},
-	}
-	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		assert.Contains(t, r.URL.String(), "/_/s/go/p/")
-		json.NewEncoder(w).Encode(response)
-	}))
-	defer svr.Close()
-	depsdevBaseURL = svr.URL
+	dir, srv := setup(t)
+	defer srv.Close()
 	licensesInEvidence = "true"
 
 	run()

go.mod

@@ -50,7 +50,7 @@ require (
 	github.com/klauspost/asmfmt v1.3.2 // indirect
 	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
-	github.com/kr/text v0.2.0 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect