Skip to content

Commit

Permalink
fix 349 by adding api keys to the elasticsearch consumer so that it c…
Browse files Browse the repository at this point in the history 
…an work with elasticsearch saas
  • Loading branch information
@northdpole @ptzianos
northdpole authored and ptzianos committed Sep 15, 2024
1 parent 360c5d3 commit 1181aa3
Show file tree
Hide file tree
Showing 3 changed files with 103 additions and 41 deletions.
65 changes: 40 additions & 25 deletions components/consumers/elasticsearch/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"flag"
"fmt"
"log"
"log/slog"
"strings"
"time"

Expand All @@ -14,33 +15,28 @@ import (
"github.com/ocurity/dracon/pkg/enumtransformers"
"github.com/ocurity/dracon/pkg/templating"

// TODO: Support multiple versions of ES
elasticsearch "github.com/elastic/go-elasticsearch/v8"
)

var (
esUrls string
esAddrs []string
esIndex string
esUrls string
esAddrs []string
esIndex string

esAPIKey string
esCloudID string

basicAuthUser string
basicAuthPass string
issueTemplate string
)

func init() {
flag.StringVar(&esUrls, "es-urls", "", "[OPTIONAL] URLs to connect to elasticsearch comma separated. Can also use env variable ELASTICSEARCH_URL")
flag.StringVar(&esIndex, "es-index", "", "the index in elasticsearch to push results to")
flag.StringVar(&basicAuthUser, "basic-auth-user", "", "[OPTIONAL] the basic auth username")
flag.StringVar(&basicAuthPass, "basic-auth-pass", "", "[OPTIONAL] the basic auth password")
flag.StringVar(&issueTemplate, "descriptionTemplate", "", "a Go Template string describing how to show Raw or Enriched issues")
}

func parseFlags() error {
if err := consumers.ParseFlags(); err != nil {
return err
}
if len(esIndex) < 1 {
return fmt.Errorf("es-index is undefined")
if len(esIndex) == 0 {
return fmt.Errorf("esIndex '%s' is undefined", esIndex)
}
if len(esUrls) > 0 {
for _, u := range strings.Split(esUrls, ",") {
Expand All @@ -51,6 +47,19 @@ func parseFlags() error {
}

func main() {
flag.StringVar(&esIndex, "esIndex", "", "the index in elasticsearch to push results to")
flag.StringVar(&issueTemplate, "descriptionTemplate", "", "a Go Template string describing how to show Raw or Enriched issues")

// es SaaS options
flag.StringVar(&esAPIKey, "esAPIKey", "", "the api key in elasticsearch to contact results to")
flag.StringVar(&esCloudID, "esCloudID", "", "the cloud id in elasticsearch to contact results to")

// es self-hosted options
flag.StringVar(&esUrls, "esURL", "", "[OPTIONAL] URLs to connect to elasticsearch comma separated. Can also use env variable ELASTICSEARCH_URL")
flag.StringVar(&basicAuthUser, "basic-auth-user", "", "[OPTIONAL] the basic auth username")
flag.StringVar(&basicAuthPass, "basic-auth-pass", "", "[OPTIONAL] the basic auth password")
flag.Parse()

if err := parseFlags(); err != nil {
log.Fatal(err)
}
Expand All @@ -61,15 +70,16 @@ func main() {
}

if consumers.Raw {
log.Print("Parsing Raw results")
slog.Debug("Parsing Raw results")
responses, err := consumers.LoadToolResponse()
if err != nil {
log.Fatal("could not load raw results, file malformed: ", err)
}
numIssues := 0
for _, res := range responses {
scanStartTime := res.GetScanInfo().GetScanStartTime().AsTime()
numIssues += len(res.GetIssues())
for _, iss := range res.GetIssues() {
log.Printf("Pushing %d, issues to es \n", len(responses))
b, err := getRawIssue(scanStartTime, res, iss)
if err != nil {
log.Fatal("Could not parse raw issue", err)
Expand All @@ -81,26 +91,29 @@ func main() {
}
}
}
slog.Info("Pushed", "numIssues", numIssues, "issues to Elasticsearch", "")
} else {
log.Print("Parsing Enriched results")
responses, err := consumers.LoadEnrichedToolResponse()
if err != nil {
log.Fatal("could not load enriched results, file malformed: ", err)
}
numIssues := 0
for _, res := range responses {
scanStartTime := res.GetOriginalResults().GetScanInfo().GetScanStartTime().AsTime()
numIssues += len(res.GetIssues())
for _, iss := range res.GetIssues() {
b, err := getEnrichedIssue(scanStartTime, res, iss)
if err != nil {
log.Fatal("Could not parse enriched issue", err)
}
res, err := es.Index(esIndex, bytes.NewBuffer(b))
log.Printf("%+v", res)
if err != nil {
log.Fatal("Could not push enriched issue", err)
if err != nil || res.IsError() {
log.Fatal("Could not push enriched issue", err, "received", res.StatusCode)
}
}
}
slog.Info("Pushed", "numIssues", numIssues, "issues to Elasticsearch", "")
}
}

Expand Down Expand Up @@ -194,7 +207,12 @@ func getESClient() (*elasticsearch.Client, error) {
esConfig.Username = basicAuthUser
esConfig.Password = basicAuthPass
}

if esAPIKey != "" {
esConfig.APIKey = esAPIKey
}
if esCloudID != "" {
esConfig.CloudID = esCloudID
}
if len(esAddrs) > 0 {
esConfig.Addresses = esAddrs
}
Expand All @@ -217,11 +235,8 @@ func getESClient() (*elasticsearch.Client, error) {
if err := json.NewDecoder(res.Body).Decode(&info); err != nil {
return nil, err
}
switch info.Version.Number[0] {
case '8':
// noop - we support this version
default:
err = fmt.Errorf("unsupported ES Server version %s", info.Version.Number)
if info.Version.Number[0] != '8' {
return nil, fmt.Errorf("unsupported ES Server version %s", info.Version.Number)
}
return es, err
}
57 changes: 46 additions & 11 deletions components/consumers/elasticsearch/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (

v1 "github.com/ocurity/dracon/api/proto/v1"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -55,21 +54,21 @@ func TestEsPushBasicAuth(t *testing.T) {

if r.Method == http.MethodGet {
uname, pass, ok := r.BasicAuth()
assert.Equal(t, uname, "foo")
assert.Equal(t, pass, "bar")
assert.Equal(t, ok, true)
require.Equal(t, uname, "foo")
require.Equal(t, pass, "bar")
require.Equal(t, ok, true)

_, err = w.Write([]byte(info))
require.NoError(t, err)
} else if r.Method == http.MethodPost {
// assert non authed operation (write results to index)
assert.Equal(t, buf.String(), string(esIn))
assert.Equal(t, r.RequestURI, "/"+esIndex+"/_doc")
require.Equal(t, buf.String(), string(esIn))
require.Equal(t, r.RequestURI, "/"+esIndex+"/_doc")

uname, pass, ok := r.BasicAuth()
assert.Equal(t, uname, "foo")
assert.Equal(t, pass, "bar")
assert.Equal(t, ok, true)
require.Equal(t, uname, "foo")
require.Equal(t, pass, "bar")
require.Equal(t, ok, true)

_, err = w.Write([]byte(want))
require.NoError(t, err)
Expand Down Expand Up @@ -99,8 +98,8 @@ func TestEsPush(t *testing.T) {
_, err = w.Write([]byte(info))
} else if r.Method == http.MethodPost {
// assert non authed operation (write results to index)
assert.Equal(t, buf.String(), string(esIn))
assert.Equal(t, r.RequestURI, "/"+esIndex+"/_doc")
require.Equal(t, buf.String(), string(esIn))
require.Equal(t, r.RequestURI, "/"+esIndex+"/_doc")
_, err = w.Write([]byte(want))
}
require.NoError(t, err)
Expand All @@ -112,3 +111,39 @@ func TestEsPush(t *testing.T) {
_, err = client.Index(esIndex, bytes.NewBuffer(esIn))
require.NoError(t, err)
}
func TestEsPushAPIKey(t *testing.T) {
esIndex = "dracon-es-test"

esStub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buf := new(bytes.Buffer)
_, err := buf.ReadFrom(r.Body)
require.NoError(t, err)

w.Header().Set("X-Elastic-Product", "Elasticsearch")
w.WriteHeader(http.StatusOK)

require.Equal(t, r.Header.Get("Authorization"), "APIKey foo")

if r.Method == http.MethodGet {
_, err = w.Write([]byte(info))
require.NoError(t, err)
} else if r.Method == http.MethodPost {
// assert non authed operation (write results to index)
require.Equal(t, buf.String(), string(esIn))
require.Equal(t, r.RequestURI, "/"+esIndex+"/_doc")

_, err = w.Write([]byte(want))
require.NoError(t, err)
}
}))
defer esStub.Close()
os.Setenv("ELASTICSEARCH_URL", esStub.URL)

// apikey ops
esAPIKey = "foo"
esCloudID = esStub.Config.Addr
client, err := getESClient()
require.NoError(t, err)
_, err = client.Index(esIndex, bytes.NewBuffer(esIn))
require.NoError(t, err)
}
22 changes: 17 additions & 5 deletions components/consumers/elasticsearch/task.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,18 @@ spec:
- name: consumer-elasticsearch-description-template
type: string
default: ""
- name: consumer-elasticsearch-api-key
type: string
default: ""
- name: consumer-elasticsearch-index-name
type: string
default: ""
- name: consumer-elasticsearch-index
type: string
default: ""
- name: consumer-elasticsearch-cloud-id
type: string
default: ""
workspaces:
- name: output
description: The workspace containing the source-code to scan.
Expand All @@ -22,11 +34,11 @@ spec:
imagePullPolicy: IfNotPresent
image: '{{ default "ghcr.io/ocurity/dracon" .Values.image.registry }}/components/consumers/elasticsearch:{{ .Chart.AppVersion }}'
command: ["/app/components/consumers/elasticsearch/elasticsearch"]
env:
- name: ELASTICSEARCH_URL
value: "$(params.consumer-elasticsearch-url)"
args: [
"-in", "$(workspaces.output.path)/.dracon/enrichers/",
"-es-index", "dracon",
"-descriptionTemplate","$(params.consumer-elasticsearch-description-template)"
"-descriptionTemplate","$(params.consumer-elasticsearch-description-template)",
"-esIndex", "$(params.consumer-elasticsearch-index-name)",
"-esAPIKey", "$(params.consumer-elasticsearch-api-key)",
"-esURL", "$(params.consumer-elasticsearch-url)",
"-esCloudID", "$(params.consumer-elasticsearch-cloud-id)",
]

0 comments on commit 1181aa3

Please sign in to comment.