diff --git a/Makefile b/Makefile index f0c6308bd..44100650d 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,8 @@ clean: clean-protos clean-migrations-compose .PHONY: lint install-lint-tools tests go-tests fmt fmt-proto fmt-go install-go-fmt-tools migration-tests lint: - @reviewdog -fail-on-error $$([ "${CI}" = "true" ] && echo "-reporter=github-pr-review") -diff="git diff origin/main" -filter-mode=added -tee +# we need to redirect stderr to stdout because Github actions don't capture the stderr lolz + @reviewdog -fail-on-error -diff="git diff origin/main" -filter-mode=added 2>&1 install-lint-tools: @go install honnef.co/go/tools/cmd/staticcheck@latest diff --git a/cmd/draconctl/main.go b/cmd/draconctl/main.go index cc238b92a..448435df4 100644 --- a/cmd/draconctl/main.go +++ b/cmd/draconctl/main.go @@ -1,6 +1,7 @@ package main import ( + "log/slog" "os" "github.com/spf13/cobra" @@ -20,11 +21,10 @@ func main() { ID: "top-level", Title: "Top-level Commands:", }) - pipelines.RegisterPipelinesSubcommands(rootCmd) migrations.RegisterMigrationsSubcommands(rootCmd) components.RegisterComponentsSubcommands(rootCmd) - + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{}))) err := rootCmd.Execute() if err != nil { os.Exit(1) diff --git a/components/component.go b/components/component.go new file mode 100644 index 000000000..8486be2a2 --- /dev/null +++ b/components/component.go @@ -0,0 +1,10 @@ +package components + +const ( + // EnvDraconStartTime Start Time of Dracon Scan in RFC3339. + EnvDraconStartTime = "DRACON_SCAN_TIME" + // EnvDraconScanID the ID of the dracon scan. + EnvDraconScanID = "DRACON_SCAN_ID" + // EnvDraconScanTags the tags of the dracon scan. + EnvDraconScanTags = "DRACON_SCAN_TAGS" +) diff --git a/components/consumers/consumer.go b/components/consumers/consumer.go index 7cfd8dd2d..67586d829 100644 --- a/components/consumers/consumer.go +++ b/components/consumers/consumer.go @@ -5,35 +5,38 @@ package consumers import ( "flag" "fmt" + "log/slog" + "os" - v1 "github.com/ocurity/dracon/api/proto/v1" - + draconapiv1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components" "github.com/ocurity/dracon/pkg/putil" ) -const ( - // EnvDraconStartTime Start Time of Dracon Scan in RFC3339. - EnvDraconStartTime = "DRACON_SCAN_TIME" - // EnvDraconScanID the ID of the dracon scan. - EnvDraconScanID = "DRACON_SCAN_ID" - // EnvDraconScanTags the tags of the dracon scan. - EnvDraconScanTags = "DRACON_SCAN_TAGS" -) - var ( inResults string // Raw represents if the non-enriched results should be used. Raw bool + // debug flag initializes the logger with a debug level + debug bool ) func init() { flag.StringVar(&inResults, "in", "", "the directory where dracon producer/enricher outputs are") flag.BoolVar(&Raw, "raw", false, "if the non-enriched results should be used") + flag.BoolVar(&debug, "debug", false, "turn on debug logging") } // ParseFlags will parse the input flags for the consumer and perform simple validation. func ParseFlags() error { flag.Parse() + + logLevel := slog.LevelInfo + if debug { + logLevel = slog.LevelDebug + } + + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel})).With("scanID", os.Getenv(components.EnvDraconScanID))) if len(inResults) < 1 { return fmt.Errorf("in is undefined") } @@ -41,11 +44,11 @@ func ParseFlags() error { } // LoadToolResponse loads raw results from producers. -func LoadToolResponse() ([]*v1.LaunchToolResponse, error) { +func LoadToolResponse() ([]*draconapiv1.LaunchToolResponse, error) { return putil.LoadToolResponse(inResults) } // LoadEnrichedToolResponse loads enriched results from the enricher. -func LoadEnrichedToolResponse() ([]*v1.EnrichedLaunchToolResponse, error) { +func LoadEnrichedToolResponse() ([]*draconapiv1.EnrichedLaunchToolResponse, error) { return putil.LoadEnrichedToolResponse(inResults) } diff --git a/components/consumers/consumer_test.go b/components/consumers/consumer_test.go index c10fbd3f3..834f1f7ec 100644 --- a/components/consumers/consumer_test.go +++ b/components/consumers/consumer_test.go @@ -6,11 +6,11 @@ import ( "testing" "time" - v1 "github.com/ocurity/dracon/api/proto/v1" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + draconapiv1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components" "github.com/ocurity/dracon/pkg/putil" ) @@ -23,7 +23,7 @@ func TestLoadToolResponse(t *testing.T) { defer require.NoError(t, os.Remove(tmpFile.Name())) - issues := []*v1.Issue{ + issues := []*draconapiv1.Issue{ { Target: "/dracon/source/foobar", Title: "/dracon/source/barfoo", @@ -39,9 +39,9 @@ func TestLoadToolResponse(t *testing.T) { scanTags, err := json.Marshal(tags) assert.NoError(t, err) - require.NoError(t, os.Setenv(EnvDraconStartTime, timestamp)) - require.NoError(t, os.Setenv(EnvDraconScanID, scanID)) - require.NoError(t, os.Setenv(EnvDraconScanTags, string(scanTags))) + require.NoError(t, os.Setenv(components.EnvDraconStartTime, timestamp)) + require.NoError(t, os.Setenv(components.EnvDraconScanID, scanID)) + require.NoError(t, os.Setenv(components.EnvDraconScanTags, string(scanTags))) resultTempDir := tmpFile.Name() resultFile := "test-tool" diff --git a/components/producers/producer.go b/components/producers/producer.go index e179cf2f4..b2442d2f8 100644 --- a/components/producers/producer.go +++ b/components/producers/producer.go @@ -10,13 +10,14 @@ import ( "flag" "fmt" "io" - "log" + "log/slog" "os" "path/filepath" "strings" "time" - v1 "github.com/ocurity/dracon/api/proto/v1" + draconapiv1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components" "github.com/ocurity/dracon/pkg/putil" ) @@ -28,26 +29,28 @@ var ( OutFile string // Append flag will append to the outfile instead of overwriting, useful when there's multiple inresults. Append bool + // debug flag initializes the logger with a debug level + debug bool ) const ( sourceDir = "/workspace/output" - - // EnvDraconStartTime Start Time of Dracon Scan in RFC3339. - EnvDraconStartTime = "DRACON_SCAN_TIME" - // EnvDraconScanID the ID of the dracon scan. - EnvDraconScanID = "DRACON_SCAN_ID" - // EnvDraconScanTags the tags of the dracon scan. - EnvDraconScanTags = "DRACON_SCAN_TAGS" ) // ParseFlags will parse the input flags for the producer and perform simple validation. func ParseFlags() error { flag.StringVar(&InResults, "in", "", "") flag.StringVar(&OutFile, "out", "", "") + flag.BoolVar(&debug, "debug", false, "turn on debug logging") flag.BoolVar(&Append, "append", false, "Append to output file instead of overwriting it") flag.Parse() + logLevel := slog.LevelInfo + if debug { + logLevel = slog.LevelDebug + } + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel})).With("scanID", os.Getenv(components.EnvDraconScanID))) + if InResults == "" { return fmt.Errorf("in is undefined") } @@ -113,28 +116,28 @@ func ParseMultiJSONMessages(in []byte) ([]interface{}, error) { // WriteDraconOut provides a generic method to write the resulting protobuf to the output file. func WriteDraconOut( toolName string, - issues []*v1.Issue, + issues []*draconapiv1.Issue, ) error { source := getSource() - cleanIssues := []*v1.Issue{} + cleanIssues := []*draconapiv1.Issue{} for _, iss := range issues { iss.Description = strings.ReplaceAll(iss.Description, sourceDir, ".") iss.Title = strings.ReplaceAll(iss.Title, sourceDir, ".") iss.Target = strings.ReplaceAll(iss.Target, sourceDir, ".") iss.Source = source cleanIssues = append(cleanIssues, iss) - log.Printf("found issue: %+v\n", iss) + slog.Info(fmt.Sprintf("found issue: %+v\n", iss)) } - scanStartTime := strings.TrimSpace(os.Getenv(EnvDraconStartTime)) + scanStartTime := strings.TrimSpace(os.Getenv(components.EnvDraconStartTime)) if scanStartTime == "" { scanStartTime = time.Now().UTC().Format(time.RFC3339) } - scanUUUID := strings.TrimSpace(os.Getenv(EnvDraconScanID)) - scanTagsStr := strings.TrimSpace(os.Getenv(EnvDraconScanTags)) + scanUUUID := strings.TrimSpace(os.Getenv(components.EnvDraconScanID)) + scanTagsStr := strings.TrimSpace(os.Getenv(components.EnvDraconScanTags)) scanTags := map[string]string{} err := json.Unmarshal([]byte(scanTagsStr), &scanTags) if err != nil { - log.Printf("scan with uuid %s does not have any tags, err:%s", scanUUUID, err) + slog.Error(fmt.Sprintf("scan does not have any tags, err:%s", err)) } stat, err := os.Stat(OutFile) @@ -153,7 +156,7 @@ func getSource() string { dat, err := os.ReadFile(sourceMetaPath) if err != nil { - log.Println(err) + slog.Error(err.Error()) } return strings.TrimSpace(string(dat)) } diff --git a/components/producers/producer_test.go b/components/producers/producer_test.go index 7d384c401..5316b53ce 100644 --- a/components/producers/producer_test.go +++ b/components/producers/producer_test.go @@ -7,6 +7,7 @@ import ( "time" v1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components" "github.com/mitchellh/mapstructure" "github.com/stretchr/testify/assert" @@ -25,8 +26,8 @@ func TestWriteDraconOut(t *testing.T) { baseTime := time.Now().UTC() timestamp := baseTime.Format(time.RFC3339) - require.NoError(t, os.Setenv(EnvDraconStartTime, timestamp)) - require.NoError(t, os.Setenv(EnvDraconScanID, "ab3d3290-cd9f-482c-97dc-ec48bdfcc4de")) + require.NoError(t, os.Setenv(components.EnvDraconStartTime, timestamp)) + require.NoError(t, os.Setenv(components.EnvDraconScanID, "ab3d3290-cd9f-482c-97dc-ec48bdfcc4de")) OutFile = tmpFile.Name() Append = false @@ -66,8 +67,8 @@ func TestWriteDraconOutAppend(t *testing.T) { baseTime := time.Now().UTC() timestamp := baseTime.Format(time.RFC3339) - require.NoError(t, os.Setenv(EnvDraconStartTime, timestamp)) - require.NoError(t, os.Setenv(EnvDraconScanID, "ab3d3290-cd9f-482c-97dc-ec48bdfcc4de")) + require.NoError(t, os.Setenv(components.EnvDraconStartTime, timestamp)) + require.NoError(t, os.Setenv(components.EnvDraconScanID, "ab3d3290-cd9f-482c-97dc-ec48bdfcc4de")) OutFile = tmpFile.Name() Append = true diff --git a/deploy/dracon/chart/templates/kb-ingress.yaml b/deploy/dracon/chart/templates/kb-ingress.yaml index 8d2307d99..823f5a5ee 100644 --- a/deploy/dracon/chart/templates/kb-ingress.yaml +++ b/deploy/dracon/chart/templates/kb-ingress.yaml @@ -1,4 +1,4 @@ -{{ if .Values.kibana.enabled }} +{{ if .Values.kibana.ingress.enabled }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/deploy/dracon/chart/templates/migrations-job.yaml b/deploy/dracon/chart/templates/migrations-job.yaml index f78b84826..50f13713f 100644 --- a/deploy/dracon/chart/templates/migrations-job.yaml +++ b/deploy/dracon/chart/templates/migrations-job.yaml @@ -10,7 +10,7 @@ metadata: # job is considered part of the release. "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-weight": "-5" - "helm.sh/hook-delete-policy": hook-succeeded + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded, hook-failed spec: template: metadata: diff --git a/deploy/dracon/chart/values.yaml b/deploy/dracon/chart/values.yaml index 216c2937e..4db9933dc 100644 --- a/deploy/dracon/chart/values.yaml +++ b/deploy/dracon/chart/values.yaml @@ -48,9 +48,6 @@ postgresql: postgresPassword: "" fullnameOverride: "" -tekton: - enabled: true - # this section controls aspects of managing a database used to store enrichments # the database should use the Postgres dialect. enrichmentDB: diff --git a/deploy/dracon/values/dev.yaml b/deploy/dracon/values/dev.yaml index d779c0b6c..fad78b773 100644 --- a/deploy/dracon/values/dev.yaml +++ b/deploy/dracon/values/dev.yaml @@ -7,6 +7,7 @@ kibana: enabled: true version: 8.3.2 ingress: + enabled: true className: nginx host: kibana.dracon.localhost tlsEnabled: false diff --git a/pkg/k8s/fake/clientset.go b/pkg/k8s/fake/clientset.go index 91aeec105..24caa557e 100644 --- a/pkg/k8s/fake/clientset.go +++ b/pkg/k8s/fake/clientset.go @@ -5,6 +5,7 @@ import ( tektonv1beta1api "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" tektonv1beta1fakeclient "github.com/tektoncd/pipeline/pkg/client/clientset/versioned/typed/pipeline/v1beta1/fake" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/meta/testrestmapper" "k8s.io/apimachinery/pkg/runtime" @@ -39,9 +40,9 @@ func NewSchemeAndCodecs() (*runtime.Scheme, *serializer.CodecFactory, error) { var _ k8s.ClientInterface = (*DraconClientSet)(nil) -// ApplyHookType is the signature of the function that will be called to mock -// an Apply invocation -type ApplyHookType func(clientset ClientsetSubset, obj runtime.Object, namespace string, forceConflicts bool) error +// ApplyHookType is the signature of the function that will be called to +// examine the parameters that the apply function was called with. +type ApplyHookType func(obj runtime.Object, namespace string, forceConflicts bool) error // ClientsetSubset is just the fake clientset struct type ClientsetSubset struct { @@ -52,6 +53,7 @@ type ClientsetSubset struct { // DraconClientSet is a mock implementation of the `k8s.Clientset` type DraconClientSet struct { ClientsetSubset + objectTracker testing.ObjectTracker discovery *fakediscovery.FakeDiscovery ApplyHook ApplyHookType MetaRESTMapper meta.RESTMapper @@ -62,7 +64,7 @@ type DraconClientSet struct { // a correct response for all known types offered by the `k8s.ClientInterface`. func NewFakeTypedClient(objects ...runtime.Object) (DraconClientSet, error) { return NewFakeTypedClientWithApplyHook( - func(_ ClientsetSubset, _ runtime.Object, _ string, _ bool) error { return nil }, + func(_ runtime.Object, _ string, _ bool) error { return nil }, objects..., ) } @@ -96,6 +98,7 @@ func NewFakeTypedClientWithApplyHook(applyHook ApplyHookType, objects ...runtime }) return DraconClientSet{ + objectTracker: objectTracker, discovery: &fakediscovery.FakeDiscovery{ Fake: &fakeCoreK8sClient.Fake, FakedServerVersion: &version.Info{ @@ -118,7 +121,16 @@ func NewFakeTypedClientWithApplyHook(applyHook ApplyHookType, objects ...runtime // Apply mocks the `kubectl apply` func (f DraconClientSet) Apply(_ context.Context, obj runtime.Object, namespace string, forceConflicts bool) error { - return f.ApplyHook(f.ClientsetSubset, obj, namespace, forceConflicts) + if err := f.ApplyHook(obj, namespace, forceConflicts); err != nil { + return err + } + gvk := obj.GetObjectKind().GroupVersionKind() + gvr, _ := meta.UnsafeGuessKindToResource(gvk) + + if err := f.objectTracker.Create(gvr, obj, namespace); k8serrors.IsAlreadyExists(err) { + return f.objectTracker.Update(gvr, obj, namespace) + } + return nil } // RESTMapper returns an instance implementing the `meta.RESTMapper` interface