From 59e69b561da5ed518e39abb220975a69a5b90654 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Thu, 12 Sep 2024 15:34:50 +0530 Subject: [PATCH 1/7] feat: added linear issue tracker support to nuclei (#5601) * feat: added linear issue tracker support to nuclei * misc * feat: fixed unmarshal issues * added linear config --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- cmd/nuclei/issue-tracker-config.yaml | 21 +- go.mod | 1 + go.sum | 2 + pkg/reporting/options.go | 3 + pkg/reporting/reporting.go | 11 + .../trackers/linear/jsonutil/jsonutil.go | 312 ++++++++++++++ pkg/reporting/trackers/linear/linear.go | 404 ++++++++++++++++++ 7 files changed, 753 insertions(+), 1 deletion(-) create mode 100644 pkg/reporting/trackers/linear/jsonutil/jsonutil.go create mode 100644 pkg/reporting/trackers/linear/linear.go diff --git a/cmd/nuclei/issue-tracker-config.yaml b/cmd/nuclei/issue-tracker-config.yaml index 51778eb0ad..b7e0e6dafc 100644 --- a/cmd/nuclei/issue-tracker-config.yaml +++ b/cmd/nuclei/issue-tracker-config.yaml @@ -142,4 +142,23 @@ # # Username for the elasticsearch instance # username: test # # Password is the password for elasticsearch instance -# password: test \ No newline at end of file +# password: test +#linear: +# # api-key is the API key for the linear account +# api-key: "" +# # allow-list sets a tracker level filter to only create issues for templates with +# # these severity labels or tags (does not affect exporters. set those globally) +# deny-list: +# severity: critical +# # deny-list sets a tracker level filter to never create issues for templates with +# # these severity labels or tags (does not affect exporters. set those globally) +# deny-list: +# severity: low +# # team-id is the ID of the team in Linear +# team-id: "" +# # project-id is the ID of the project in Linear +# project-id: "" +# # duplicate-issue-check flag to enable duplicate tracking issue check +# duplicate-issue-check: false +# # open-state-id is the ID of the open state in Linear +# open-state-id: "" diff --git a/go.mod b/go.mod index e24efbbe45..7b5e247bf7 100644 --- a/go.mod +++ b/go.mod @@ -101,6 +101,7 @@ require ( github.com/projectdiscovery/wappalyzergo v0.1.14 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 + github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 github.com/stretchr/testify v1.9.0 github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 diff --git a/go.sum b/go.sum index 77b6bdfd3b..70ee685efe 100644 --- a/go.sum +++ b/go.sum @@ -966,6 +966,8 @@ github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFt github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= diff --git a/pkg/reporting/options.go b/pkg/reporting/options.go index 06a749d658..c5090de014 100644 --- a/pkg/reporting/options.go +++ b/pkg/reporting/options.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear" "github.com/projectdiscovery/retryablehttp-go" ) @@ -29,6 +30,8 @@ type Options struct { Gitea *gitea.Options `yaml:"gitea"` // Jira contains configuration options for Jira Issue Tracker Jira *jira.Options `yaml:"jira"` + // Linear contains configuration options for Linear Issue Tracker + Linear *linear.Options `yaml:"linear"` // MarkdownExporter contains configuration options for Markdown Exporter Module MarkdownExporter *markdown.Options `yaml:"markdown"` // SarifExporter contains configuration options for Sarif Exporter Module diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go index 889f92f3f7..c6a7d63e10 100644 --- a/pkg/reporting/reporting.go +++ b/pkg/reporting/reporting.go @@ -28,6 +28,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" ) @@ -112,6 +113,15 @@ func New(options *Options, db string, doNotDedupe bool) (Client, error) { } client.trackers = append(client.trackers, tracker) } + if options.Linear != nil { + options.Linear.HttpClient = options.HttpClient + options.Linear.OmitRaw = options.OmitRaw + tracker, err := linear.New(options.Linear) + if err != nil { + return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation) + } + client.trackers = append(client.trackers, tracker) + } if options.MarkdownExporter != nil { exporter, err := markdown.New(options.MarkdownExporter) if err != nil { @@ -195,6 +205,7 @@ func CreateConfigIfNotExists() error { GitLab: &gitlab.Options{}, Gitea: &gitea.Options{}, Jira: &jira.Options{}, + Linear: &linear.Options{}, MarkdownExporter: &markdown.Options{}, SarifExporter: &sarif.Options{}, ElasticsearchExporter: &es.Options{}, diff --git a/pkg/reporting/trackers/linear/jsonutil/jsonutil.go b/pkg/reporting/trackers/linear/jsonutil/jsonutil.go new file mode 100644 index 0000000000..cf66b7aa1e --- /dev/null +++ b/pkg/reporting/trackers/linear/jsonutil/jsonutil.go @@ -0,0 +1,312 @@ +// Package jsonutil provides a function for decoding JSON +// into a GraphQL query data structure. +// +// Taken from: https://github.com/shurcooL/graphql/blob/ed46e5a4646634fc16cb07c3b8db389542cc8847/internal/jsonutil/graphql.go +package jsonutil + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "reflect" + "strings" +) + +// UnmarshalGraphQL parses the JSON-encoded GraphQL response data and stores +// the result in the GraphQL query data structure pointed to by v. +// +// The implementation is created on top of the JSON tokenizer available +// in "encoding/json".Decoder. +func UnmarshalGraphQL(data []byte, v any) error { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + err := (&decoder{tokenizer: dec}).Decode(v) + if err != nil { + return err + } + tok, err := dec.Token() + switch err { + case io.EOF: + // Expect to get io.EOF. There shouldn't be any more + // tokens left after we've decoded v successfully. + return nil + case nil: + return fmt.Errorf("invalid token '%v' after top-level value", tok) + default: + return err + } +} + +// decoder is a JSON decoder that performs custom unmarshaling behavior +// for GraphQL query data structures. It's implemented on top of a JSON tokenizer. +type decoder struct { + tokenizer interface { + Token() (json.Token, error) + } + + // Stack of what part of input JSON we're in the middle of - objects, arrays. + parseState []json.Delim + + // Stacks of values where to unmarshal. + // The top of each stack is the reflect.Value where to unmarshal next JSON value. + // + // The reason there's more than one stack is because we might be unmarshaling + // a single JSON value into multiple GraphQL fragments or embedded structs, so + // we keep track of them all. + vs [][]reflect.Value +} + +// Decode decodes a single JSON value from d.tokenizer into v. +func (d *decoder) Decode(v any) error { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr { + return fmt.Errorf("cannot decode into non-pointer %T", v) + } + d.vs = [][]reflect.Value{{rv.Elem()}} + return d.decode() +} + +// decode decodes a single JSON value from d.tokenizer into d.vs. +func (d *decoder) decode() error { + // The loop invariant is that the top of each d.vs stack + // is where we try to unmarshal the next JSON value we see. + for len(d.vs) > 0 { + tok, err := d.tokenizer.Token() + if err == io.EOF { + return errors.New("unexpected end of JSON input") + } else if err != nil { + return err + } + + switch { + + // Are we inside an object and seeing next key (rather than end of object)? + case d.state() == '{' && tok != json.Delim('}'): + key, ok := tok.(string) + if !ok { + return errors.New("unexpected non-key in JSON input") + } + someFieldExist := false + for i := range d.vs { + v := d.vs[i][len(d.vs[i])-1] + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + var f reflect.Value + if v.Kind() == reflect.Struct { + f = fieldByGraphQLName(v, key) + if f.IsValid() { + someFieldExist = true + } + } + d.vs[i] = append(d.vs[i], f) + } + if !someFieldExist { + return fmt.Errorf("struct field for %q doesn't exist in any of %v places to unmarshal", key, len(d.vs)) + } + + // We've just consumed the current token, which was the key. + // Read the next token, which should be the value, and let the rest of code process it. + tok, err = d.tokenizer.Token() + if err == io.EOF { + return errors.New("unexpected end of JSON input") + } else if err != nil { + return err + } + + // Are we inside an array and seeing next value (rather than end of array)? + case d.state() == '[' && tok != json.Delim(']'): + someSliceExist := false + for i := range d.vs { + v := d.vs[i][len(d.vs[i])-1] + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + var f reflect.Value + if v.Kind() == reflect.Slice { + v.Set(reflect.Append(v, reflect.Zero(v.Type().Elem()))) // v = append(v, T). + f = v.Index(v.Len() - 1) + someSliceExist = true + } + d.vs[i] = append(d.vs[i], f) + } + if !someSliceExist { + return fmt.Errorf("slice doesn't exist in any of %v places to unmarshal", len(d.vs)) + } + } + + switch tok := tok.(type) { + case string, json.Number, bool, nil: + // Value. + + for i := range d.vs { + v := d.vs[i][len(d.vs[i])-1] + if !v.IsValid() { + continue + } + err := unmarshalValue(tok, v) + if err != nil { + return err + } + } + d.popAllVs() + + case json.Delim: + switch tok { + case '{': + // Start of object. + + d.pushState(tok) + + frontier := make([]reflect.Value, len(d.vs)) // Places to look for GraphQL fragments/embedded structs. + for i := range d.vs { + v := d.vs[i][len(d.vs[i])-1] + frontier[i] = v + // TODO: Do this recursively or not? Add a test case if needed. + if v.Kind() == reflect.Ptr && v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) // v = new(T). + } + } + // Find GraphQL fragments/embedded structs recursively, adding to frontier + // as new ones are discovered and exploring them further. + for len(frontier) > 0 { + v := frontier[0] + frontier = frontier[1:] + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + if v.Kind() != reflect.Struct { + continue + } + for i := 0; i < v.NumField(); i++ { + if isGraphQLFragment(v.Type().Field(i)) || v.Type().Field(i).Anonymous { + // Add GraphQL fragment or embedded struct. + d.vs = append(d.vs, []reflect.Value{v.Field(i)}) + frontier = append(frontier, v.Field(i)) + } + } + } + case '[': + // Start of array. + + d.pushState(tok) + + for i := range d.vs { + v := d.vs[i][len(d.vs[i])-1] + // TODO: Confirm this is needed, write a test case. + //if v.Kind() == reflect.Ptr && v.IsNil() { + // v.Set(reflect.New(v.Type().Elem())) // v = new(T). + //} + + // Reset slice to empty (in case it had non-zero initial value). + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + if v.Kind() != reflect.Slice { + continue + } + v.Set(reflect.MakeSlice(v.Type(), 0, 0)) // v = make(T, 0, 0). + } + case '}', ']': + // End of object or array. + d.popAllVs() + d.popState() + default: + return errors.New("unexpected delimiter in JSON input") + } + default: + return errors.New("unexpected token in JSON input") + } + } + return nil +} + +// pushState pushes a new parse state s onto the stack. +func (d *decoder) pushState(s json.Delim) { + d.parseState = append(d.parseState, s) +} + +// popState pops a parse state (already obtained) off the stack. +// The stack must be non-empty. +func (d *decoder) popState() { + d.parseState = d.parseState[:len(d.parseState)-1] +} + +// state reports the parse state on top of stack, or 0 if empty. +func (d *decoder) state() json.Delim { + if len(d.parseState) == 0 { + return 0 + } + return d.parseState[len(d.parseState)-1] +} + +// popAllVs pops from all d.vs stacks, keeping only non-empty ones. +func (d *decoder) popAllVs() { + var nonEmpty [][]reflect.Value + for i := range d.vs { + d.vs[i] = d.vs[i][:len(d.vs[i])-1] + if len(d.vs[i]) > 0 { + nonEmpty = append(nonEmpty, d.vs[i]) + } + } + d.vs = nonEmpty +} + +// fieldByGraphQLName returns an exported struct field of struct v +// that matches GraphQL name, or invalid reflect.Value if none found. +func fieldByGraphQLName(v reflect.Value, name string) reflect.Value { + for i := 0; i < v.NumField(); i++ { + if v.Type().Field(i).PkgPath != "" { + // Skip unexported field. + continue + } + if hasGraphQLName(v.Type().Field(i), name) { + return v.Field(i) + } + } + return reflect.Value{} +} + +// hasGraphQLName reports whether struct field f has GraphQL name. +func hasGraphQLName(f reflect.StructField, name string) bool { + value, ok := f.Tag.Lookup("graphql") + if !ok { + // TODO: caseconv package is relatively slow. Optimize it, then consider using it here. + //return caseconv.MixedCapsToLowerCamelCase(f.Name) == name + return strings.EqualFold(f.Name, name) + } + value = strings.TrimSpace(value) // TODO: Parse better. + if strings.HasPrefix(value, "...") { + // GraphQL fragment. It doesn't have a name. + return false + } + // Cut off anything that follows the field name, + // such as field arguments, aliases, directives. + if i := strings.IndexAny(value, "(:@"); i != -1 { + value = value[:i] + } + return strings.TrimSpace(value) == name +} + +// isGraphQLFragment reports whether struct field f is a GraphQL fragment. +func isGraphQLFragment(f reflect.StructField) bool { + value, ok := f.Tag.Lookup("graphql") + if !ok { + return false + } + value = strings.TrimSpace(value) // TODO: Parse better. + return strings.HasPrefix(value, "...") +} + +// unmarshalValue unmarshals JSON value into v. +// v must be addressable and not obtained by the use of unexported +// struct fields, otherwise unmarshalValue will panic. +func unmarshalValue(value json.Token, v reflect.Value) error { + b, err := json.Marshal(value) // TODO: Short-circuit (if profiling says it's worth it). + if err != nil { + return err + } + return json.Unmarshal(b, v.Addr().Interface()) +} diff --git a/pkg/reporting/trackers/linear/linear.go b/pkg/reporting/trackers/linear/linear.go new file mode 100644 index 0000000000..1ad9552f26 --- /dev/null +++ b/pkg/reporting/trackers/linear/linear.go @@ -0,0 +1,404 @@ +package linear + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + "github.com/shurcooL/graphql" + + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v3/pkg/output" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/format" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear/jsonutil" + "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/retryablehttp-go" +) + +// Integration is a client for linear issue tracker integration +type Integration struct { + url string + httpclient *http.Client + options *Options +} + +// Options contains the configuration options for linear issue tracker client +type Options struct { + // APIKey is the API key for linear account. + APIKey string `yaml:"api-key" validate:"required"` + + // AllowList contains a list of allowed events for this tracker + AllowList *filters.Filter `yaml:"allow-list"` + // DenyList contains a list of denied events for this tracker + DenyList *filters.Filter `yaml:"deny-list"` + + // TeamID is the team id for the project + TeamID string `yaml:"team-id"` + // ProjectID is the project id for the project + ProjectID string `yaml:"project-id"` + // DuplicateIssueCheck is a bool to enable duplicate tracking issue check and update the newest + DuplicateIssueCheck bool `yaml:"duplicate-issue-check" default:"false"` + + // OpenStateID is the id of the open state for the project + OpenStateID string `yaml:"open-state-id"` + + HttpClient *retryablehttp.Client `yaml:"-"` + OmitRaw bool `yaml:"-"` +} + +// New creates a new issue tracker integration client based on options. +func New(options *Options) (*Integration, error) { + httpClient := &http.Client{ + Transport: &addHeaderTransport{ + T: http.DefaultTransport, + Key: options.APIKey, + }, + } + + integration := &Integration{ + url: "https://api.linear.app/graphql", + options: options, + httpclient: httpClient, + } + + return integration, nil +} + +// CreateIssue creates an issue in the tracker +func (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIssueResponse, error) { + summary := format.Summary(event) + description := format.CreateReportDescription(event, util.MarkdownFormatter{}, i.options.OmitRaw) + _ = description + + ctx := context.Background() + + var err error + var existingIssue *linearIssue + if i.options.DuplicateIssueCheck { + existingIssue, err = i.findIssueByTitle(ctx, summary) + if err != nil && !errors.Is(err, io.EOF) { + return nil, err + } + } + + if existingIssue == nil { + // Create a new issue + createdIssue, err := i.createIssueLinear(ctx, summary, description, priorityFromSeverity(event.Info.SeverityHolder.Severity)) + if err != nil { + return nil, err + } + return &filters.CreateIssueResponse{ + IssueID: types.ToString(createdIssue.ID), + IssueURL: types.ToString(createdIssue.URL), + }, nil + } else { + if existingIssue.State.Name == "Done" { + // Update the issue state to open + var issueUpdateInput struct { + StateID string `json:"stateId"` + } + issueUpdateInput.StateID = i.options.OpenStateID + variables := map[string]interface{}{ + "issueUpdateInput": issueUpdateInput, + "issueID": types.ToString(existingIssue.ID), + } + var resp struct { + LastSyncID string `json:"lastSyncId"` + } + err := i.doGraphqlRequest(ctx, existingIssueUpdateStateMutation, &resp, variables, "IssueUpdate") + if err != nil { + return nil, fmt.Errorf("error reopening issue %s: %s", existingIssue.ID, err) + } + } + + commentInput := map[string]interface{}{ + "issueId": types.ToString(existingIssue.ID), + "body": description, + } + variables := map[string]interface{}{ + "commentCreateInput": commentInput, + } + var resp struct { + LastSyncID string `json:"lastSyncId"` + } + err := i.doGraphqlRequest(ctx, commentCreateExistingTicketMutation, &resp, variables, "CommentCreate") + if err != nil { + return nil, fmt.Errorf("error commenting on issue %s: %s", existingIssue.ID, err) + } + return &filters.CreateIssueResponse{ + IssueID: types.ToString(existingIssue.ID), + IssueURL: types.ToString(existingIssue.URL), + }, nil + } +} + +func priorityFromSeverity(sev severity.Severity) float64 { + switch sev { + case severity.Critical: + return linearPriorityCritical + case severity.High: + return linearPriorityHigh + case severity.Medium: + return linearPriorityMedium + case severity.Low: + return linearPriorityLow + default: + return linearPriorityNone + } +} + +type createIssueMutation struct { + IssueCreate struct { + Issue struct { + ID graphql.ID + Title graphql.String + Identifier graphql.String + State struct { + Name graphql.String + } + URL graphql.String + } + } +} + +const ( + createIssueGraphQLMutation = `mutation CreateIssue($input: IssueCreateInput!) { + issueCreate(input: $input) { + issue { + id + title + identifier + state { + name + } + url + } + } +}` + + searchExistingTicketQuery = `query ($teamID: ID, $projectID: ID, $title: String!) { + issues(filter: { + title: { eq: $title }, + team: { id: { eq: $teamID } } + project: { id: { eq: $projectID } } + }) { + nodes { + id + title + identifier + state { + name + } + url + } + } +} +` + + existingIssueUpdateStateMutation = `mutation IssueUpdate($issueUpdateInput: IssueUpdateInput!, $issueID: String!) { + issueUpdate(input: $issueUpdateInput, id: $issueID) { + lastSyncId + } +} +` + + commentCreateExistingTicketMutation = `mutation CommentCreate($commentCreateInput: CommentCreateInput!) { + commentCreate(input: $commentCreateInput) { + lastSyncId + } +} +` +) + +func (i *Integration) createIssueLinear(ctx context.Context, title, description string, priority float64) (*linearIssue, error) { + var mutation createIssueMutation + input := map[string]interface{}{ + "title": title, + "description": description, + "priority": priority, + } + if i.options.TeamID != "" { + input["teamId"] = graphql.ID(i.options.TeamID) + } + if i.options.ProjectID != "" { + input["projectId"] = i.options.ProjectID + } + + variables := map[string]interface{}{ + "input": input, + } + + err := i.doGraphqlRequest(ctx, createIssueGraphQLMutation, &mutation, variables, "CreateIssue") + if err != nil { + return nil, err + } + + return &linearIssue{ + ID: mutation.IssueCreate.Issue.ID, + Title: mutation.IssueCreate.Issue.Title, + Identifier: mutation.IssueCreate.Issue.Identifier, + State: struct { + Name graphql.String + }{ + Name: mutation.IssueCreate.Issue.State.Name, + }, + URL: mutation.IssueCreate.Issue.URL, + }, nil +} + +func (i *Integration) findIssueByTitle(ctx context.Context, title string) (*linearIssue, error) { + var query findExistingIssuesSearch + variables := map[string]interface{}{ + "title": graphql.String(title), + } + if i.options.TeamID != "" { + variables["teamId"] = graphql.ID(i.options.TeamID) + } + if i.options.ProjectID != "" { + variables["projectID"] = graphql.ID(i.options.ProjectID) + } + + err := i.doGraphqlRequest(ctx, searchExistingTicketQuery, &query, variables, "") + if err != nil { + return nil, err + } + + if len(query.Issues.Nodes) > 0 { + return &query.Issues.Nodes[0], nil + } + return nil, io.EOF +} + +func (i *Integration) Name() string { + return "linear" +} + +func (i *Integration) CloseIssue(event *output.ResultEvent) error { + // TODO: Unimplemented for now as not used in many places + // and overhead of maintaining our own API for this. + // This is too much code as it is :( + return nil +} + +// ShouldFilter determines if an issue should be logged to this tracker +func (i *Integration) ShouldFilter(event *output.ResultEvent) bool { + if i.options.AllowList != nil && !i.options.AllowList.GetMatch(event) { + return false + } + + if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) { + return false + } + + return true +} + +type linearIssue struct { + ID graphql.ID + Title graphql.String + Identifier graphql.String + State struct { + Name graphql.String + } + URL graphql.String +} + +type findExistingIssuesSearch struct { + Issues struct { + Nodes []linearIssue + } +} + +// Custom transport to add the API key to the header +type addHeaderTransport struct { + T http.RoundTripper + Key string +} + +func (adt *addHeaderTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Add("Authorization", adt.Key) + return adt.T.RoundTrip(req) +} + +const ( + linearPriorityNone = float64(0) + linearPriorityCritical = float64(1) + linearPriorityHigh = float64(2) + linearPriorityMedium = float64(3) + linearPriorityLow = float64(4) +) + +// errors represents the "errors" array in a response from a GraphQL server. +// If returned via error interface, the slice is expected to contain at least 1 element. +// +// Specification: https://spec.graphql.org/October2021/#sec-Errors. +type errorsGraphql []struct { + Message string + Locations []struct { + Line int + Column int + } +} + +// Error implements error interface. +func (e errorsGraphql) Error() string { + return e[0].Message +} + +// do executes a single GraphQL operation. +func (i *Integration) doGraphqlRequest(ctx context.Context, query string, v any, variables map[string]any, operationName string) error { + in := struct { + Query string `json:"query"` + Variables map[string]any `json:"variables,omitempty"` + OperationName string `json:"operationName,omitempty"` + }{ + Query: query, + Variables: variables, + OperationName: operationName, + } + + var buf bytes.Buffer + err := json.NewEncoder(&buf).Encode(in) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, i.url, &buf) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := i.httpclient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("non-200 OK status code: %v body: %q", resp.Status, body) + } + var out struct { + Data *json.RawMessage + Errors errorsGraphql + //Extensions any // Unused. + } + err = json.NewDecoder(resp.Body).Decode(&out) + if err != nil { + return err + } + if out.Data != nil { + err := jsonutil.UnmarshalGraphQL(*out.Data, v) + if err != nil { + return err + } + } + if len(out.Errors) > 0 { + return out.Errors + } + return nil +} From 39f8be2125c5f7e988f0a56884b908cee07f974a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Sep 2024 10:05:56 +0000 Subject: [PATCH 2/7] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.14 to 0.1.18. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.14...v0.1.18) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 7b5e247bf7..b0bbb7570b 100644 --- a/go.mod +++ b/go.mod @@ -98,7 +98,7 @@ require ( github.com/projectdiscovery/uncover v1.0.9 github.com/projectdiscovery/useragent v0.0.65 github.com/projectdiscovery/utils v0.2.8 - github.com/projectdiscovery/wappalyzergo v0.1.14 + github.com/projectdiscovery/wappalyzergo v0.1.18 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 diff --git a/go.sum b/go.sum index 70ee685efe..5f64afe1e8 100644 --- a/go.sum +++ b/go.sum @@ -625,8 +625,6 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kitabisa/go-ci v1.0.2 h1:rqHf8KEbQOxVb998TbqGRo70Z7ol44io7/jLYJUvKp8= -github.com/kitabisa/go-ci v1.0.2/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U= github.com/kitabisa/go-ci v1.0.3 h1:JmIUIvcercRQc/9x/v02ydCCqU4MadSHaNaOF8T2pGA= github.com/kitabisa/go-ci v1.0.3/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -890,8 +888,8 @@ github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19 github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A= github.com/projectdiscovery/utils v0.2.8 h1:++NcCJ+lXEfNBHKBs6q+cWa8JrVS8cYdGcW9jOgZebI= github.com/projectdiscovery/utils v0.2.8/go.mod h1:UYJ8GKbZaezPFRT/cOk01LreQZ1QK0fko1EUnSUiSGU= -github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4= -github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= +github.com/projectdiscovery/wappalyzergo v0.1.18 h1:fFgETis0HcsNE7wREaUPYP45JqIyHgGorJaVp1RH7g4= +github.com/projectdiscovery/wappalyzergo v0.1.18/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= From bd6330f72aa45d1fb8bacc62fe9a41e0aeb92bb5 Mon Sep 17 00:00:00 2001 From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com> Date: Thu, 12 Sep 2024 16:13:49 +0530 Subject: [PATCH 3/7] feat: upload existing scan results (#5603) * feat: upload existing scan results * fix lint test * misc update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- cmd/nuclei/main.go | 11 +++++++++- internal/runner/runner.go | 46 +++++++++++++++++++++++++++++++++++++++ pkg/types/types.go | 2 ++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index de29d6581e..8fe5b29573 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -123,6 +123,13 @@ func main() { runner.ParseOptions(options) + if options.ScanUploadFile != "" { + if err := runner.UploadResultsToCloud(options); err != nil { + gologger.Fatal().Msgf("could not upload scan results to cloud dashboard: %s\n", err) + } + return + } + nucleiRunner, err := runner.New(options) if err != nil { gologger.Fatal().Msgf("Could not create runner: %s\n", err) @@ -420,9 +427,11 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.CreateGroup("cloud", "Cloud", flagSet.DynamicVar(&pdcpauth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"), flagSet.StringVarP(&options.TeamID, "team-id", "tid", _pdcp.TeamIDEnv, "upload scan results to given team id (optional)"), - flagSet.BoolVarP(&options.EnableCloudUpload, "cloud-upload", "cup", false, "upload scan results to pdcp dashboard"), + flagSet.BoolVarP(&options.EnableCloudUpload, "cloud-upload", "cup", false, "upload scan results to pdcp dashboard [DEPRECATED use -dashboard]"), flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to existing scan id (optional)"), flagSet.StringVarP(&options.ScanName, "scan-name", "sname", "", "scan name to set (optional)"), + flagSet.BoolVarP(&options.EnableCloudUpload, "dashboard", "pd", false, "upload / view nuclei results in projectdiscovery cloud (pdcp) UI dashboard"), + flagSet.StringVarP(&options.ScanUploadFile, "dashboard-upload", "pdu", "", "upload / view nuclei results file (jsonl) in projectdiscovery cloud (pdcp) UI dashboard"), ) flagSet.CreateGroup("Authentication", "Authentication", diff --git a/internal/runner/runner.go b/internal/runner/runner.go index bfb2bc64be..b36d8ed584 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -784,6 +784,52 @@ func (r *Runner) SaveResumeConfig(path string) error { return os.WriteFile(path, data, permissionutil.ConfigFilePermission) } +// upload existing scan results to cloud with progress +func UploadResultsToCloud(options *types.Options) error { + h := &pdcpauth.PDCPCredHandler{} + creds, err := h.GetCreds() + if err != nil { + return errors.Wrap(err, "could not get credentials for cloud upload") + } + ctx := context.TODO() + uploadWriter, err := pdcp.NewUploadWriter(ctx, creds) + if err != nil { + return errors.Wrap(err, "could not create upload writer") + } + if options.ScanID != "" { + _ = uploadWriter.SetScanID(options.ScanID) + } + if options.ScanName != "" { + uploadWriter.SetScanName(options.ScanName) + } + if options.TeamID != "" { + uploadWriter.SetTeamID(options.TeamID) + } + + // Open file to count the number of results first + file, err := os.Open(options.ScanUploadFile) + if err != nil { + return errors.Wrap(err, "could not open scan upload file") + } + defer file.Close() + + gologger.Info().Msgf("Uploading scan results to cloud dashboard from %s", options.ScanUploadFile) + dec := json.NewDecoder(file) + for dec.More() { + var r output.ResultEvent + err := dec.Decode(&r) + if err != nil { + gologger.Warning().Msgf("Could not decode jsonl: %s\n", err) + continue + } + if err = uploadWriter.Write(&r); err != nil { + gologger.Warning().Msgf("[%s] failed to upload: %s\n", r.TemplateID, err) + } + } + uploadWriter.Close() + return nil +} + type WalkFunc func(reflect.Value, reflect.StructField) // Walk traverses a struct and executes a callback function on each value in the struct. diff --git a/pkg/types/types.go b/pkg/types/types.go index cab1aacf54..51375e7342 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -384,6 +384,8 @@ type Options struct { ScanID string // ScanName is the name of the scan to be uploaded ScanName string + // ScanUploadFile is the jsonl file to upload scan results to cloud + ScanUploadFile string // TeamID is the team ID to use for cloud upload TeamID string // JsConcurrency is the number of concurrent js routines to run From 72da91a3999838fd32d1bd1703f203be5e143b34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Sep 2024 16:14:27 +0530 Subject: [PATCH 4/7] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.56 to 0.0.58 (#5619) Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.56 to 0.0.58. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.56...v0.0.58) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b0bbb7570b..fc2934614e 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.1.1 github.com/projectdiscovery/fastdialer v0.2.7 - github.com/projectdiscovery/hmap v0.0.57 + github.com/projectdiscovery/hmap v0.0.58 github.com/projectdiscovery/interactsh v1.2.0 github.com/projectdiscovery/rawhttp v0.1.65 github.com/projectdiscovery/retryabledns v1.0.74 diff --git a/go.sum b/go.sum index 5f64afe1e8..b1a6fc5f92 100644 --- a/go.sum +++ b/go.sum @@ -850,8 +850,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E= github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8= -github.com/projectdiscovery/hmap v0.0.57 h1:n45T9Me/fL0fqdVZ+Pi/eVizBguNRCleThqxgzNPLrA= -github.com/projectdiscovery/hmap v0.0.57/go.mod h1:aa7egkpPIBukf5tis936KSddiXfEvWUKYLzrtV3CH8M= +github.com/projectdiscovery/hmap v0.0.58 h1:SoXmnmYS2egPSRgFKgUhHozu8QvPIUKAuoDpuii9jkw= +github.com/projectdiscovery/hmap v0.0.58/go.mod h1:nQTelqkgxU6vuuU+qmQloMrDBxYZMt2TTO0fV86yXN4= github.com/projectdiscovery/httpx v1.6.8 h1:k0Y5g3ue/7QbDP0+LykIxp/VhPDLfau3UEUyuxtP7qE= github.com/projectdiscovery/httpx v1.6.8/go.mod h1:7BIsDxyRwkBjthqFmEajXrA5f3yb4tlVfLmpNdf0ZXA= github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8= From a45e4bbd19b45d3154c8c11ebaf645d3a34c0ebd Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Thu, 12 Sep 2024 14:35:32 +0300 Subject: [PATCH 5/7] move code around (#5626) --- lib/sdk_private.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/sdk_private.go b/lib/sdk_private.go index b65ecf2a2f..cacd9a1ca1 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -108,14 +108,6 @@ func (e *NucleiEngine) init(ctx context.Context) error { return err } - if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { - httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{}) - if err != nil { - return err - } - e.httpClient = httpclient - } - e.parser = templates.NewParser() if sharedInit == nil || protocolstate.ShouldInit() { @@ -126,6 +118,14 @@ func (e *NucleiEngine) init(ctx context.Context) error { _ = protocolinit.Init(e.opts) }) + if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { + httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{}) + if err != nil { + return err + } + e.httpClient = httpclient + } + e.applyRequiredDefaults(ctx) var err error From 2ac9aaf8717fddc4a94e766d0136619e4cf141e3 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 13 Sep 2024 23:45:27 +0530 Subject: [PATCH 6/7] bugfix: fixed misc issues with linear integration (#5630) --- pkg/reporting/trackers/linear/linear.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/reporting/trackers/linear/linear.go b/pkg/reporting/trackers/linear/linear.go index 1ad9552f26..11712039f8 100644 --- a/pkg/reporting/trackers/linear/linear.go +++ b/pkg/reporting/trackers/linear/linear.go @@ -109,7 +109,9 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIss "issueID": types.ToString(existingIssue.ID), } var resp struct { - LastSyncID string `json:"lastSyncId"` + IssueUpdate struct { + LastSyncID int `json:"lastSyncId"` + } } err := i.doGraphqlRequest(ctx, existingIssueUpdateStateMutation, &resp, variables, "IssueUpdate") if err != nil { @@ -125,7 +127,9 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIss "commentCreateInput": commentInput, } var resp struct { - LastSyncID string `json:"lastSyncId"` + CommentCreate struct { + LastSyncID int `json:"lastSyncId"` + } } err := i.doGraphqlRequest(ctx, commentCreateExistingTicketMutation, &resp, variables, "CommentCreate") if err != nil { @@ -387,6 +391,7 @@ func (i *Integration) doGraphqlRequest(ctx context.Context, query string, v any, Errors errorsGraphql //Extensions any // Unused. } + err = json.NewDecoder(resp.Body).Decode(&out) if err != nil { return err From 87e99be4f6ba63db85de76c9e1d2389e653a7956 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Sat, 14 Sep 2024 00:06:08 +0530 Subject: [PATCH 7/7] scan error formatting (#5628) --- pkg/scan/scan_context.go | 27 +++++++++--------------- pkg/tmplexec/exec.go | 43 +++++++++++++++++---------------------- pkg/tmplexec/interface.go | 39 +++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 41 deletions(-) diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go index 45456ddcac..51b98007a6 100644 --- a/pkg/scan/scan_context.go +++ b/pkg/scan/scan_context.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/utils/errkit" ) type ScanContextOption func(*ScanContext) @@ -30,7 +31,7 @@ type ScanContext struct { OnWarning func(string) // unexported state fields - errors []error + error error warnings []string events []*output.InternalWrappedEvent results []*output.ResultEvent @@ -52,8 +53,8 @@ func (s *ScanContext) Context() context.Context { return s.ctx } -func (s *ScanContext) GenerateErrorMessage() string { - return joinErrors(s.errors) +func (s *ScanContext) GenerateErrorMessage() error { + return s.error } // GenerateResult returns final results slice from all events @@ -94,13 +95,16 @@ func (s *ScanContext) LogError(err error) { if err == nil { return } - if s.OnError != nil { s.OnError(err) } - s.errors = append(s.errors, err) + if s.error == nil { + s.error = err + } else { + s.error = errkit.Append(s.error, err) + } - errorMessage := s.GenerateErrorMessage() + errorMessage := s.GenerateErrorMessage().Error() for _, result := range s.results { result.Error = errorMessage @@ -129,14 +133,3 @@ func (s *ScanContext) LogWarning(format string, args ...any) { } } } - -// joinErrors joins multiple errors and returns a single error string -func joinErrors(errors []error) string { - var errorMessages []string - for _, e := range errors { - if e != nil { - errorMessages = append(errorMessages, e.Error()) - } - } - return strings.Join(errorMessages, "; ") -} diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index 4ca9badf70..149deaa4d5 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -20,7 +20,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto" - "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" "github.com/projectdiscovery/utils/errkit" ) @@ -207,7 +206,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { ctx.LogError(errx) if lastMatcherEvent != nil { - lastMatcherEvent.InternalEvent["error"] = tryParseCause(fmt.Errorf("%s", ctx.GenerateErrorMessage())) + lastMatcherEvent.InternalEvent["error"] = getErrorCause(ctx.GenerateErrorMessage()) writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) } @@ -222,7 +221,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { Info: e.options.TemplateInfo, Type: e.getTemplateType(), Host: ctx.Input.MetaInput.Input, - Error: tryParseCause(fmt.Errorf("%s", ctx.GenerateErrorMessage())), + Error: getErrorCause(ctx.GenerateErrorMessage()), }, }, OperatorsResult: &operators.Result{ @@ -235,31 +234,27 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { return executed.Load() || matched.Load(), errx } -// tryParseCause tries to parse the cause of given error +// getErrorCause tries to parse the cause of given error // this is legacy support due to use of errorutil in existing libraries // but this should not be required once all libraries are updated -func tryParseCause(err error) string { - errStr := "" - errX := errkit.FromError(err) - if errX != nil { - var errCause error - - if len(errX.Errors()) > 1 { - errCause = errX.Errors()[0] - } - if errCause == nil { - errCause = errX +func getErrorCause(err error) string { + if err == nil { + return "" + } + errx := errkit.FromError(err) + var cause error + for _, e := range errx.Errors() { + if e != nil && strings.Contains(e.Error(), "context deadline exceeded") { + continue } - - msg := strings.Trim(errCause.Error(), "{} ") - parts := strings.Split(msg, ":") - errCause = errkit.New("%s", parts[len(parts)-1]) - errKind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic).String() - errStr = errCause.Error() - errStr = strings.TrimSpace(strings.Replace(errStr, "errKind="+errKind, "", -1)) + cause = e + break } - - return errStr + if cause == nil { + cause = errkit.Append(errkit.New("could not get error cause"), errx) + } + // parseScanError prettifies the error message and removes everything except the cause + return parseScanError(cause.Error()) } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. diff --git a/pkg/tmplexec/interface.go b/pkg/tmplexec/interface.go index 67f9621116..36f139cc57 100644 --- a/pkg/tmplexec/interface.go +++ b/pkg/tmplexec/interface.go @@ -1,10 +1,15 @@ package tmplexec import ( + "errors" + "regexp" + "strings" + "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto" + "github.com/projectdiscovery/utils/errkit" ) var ( @@ -30,3 +35,37 @@ type TemplateEngine interface { // Name returns name of template engine Name() string } + +var ( + // A temporary fix to remove errKind from error message + // this is because errkit is not used everywhere yet + reNoKind = regexp.MustCompile(`([\[][^][]+[\]]|errKind=[^ ]+) `) +) + +// parseScanError parses given scan error and only returning the cause +// instead of inefficient one +func parseScanError(msg string) string { + if msg == "" { + return "" + } + if strings.HasPrefix(msg, "ReadStatusLine:") { + // last index is actual error (from rawhttp) + parts := strings.Split(msg, ":") + msg = strings.TrimSpace(parts[len(parts)-1]) + } + if strings.Contains(msg, "read ") { + // same here + parts := strings.Split(msg, ":") + msg = strings.TrimSpace(parts[len(parts)-1]) + } + e := errkit.FromError(errors.New(msg)) + for _, err := range e.Errors() { + if err != nil && strings.Contains(err.Error(), "context deadline exceeded") { + continue + } + msg = reNoKind.ReplaceAllString(err.Error(), "") + return msg + } + wrapped := errkit.Append(errkit.New("failed to get error cause"), e).Error() + return reNoKind.ReplaceAllString(wrapped, "") +}