Merge branch 'tierpod:master' into upstream

.github/workflows/build.yml

@@ -1,11 +1,19 @@
 name: Build
-on: [push]
+on:
+  push:
+  pull_request:
+    paths-ignore:
+      - '**.md'
 
 jobs:
 
   build:
     name: Build
     runs-on: ubuntu-latest
+    # https://github.com/orgs/community/discussions/57827
+    if: github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name !=
+      github.event.pull_request.base.repo.full_name
 
     steps:
 

README.md

@@ -90,6 +90,8 @@ Copy config/config.dist.yaml to config.yaml and change parameters:
 
 * **lookup_addr** (bool): perform reverse lookup? If enabled, may take some time.
 
+* **lookup_limit** (int): limit lookup pool size; must be positive; default = 50
+
 * **merge_reports** (bool): merge multiple similar reports to one?
 
 * **log_debug** (bool): print debug log messages?
@@ -111,6 +113,8 @@ Copy config/config.dist.yaml to config.yaml and change parameters:
   * **delete** (bool): delete email messages from IMAP server if reports are fetched successfully
 
   * **debug** (bool): print debug messages during IMAP session?
+
+  * **security** (str): select encryption between "tls" (default), "starttls" or "plaintext"
 
 **output** section:
 

cmd/dmarc-report-converter/config.go

@@ -13,6 +13,7 @@ type config struct {
 	Input        Input  `yaml:"input"`
 	Output       Output `yaml:"output"`
 	LookupAddr   bool   `yaml:"lookup_addr"`
+	LookupLimit  int    `yaml:"lookup_limit"`
 	MergeReports bool   `yaml:"merge_reports"`
 	LogDebug     bool   `yaml:"log_debug"`
 	LogDatetime  bool   `yaml:"log_datetime"`
@@ -33,6 +34,7 @@ type IMAP struct {
 	Mailbox  string `yaml:"mailbox"`
 	Debug    bool   `yaml:"debug"`
 	Delete   bool   `yaml:"delete"`
+	Security string `yaml:"security"`
 }
 
 // IsConfigured return true if IMAP is configured
@@ -72,10 +74,22 @@ func loadConfig(path string) (*config, error) {
 		return nil, err
 	}
 
+	if c.LookupLimit < 1 {
+		c.LookupLimit = 50
+	}
+
 	if c.Input.Dir == "" {
 		return nil, fmt.Errorf("input.dir is not configured")
 	}
 
+	if c.Input.IMAP.Security == "" {
+		c.Input.IMAP.Security = "tls"
+	}
+
+	if c.Input.IMAP.Security != "tls" && c.Input.IMAP.Security != "starttls" && c.Input.IMAP.Security != "plaintext" {
+		return nil, fmt.Errorf("'input.imap.security' must be one of: tls, starttls, plaintext")
+	}
+
 	// Determine which template is used based upon Output.Format.
 	t := txtTmpl
 	switch c.Output.Format {

cmd/dmarc-report-converter/convert.go

@@ -7,15 +7,11 @@ import (
 	"github.com/tierpod/dmarc-report-converter/pkg/dmarc"
 )
 
-func readParse(r io.Reader, fname string, lookupAddr bool) (dmarc.Report, error) {
-	var report dmarc.Report
-	var err error
-
+// readParse is a helper function that passes r, lookupAddr, and lookupLimit to
+// dmarc.ReadParse.
+//
+// fname is the file name associated with r and is only used for debug logging.
+func readParse(r io.Reader, fname string, lookupAddr bool, lookupLimit int) (dmarc.Report, error) {
 	log.Printf("[DEBUG] parse: %v", fname)
-
-	report, err = dmarc.ReadParse(r, lookupAddr)
-	if err != nil {
-		return dmarc.Report{}, err
-	}
-	return report, nil
+	return dmarc.ReadParse(r, lookupAddr, lookupLimit)
 }

cmd/dmarc-report-converter/files.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"io/fs"
 	"log"
 	"os"
 	"path/filepath"
@@ -85,7 +86,24 @@ func (c *filesConverter) find() error {
 		}
 	}
 
-	files, err := filepath.Glob(filepath.Join(c.cfg.Input.Dir, "*.*"))
+	// Walk Input.Dir for a list of files to process, skipping eml files.
+	var files []string
+	err = filepath.Walk(c.cfg.Input.Dir, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			// Process the root dir but don't recurse
+			if path != c.cfg.Input.Dir {
+				return filepath.SkipDir
+			}
+		} else if filepath.Ext(path) != ".eml" {
+			files = append(files, path)
+		}
+
+		return nil
+	})
 	if err != nil {
 		return err
 	}
@@ -106,7 +124,7 @@ func (c *filesConverter) convert() {
 			continue
 		}
 
-		report, err := readParse(file, f, c.cfg.LookupAddr)
+		report, err := readParse(file, f, c.cfg.LookupAddr, c.cfg.LookupLimit)
 		if err != nil {
 			file.Close()
 			log.Printf("[ERROR] files: %v in file %v, skip", err, f)

cmd/dmarc-report-converter/imap.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -16,7 +17,32 @@ func fetchIMAPAttachments(cfg *config) error {
 	log.Printf("[INFO] imap: connecting to server %v", cfg.Input.IMAP.Server)
 
 	// connect to server
-	c, err := client.DialTLS(cfg.Input.IMAP.Server, nil)
+	var c *client.Client
+	var err error
+	if cfg.Input.IMAP.Security == "plaintext" {
+		log.Printf("[WARN] Without encryption your credentials may be stolen. Be careful!")
+		c, err = client.Dial(cfg.Input.IMAP.Server)
+	} else if cfg.Input.IMAP.Security == "starttls" {
+		// go-imap v2 will replace all the following lines with
+		// c, err = client.DialStartTLS(cfg.Input.IMAP.Server, nil)
+		// and there will be no need to import "errors"
+		c, err = client.Dial(cfg.Input.IMAP.Server)
+		if err == nil {
+			sstRet, sstErr := c.SupportStartTLS()
+			if sstErr != nil {
+				err = sstErr
+			} else if !sstRet {
+				err = errors.New("server doesn't support starttls")
+			} else {
+				err = c.StartTLS(nil)
+			}
+		}
+		if err != nil {
+			c.Logout()
+		}
+	} else {
+		c, err = client.DialTLS(cfg.Input.IMAP.Server, nil)
+	}
 	if err != nil {
 		return err
 	}

cmd/dmarc-report-converter/output.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"encoding/xml"
 	"fmt"
 	"io"
 	"log"
@@ -65,12 +66,28 @@ func (o *output) do(d dmarc.Report) error {
 }
 
 func (o *output) template(d dmarc.Report) error {
-	err := o.cfg.Output.template.Execute(o.w, d)
-	if err != nil {
-		return err
+	data := struct {
+		AssetsPath string
+		Report     dmarc.Report
+
+		// Deprecated
+		XMLName         xml.Name
+		ReportMetadata  dmarc.ReportMetadata
+		PolicyPublished dmarc.PolicyPublished
+		Records         []dmarc.Record
+		MessagesStats   dmarc.MessagesStats
+	}{
+		o.cfg.Output.AssetsPath,
+		d,
+
+		d.XMLName,
+		d.ReportMetadata,
+		d.PolicyPublished,
+		d.Records,
+		d.MessagesStats,
 	}
 
-	return nil
+	return o.cfg.Output.template.Execute(o.w, data)
 }
 
 func (o *output) templateHTML(d dmarc.Report) error {
@@ -79,12 +96,7 @@ func (o *output) templateHTML(d dmarc.Report) error {
 		Report     dmarc.Report
 	}{o.cfg.Output.AssetsPath, d}
 
-	err := o.cfg.Output.template.Execute(o.w, data)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return o.cfg.Output.template.Execute(o.w, data)
 }
 
 func (o *output) json(d dmarc.Report) error {

cmd/dmarc-report-converter/output_test.go

@@ -0,0 +1,158 @@
+package main
+
+import (
+	"bytes"
+	"html/template"
+	"testing"
+	"time"
+
+	"github.com/tierpod/dmarc-report-converter/pkg/dmarc"
+)
+
+func TestExternalTemplate(t *testing.T) {
+	r := `<?xml version="1.0"?>
+<feedback>
+  <report_metadata>
+    <org_name>Org 1</org_name>
+    <email>[email protected]</email>
+    <report_id>1712279633.907274</report_id>
+    <date_range>
+      <begin>1712188800</begin>
+      <end>1712275199</end>
+    </date_range>
+  </report_metadata>
+  <policy_published>
+    <domain>report.test</domain>
+    <adkim>r</adkim>
+    <aspf>r</aspf>
+    <p>none</p>
+    <pct>100</pct>
+  </policy_published>
+  <record>
+    <row>
+      <source_ip>1.2.3.4</source_ip>
+      <count>1</count>
+      <policy_evaluated>
+        <disposition>none</disposition>
+        <dkim>pass</dkim>
+        <spf>fail</spf>
+      </policy_evaluated>
+    </row>
+    <identifiers>
+      <header_from>headerfrom.test</header_from>
+    </identifiers>
+    <auth_results>
+      <dkim>
+        <domain>auth.test</domain>
+        <selector>1000073432</selector>
+        <result>pass</result>
+      </dkim>
+      <dkim>
+        <domain>cust.test</domain>
+        <selector>2020263919</selector>
+        <result>pass</result>
+      </dkim>
+      <spf>
+        <domain>spf.test</domain>
+        <result>pass</result>
+      </spf>
+    </auth_results>
+  </record>
+</feedback>
+`
+
+	tests := map[string]struct {
+		tmpl   string
+		expect string
+	}{
+		".AssetsPath": {
+			`{{ .AssetsPath }}`,
+			`/foo`,
+		},
+		".Report.XMLName.Local": {
+			`{{ .Report.XMLName.Local }}`,
+			`feedback`,
+		},
+		".Report.ReportMetadata": {
+			`{{ .Report.ReportMetadata }}`,
+			`{Org 1 [email protected]  1712279633.907274 {2024-04-04 00:00:00 &#43;0000 UTC 2024-04-04 23:59:59 &#43;0000 UTC}}`,
+		},
+		".Report.PolicyPublished": {
+			`{{ .Report.PolicyPublished }}`,
+			`{report.test r r none  100}`,
+		},
+		".Report.Records": {
+			"{{ range .Report.Records }}\n- {{ . }}\n{{ end -}}",
+			"\n- {{1.2.3.4 1 {none pass fail} } {headerfrom.test } {[{auth.test pass 1000073432} {cust.test pass 2020263919}] [{spf.test pass }]}}\n",
+		},
+		".Report.MessagesStats": {
+			`{{ .Report.MessagesStats }}`,
+			`{1 0 1 100}`,
+		},
+
+		// Deprecated
+		".XMLName.Local": {
+			`{{ .XMLName.Local }}`,
+			`feedback`,
+		},
+		".ReportMetadata": {
+			`{{ .ReportMetadata }}`,
+			`{Org 1 [email protected]  1712279633.907274 {2024-04-04 00:00:00 &#43;0000 UTC 2024-04-04 23:59:59 &#43;0000 UTC}}`,
+		},
+		".PolicyPublished": {
+			`{{ .PolicyPublished }}`,
+			`{report.test r r none  100}`,
+		},
+		".MessagesStats": {
+			`{{ .MessagesStats }}`,
+			`{1 0 1 100}`,
+		},
+	}
+
+	// Set the timezone so that timestamps match regardless of local system time
+	origLocal := time.Local
+
+	loc, err := time.LoadLocation("UTC")
+	if err != nil {
+		t.Errorf("unable to load UTC timezone: %s", err)
+	}
+	time.Local = loc
+	defer func() {
+		// Reset timezone
+		time.Local = origLocal
+	}()
+
+	report, err := dmarc.Parse([]byte(r), false, 1)
+	if err != nil {
+		t.Fatalf("unexpected error parsing XML: %s", err)
+	}
+
+	for name, test := range tests {
+		conf := config{
+			Output: Output{
+				AssetsPath: "/foo",
+				template: template.Must(template.New("report").Funcs(
+					template.FuncMap{
+						"now": func(fmt string) string {
+							return time.Now().Format(fmt)
+						},
+					},
+				).Parse(test.tmpl)),
+			},
+		}
+
+		var buf bytes.Buffer
+		out := newOutput(&conf)
+		out.w = &buf
+
+		err = out.template(report)
+		if err != nil {
+			t.Fatalf("%s: unexpected error building template: %s", name, err)
+		}
+
+		if buf.String() != test.expect {
+			t.Errorf("%s\nWANT:\n%s\nGOT:\n%s", name, test.expect, buf.String())
+		}
+
+	}
+}