Support Parsing Zeek Logs Collected By Multiple Remote Agents (e.g. S…

…ysmon) (#591)

.gitignore

@@ -6,3 +6,4 @@
 *.exe
 /rita
 /vendor/
+/rita-html-report/

commands/commands.go

@@ -2,7 +2,6 @@ package commands
 
 import (
 	"runtime"
-	"strconv"
 
 	"github.com/activecm/rita/resources"
 	log "github.com/sirupsen/logrus"
@@ -119,6 +118,11 @@ var (
 		Usage: "Use a specific `DELIM` string for non-human-readable output",
 		Value: ",", //default to comma-separated
 	}
+
+	netNamesFlag = cli.BoolFlag{
+		Name:  "network-names, nn",
+		Usage: "Show network names associated with IP addresses. Helps when private IPs are reused across multiple physical networks.",
+	}
 )
 
 // bootstrapCommands simply adds a given command to the allCommands array
@@ -154,11 +158,3 @@ func bootstrapCommands(commands ...cli.Command) {
 func Commands() []cli.Command {
 	return allCommands
 }
-
-//helper functions for formatting floats and integers
-func f(f float64) string {
-	return strconv.FormatFloat(f, 'g', 6, 64)
-}
-func i(i int64) string {
-	return strconv.FormatInt(i, 10)
-}

commands/formatting.go

@@ -0,0 +1,13 @@
+package commands
+
+import (
+	"strconv"
+)
+
+//helper functions for formatting floats and integers
+func f(f float64) string {
+	return strconv.FormatFloat(f, 'g', 6, 64)
+}
+func i(i int64) string {
+	return strconv.FormatInt(i, 10)
+}

commands/reporting.go

@@ -15,6 +15,7 @@ func init() {
 			"If no database is specified, a report will be created for every database.",
 		Flags: []cli.Flag{
 			configFlag,
+			netNamesFlag,
 		},
 		Action: func(c *cli.Context) error {
 			res := resources.InitResources(c.String("config"))
@@ -25,7 +26,7 @@ func init() {
 			} else {
 				databases = res.MetaDB.GetAnalyzedDatabases()
 			}
-			err := reporting.PrintHTML(databases, res)
+			err := reporting.PrintHTML(databases, c.Bool("network-names"), res)
 			if err != nil {
 				return cli.NewExitError(err.Error(), -1)
 			}

commands/show-beacons.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/activecm/rita/pkg/beacon"
 	"github.com/activecm/rita/resources"
-	"github.com/globalsign/mgo/bson"
 	"github.com/olekukonko/tablewriter"
 	"github.com/urfave/cli"
 )
@@ -21,6 +20,7 @@ func init() {
 			humanFlag,
 			configFlag,
 			delimFlag,
+			netNamesFlag,
 		},
 		Action: showBeacons,
 	}
@@ -36,7 +36,7 @@ func showBeacons(c *cli.Context) error {
 	res := resources.InitResources(c.String("config"))
 	res.DB.SelectDB(db)
 
-	data, err := getBeaconResultsView(res, 0)
+	data, err := beacon.Results(res, 0)
 
 	if err != nil {
 		res.Log.Error(err)
@@ -47,77 +47,109 @@ func showBeacons(c *cli.Context) error {
 		return cli.NewExitError("No results were found for "+db, -1)
 	}
 
+	showNetNames := c.Bool("network-names")
+
 	if c.Bool("human-readable") {
-		err := showBeaconsHuman(data)
+		err := showBeaconsHuman(data, showNetNames)
 		if err != nil {
 			return cli.NewExitError(err.Error(), -1)
 		}
 		return nil
 	}
 
-	err = showBeaconsDelim(data, c.String("delimiter"))
+	err = showBeaconsDelim(data, c.String("delimiter"), showNetNames)
 	if err != nil {
 		return cli.NewExitError(err.Error(), -1)
 	}
 	return nil
 }
 
-func showBeaconsHuman(data []beacon.AnalysisView) error {
+func showBeaconsHuman(data []beacon.Result, showNetNames bool) error {
 	table := tablewriter.NewWriter(os.Stdout)
-	table.SetHeader([]string{"Score", "Source IP", "Destination IP",
-		"Connections", "Avg. Bytes", "Intvl Range", "Size Range", "Top Intvl",
-		"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
-		"Size Skew", "Intvl Dispersion", "Size Dispersion"})
+	var headerFields []string
+	if showNetNames {
+		headerFields = []string{
+			"Score", "Source Network", "Destination Network", "Source IP", "Destination IP",
+			"Connections", "Avg. Bytes", "Intvl Range", "Size Range", "Top Intvl",
+			"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
+			"Size Skew", "Intvl Dispersion", "Size Dispersion",
+		}
+	} else {
+		headerFields = []string{
+			"Score", "Source IP", "Destination IP",
+			"Connections", "Avg. Bytes", "Intvl Range", "Size Range", "Top Intvl",
+			"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
+			"Size Skew", "Intvl Dispersion", "Size Dispersion",
+		}
+	}
+
+	table.SetHeader(headerFields)
 
 	for _, d := range data {
-		table.Append(
-			[]string{
-				f(d.Score), d.Src, d.Dst, i(d.Connections), f(d.AvgBytes),
+		var row []string
+		if showNetNames {
+			row = []string{
+				f(d.Score), d.SrcNetworkName, d.DstNetworkName,
+				d.SrcIP, d.DstIP, i(d.Connections), f(d.AvgBytes),
+				i(d.Ts.Range), i(d.Ds.Range), i(d.Ts.Mode), i(d.Ds.Mode),
+				i(d.Ts.ModeCount), i(d.Ds.ModeCount), f(d.Ts.Skew), f(d.Ds.Skew),
+				i(d.Ts.Dispersion), i(d.Ds.Dispersion),
+			}
+		} else {
+			row = []string{
+				f(d.Score), d.SrcIP, d.DstIP, i(d.Connections), f(d.AvgBytes),
 				i(d.Ts.Range), i(d.Ds.Range), i(d.Ts.Mode), i(d.Ds.Mode),
 				i(d.Ts.ModeCount), i(d.Ds.ModeCount), f(d.Ts.Skew), f(d.Ds.Skew),
 				i(d.Ts.Dispersion), i(d.Ds.Dispersion),
-			},
-		)
+			}
+		}
+		table.Append(row)
 	}
 	table.Render()
 	return nil
 }
 
-func showBeaconsDelim(data []beacon.AnalysisView, delim string) error {
-	headers := []string{"Score", "Source IP", "Destination IP",
-		"Connections", "Avg Bytes", "Intvl Range", "Size Range", "Top Intvl",
-		"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
-		"Size Skew", "Intvl Dispersion", "Size Dispersion"}
+func showBeaconsDelim(data []beacon.Result, delim string, showNetNames bool) error {
+	var headerFields []string
+	if showNetNames {
+		headerFields = []string{
+			"Score", "Source Network", "Destination Network", "Source IP", "Destination IP",
+			"Connections", "Avg. Bytes", "Intvl Range", "Size Range", "Top Intvl",
+			"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
+			"Size Skew", "Intvl Dispersion", "Size Dispersion",
+		}
+	} else {
+		headerFields = []string{
+			"Score", "Source IP", "Destination IP",
+			"Connections", "Avg. Bytes", "Intvl Range", "Size Range", "Top Intvl",
+			"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
+			"Size Skew", "Intvl Dispersion", "Size Dispersion",
+		}
+	}
 
 	// Print the headers and analytic values, separated by a delimiter
-	fmt.Println(strings.Join(headers, delim))
+	fmt.Println(strings.Join(headerFields, delim))
 	for _, d := range data {
-		fmt.Println(
-			strings.Join(
-				[]string{
-					f(d.Score), d.Src, d.Dst, i(d.Connections), f(d.AvgBytes),
-					i(d.Ts.Range), i(d.Ds.Range), i(d.Ts.Mode), i(d.Ds.Mode),
-					i(d.Ts.ModeCount), i(d.Ds.ModeCount), f(d.Ts.Skew), f(d.Ds.Skew),
-					i(d.Ts.Dispersion), i(d.Ds.Dispersion),
-				},
-				delim,
-			),
-		)
-	}
-	return nil
-}
-
-//getBeaconResultsView finds beacons greater than a given cutoffScore
-//and links the data from the unique connections table back in to the results
-func getBeaconResultsView(res *resources.Resources, cutoffScore float64) ([]beacon.AnalysisView, error) {
-	ssn := res.DB.Session.Copy()
-	defer ssn.Close()
-
-	var beacons []beacon.AnalysisView
-
-	beaconQuery := bson.M{"score": bson.M{"$gt": cutoffScore}}
 
-	err := ssn.DB(res.DB.GetSelectedDB()).C(res.Config.T.Beacon.BeaconTable).Find(beaconQuery).Sort("-score").All(&beacons)
+		var row []string
+		if showNetNames {
+			row = []string{
+				f(d.Score), d.SrcNetworkName, d.DstNetworkName,
+				d.SrcIP, d.DstIP, i(d.Connections), f(d.AvgBytes),
+				i(d.Ts.Range), i(d.Ds.Range), i(d.Ts.Mode), i(d.Ds.Mode),
+				i(d.Ts.ModeCount), i(d.Ds.ModeCount), f(d.Ts.Skew), f(d.Ds.Skew),
+				i(d.Ts.Dispersion), i(d.Ds.Dispersion),
+			}
+		} else {
+			row = []string{
+				f(d.Score), d.SrcIP, d.DstIP, i(d.Connections), f(d.AvgBytes),
+				i(d.Ts.Range), i(d.Ds.Range), i(d.Ts.Mode), i(d.Ds.Mode),
+				i(d.Ts.ModeCount), i(d.Ds.ModeCount), f(d.Ts.Skew), f(d.Ds.Skew),
+				i(d.Ts.Dispersion), i(d.Ds.Dispersion),
+			}
+		}
 
-	return beacons, err
+		fmt.Println(strings.Join(row, delim))
+	}
+	return nil
 }