Beacon Detection by FQDN (#604)

* initial poc layout and mongo query setup * added dissector and sorter portions of the beaconing process to the fqdn beacons package * beaconing by fqdn table creation functionality finished * Add ResolvedIPs to data sent to sorter * Working MVP for FQDN Beaconing Detection * moved datatypes/functions only used in fqdn beacons out of the data package * Added show-beacons-fqdn command and updated supporting files * initial all sources modification * stashin changes * Changed module fields to beaconsFQDN * removed mongodb_test.go * Addid static config for BeaconFQDN * Fixed name in static config, added flag in fsimporter * Filter out strobes for now * Fixed reporting output for dst/src IPs * updated strobes flagging and storage for fqdn beacons * Added BeaconFQDN to rita.yaml * Updated a comment * Removed todo comment for config file update * Move countAndRemoveConsecutiveDuplicates to the analyzer where it is used in beacons and beaconsFQDN, update comments in beacon and beaconfqdn sorter, add typing info to FqdnInput.DstBSONList * added initial additional worker functionality for beacons fqdn * fixed closing bug in new worker * Fix bug where dissector query was matching on src_network_name, clean up unique ip data structures. Co-authored-by: lisaSW <[email protected]> Co-authored-by: Logan Lembke <[email protected]> Co-authored-by: lisaSW <[email protected]> Co-authored-by: Logan L <[email protected]>
activecm · Mar 4, 2021 · 7c72d56 · 7c72d56
1 parent b842e05
commit 7c72d56
Show file tree

Hide file tree

Showing 22 changed files with 1,770 additions and 189 deletions.
diff --git a/commands/show-beacons-fqdn.go b/commands/show-beacons-fqdn.go
@@ -0,0 +1,156 @@
+package commands
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/activecm/rita/pkg/beaconfqdn"
+	"github.com/activecm/rita/resources"
+	"github.com/olekukonko/tablewriter"
+	"github.com/urfave/cli"
+)
+
+func init() {
+	command := cli.Command{
+		Name:      "show-beacons-fqdn",
+		Usage:     "Print hosts which show signs of C2 software (FQDN Analysis)",
+		ArgsUsage: "<database>",
+		Flags: []cli.Flag{
+			humanFlag,
+			configFlag,
+			delimFlag,
+			netNamesFlag,
+		},
+		Action: showBeaconsFQDN,
+	}
+
+	bootstrapCommands(command)
+}
+
+func showBeaconsFQDN(c *cli.Context) error {
+	db := c.Args().Get(0)
+	if db == "" {
+		return cli.NewExitError("Specify a database", -1)
+	}
+	res := resources.InitResources(c.String("config"))
+	res.DB.SelectDB(db)
+
+	data, err := beaconfqdn.Results(res, 0)
+
+	if err != nil {
+		res.Log.Error(err)
+		return cli.NewExitError(err, -1)
+	}
+
+	if !(len(data) > 0) {
+		return cli.NewExitError("No results were found for "+db, -1)
+	}
+
+	showNetNames := c.Bool("network-names")
+
+	if c.Bool("human-readable") {
+		err := showBeaconsFQDNHuman(data, showNetNames)
+		if err != nil {
+			return cli.NewExitError(err.Error(), -1)
+		}
+		return nil
+	}
+
+	err = showBeaconsFQDNDelim(data, c.String("delimiter"), showNetNames)
+	if err != nil {
+		return cli.NewExitError(err.Error(), -1)
+	}
+	return nil
+}
+
+func showBeaconsFQDNHuman(data []beaconfqdn.Result, showNetNames bool) error {
+	table := tablewriter.NewWriter(os.Stdout)
+	var headerFields []string
+	if showNetNames {
+		headerFields = []string{
+			"Score", "Source Network", "Source IP", "FQDN", "Resolved IPs",
+			"Connections", "Avg. Bytes", "Intvl Range", "Size Range", "Top Intvl",
+			"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
+			"Size Skew", "Intvl Dispersion", "Size Dispersion",
+		}
+	} else {
+		headerFields = []string{
+			"Score", "Source IP", "FQDN", "Resolved IPs",
+			"Connections", "Avg. Bytes", "Intvl Range", "Size Range", "Top Intvl",
+			"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
+			"Size Skew", "Intvl Dispersion", "Size Dispersion",
+		}
+	}
+
+	table.SetHeader(headerFields)
+
+	for _, d := range data {
+		var row []string
+
+		if showNetNames {
+			row = []string{
+				f(d.Score), d.SrcNetworkName,
+				d.SrcIP, d.FQDN, i(d.Connections), f(d.AvgBytes),
+				i(d.Ts.Range), i(d.Ds.Range), i(d.Ts.Mode), i(d.Ds.Mode),
+				i(d.Ts.ModeCount), i(d.Ds.ModeCount), f(d.Ts.Skew), f(d.Ds.Skew),
+				i(d.Ts.Dispersion), i(d.Ds.Dispersion),
+			}
+		} else {
+			row = []string{
+				f(d.Score), d.SrcIP, d.FQDN, i(d.Connections), f(d.AvgBytes),
+				i(d.Ts.Range), i(d.Ds.Range), i(d.Ts.Mode), i(d.Ds.Mode),
+				i(d.Ts.ModeCount), i(d.Ds.ModeCount), f(d.Ts.Skew), f(d.Ds.Skew),
+				i(d.Ts.Dispersion), i(d.Ds.Dispersion),
+			}
+		}
+		table.Append(row)
+	}
+	table.Render()
+	return nil
+}
+
+func showBeaconsFQDNDelim(data []beaconfqdn.Result, delim string, showNetNames bool) error {
+	var headerFields []string
+	if showNetNames {
+		headerFields = []string{
+			"Score", "Source Network", "Source IP", "FQDN",
+			"Connections", "Avg. Bytes", "Intvl Range", "Size Range", "Top Intvl",
+			"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
+			"Size Skew", "Intvl Dispersion", "Size Dispersion",
+		}
+	} else {
+		headerFields = []string{
+			"Score", "Source IP", "FQDN",
+			"Connections", "Avg. Bytes", "Intvl Range", "Size Range", "Top Intvl",
+			"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
+			"Size Skew", "Intvl Dispersion", "Size Dispersion",
+		}
+	}
+
+	// Print the headers and analytic values, separated by a delimiter
+	fmt.Println(strings.Join(headerFields, delim))
+	for _, d := range data {
+
+		var row []string
+		if showNetNames {
+			row = []string{
+				f(d.Score), d.SrcNetworkName,
+				d.SrcIP, d.FQDN, i(d.Connections), f(d.AvgBytes),
+				i(d.Ts.Range), i(d.Ds.Range), i(d.Ts.Mode), i(d.Ds.Mode),
+				i(d.Ts.ModeCount), i(d.Ds.ModeCount), f(d.Ts.Skew), f(d.Ds.Skew),
+				i(d.Ts.Dispersion), i(d.Ds.Dispersion),
+			}
+		} else {
+			row = []string{
+				f(d.Score), d.SrcIP, d.FQDN, i(d.Connections), f(d.AvgBytes),
+				i(d.Ts.Range), i(d.Ds.Range), i(d.Ts.Mode), i(d.Ds.Mode),
+				i(d.Ts.ModeCount), i(d.Ds.ModeCount), f(d.Ts.Skew), f(d.Ds.Skew),
+				i(d.Ts.Dispersion), i(d.Ds.Dispersion),
+			}
+		}
+
+		fmt.Println(strings.Join(row, delim))
+	}
+	return nil
+}
diff --git a/config/static.go b/config/static.go
@@ -19,6 +19,7 @@ type (
 		Log          LogStaticCfg         `yaml:"LogConfig"`
 		Blacklisted  BlacklistedStaticCfg `yaml:"BlackListed"`
 		Beacon       BeaconStaticCfg      `yaml:"Beacon"`
+		BeaconFQDN   BeaconFQDNStaticCfg  `yaml:"BeaconFQDN"`
 		DNS          DNSStaticCfg         `yaml:"DNS"`
 		UserAgent    UserAgentStaticCfg   `yaml:"UserAgent"`
 		Bro          BroStaticCfg         `yaml:"Bro"` // kept in for MetaDB backwards compatibility
@@ -86,6 +87,12 @@ type (
 		DefaultConnectionThresh int  `yaml:"DefaultConnectionThresh" default:"20"`
 	}
 
+	//BeaconFQDNStaticCfg is used to control the beaconing analysis module
+	BeaconFQDNStaticCfg struct {
+		Enabled                 bool `yaml:"Enabled" default:"true"`
+		DefaultConnectionThresh int  `yaml:"DefaultConnectionThresh" default:"20"`
+	}
+
 	//DNSStaticCfg is used to control the DNS analysis module
 	DNSStaticCfg struct {
 		Enabled bool `yaml:"Enabled" default:"true"`

diff --git a/config/tables.go b/config/tables.go
@@ -7,6 +7,7 @@ type (
 		DNS       DNSTableCfg
 		Structure StructureTableCfg
 		Beacon    BeaconTableCfg
+		BeaconFQDN    BeaconFQDNTableCfg
 		UserAgent UserAgentTableCfg
 		Cert      CertificateTableCfg
 		Meta      MetaTableCfg
@@ -38,6 +39,11 @@ type (
 		BeaconTable string `default:"beacon"`
 	}
 
+	//BeaconFQDNTableCfg is used to control the beaconing analysis module
+	BeaconFQDNTableCfg struct {
+		BeaconFQDNTable string `default:"beaconFQDN"`
+	}
+
 	//UserAgentTableCfg is used to control the useragent analysis module
 	UserAgentTableCfg struct {
 		UserAgentTable string `default:"useragent"`