Domain filtering and fqdn threshold hotfix (#619)

* fixed a bug with threshold count for fqdn beacons and added always include and never include functionality for domain filtering

config/static.go

@@ -105,9 +105,11 @@ type (
 
 	//FilteringStaticCfg controls address filtering
 	FilteringStaticCfg struct {
-		AlwaysInclude   []string `yaml:"AlwaysInclude" default:"[]"`
-		NeverInclude    []string `yaml:"NeverInclude" default:"[\"0.0.0.0/32\", \"127.0.0.0/8\", \"169.254.0.0/16\", \"224.0.0.0/4\", \"255.255.255.255/32\", \"::1/128\", \"fe80::/10\", \"ff00::/8\"]"`
-		InternalSubnets []string `yaml:"InternalSubnets" default:"[\"10.0.0.0/8\", \"172.16.0.0/12\", \"192.168.0.0/16\"]"`
+		AlwaysInclude       []string `yaml:"AlwaysInclude" default:"[]"`
+		NeverInclude        []string `yaml:"NeverInclude" default:"[\"0.0.0.0/32\", \"127.0.0.0/8\", \"169.254.0.0/16\", \"224.0.0.0/4\", \"255.255.255.255/32\", \"::1/128\", \"fe80::/10\", \"ff00::/8\"]"`
+		InternalSubnets     []string `yaml:"InternalSubnets" default:"[\"10.0.0.0/8\", \"172.16.0.0/12\", \"192.168.0.0/16\"]"`
+		AlwaysIncludeDomain []string `yaml:"AlwaysIncludeDomain" default:"[]"`
+		NeverIncludeDomain  []string `yaml:"NeverIncludeDomain" default:"[]"`
 	}
 
 	//StrobeStaticCfg controls the maximum number of connections between any two given hosts

etc/rita.yaml

@@ -82,7 +82,24 @@ Filtering:
     - 10.0.0.0/8 # Private-Use Networks  RFC 1918
     - 172.16.0.0/12 # Private-Use Networks  RFC 1918
     - 192.168.0.0/16 # Private-Use Networks  RFC 1918
-
+
+  # Example: AlwaysIncludeDomain: ["mydomain.com","*.mydomain.com"]
+  # This functionality overrides the NeverIncludeDomain 
+  # section, making sure that any connection records containing domains
+  # that match this list are kept and not filtered
+  # NOTE: When using wildcards, make sure the added entry is in quotes,
+  #       ie, '*.mydomain.com'. Only subdomain wildcarding 
+  #       (asterisk as the prefix) is supported
+  AlwaysIncludeDomain: []
+
+  # Example: NeverIncludeDomain: ["mydomain.com","*.mydomain.com"]
+  # This functions as a whitelisting setting, and connections involving
+  # ranges entered into this section are filtered out at import time
+  # NOTE: When using wildcards, make sure the added entry is in quotes,
+  #       ie, '*.mydomain.com'. Only subdomain wildcarding 
+  #       (asterisk as the prefix) is supported
+  NeverIncludeDomain: []
+
 BlackListed:
   Enabled: true
   # These are blacklists built into rita-blacklist. Set these to false

parser/filter.go

@@ -1,8 +1,9 @@
 package parser
 
 import (
-	"github.com/activecm/rita/util"
 	"net"
+
+	"github.com/activecm/rita/util"
 )
 
 // filterConnPair returns true if a connection pair is filtered/excluded.
@@ -54,3 +55,29 @@ func (fs *FSImporter) filterConnPair(srcIP net.IP, dstIP net.IP) bool {
 	// default to not filter the connection pair
 	return false
 }
+
+// filterDomain returns true if a domain is filtered/excluded.
+// This is determined by the following rules, in order:
+//   1. Not filtered if domain is on the AlwaysInclude list
+//   2. Filtered if domain is on the NeverInclude list
+//   5. Not filtered in all other cases
+func (fs *FSImporter) filterDomain(domain string) bool {
+	// check if on always included list
+	isDomainIncluded := util.ContainsDomain(fs.alwaysIncludedDomain, domain)
+
+	// check if on never included list
+	isDomainExcluded := util.ContainsDomain(fs.neverIncludedDomain, domain)
+
+	// if either IP is on the AlwaysInclude list, filter does not apply
+	if isDomainIncluded {
+		return false
+	}
+
+	// if either IP is on the NeverInclude list, filter applies
+	if isDomainExcluded {
+		return true
+	}
+
+	// default to not filter the connection pair
+	return false
+}

parser/filter_test.go

@@ -15,6 +15,12 @@ type testCase struct {
 	msg string
 }
 
+type testCaseDomain struct {
+	domain string
+	out    bool
+	msg    string
+}
+
 func TestFilterConnPairWithInternalSubnets(t *testing.T) {
 
 	fsTest := &FSImporter{
@@ -106,3 +112,37 @@ func TestFilterConnPairWithoutInternalSubnets(t *testing.T) {
 		assert.Equal(t, test.out, output, test.msg)
 	}
 }
+
+func TestFilterDomain(t *testing.T) {
+
+	fsTest := &FSImporter{
+		res:                  nil,
+		indexingThreads:      1,
+		parseThreads:         1,
+		internal:             util.ParseSubnets([]string{"10.0.0.0/8"}),
+		alwaysIncluded:       util.ParseSubnets([]string{"10.0.0.1/32", "10.0.0.3/32", "1.1.1.1/32", "1.1.1.3/32"}),
+		neverIncluded:        util.ParseSubnets([]string{"10.0.0.2/32", "10.0.0.3/32", "1.1.1.2/32", "1.1.1.3/32"}),
+		alwaysIncludedDomain: []string{"bad.com", "google.com", "*.myotherdomain.com"},
+		neverIncludedDomain:  []string{"good.com", "google.com", "*.mydomain.com"},
+	}
+
+	// all permutations of being on internal, always, and never lists
+	always := "bad.com"
+	never := "good.com"
+	alwaysNever := "google.com"
+	wildcardNever := "a.mydomain.com"
+	wildcardAlways := "a.myotherdomain.com"
+
+	testCases := []testCaseDomain{
+		testCaseDomain{always, false, "AlwaysIncludeDomain should keep this domain from being filtered"},
+		testCaseDomain{never, true, "NeverIncludeDomain should filter this domain"},
+		testCaseDomain{alwaysNever, false, "NeverIncludeDomain should be ovverriden by AlwaysIncludeDomain"},
+		testCaseDomain{wildcardNever, true, "NeverIncludeDomain wildcard should filter this domain"},
+		testCaseDomain{wildcardAlways, false, "AlwaysIncludeDomain wildcard should keep this domain from being filtered"},
+	}
+
+	for _, test := range testCases {
+		output := fsTest.filterDomain(test.domain)
+		assert.Equal(t, test.out, output, test.msg)
+	}
+}

parser/fsimporter.go

@@ -34,17 +34,19 @@ import (
 type (
 	//FSImporter provides the ability to import bro files from the file system
 	FSImporter struct {
-		res             *resources.Resources
-		importFiles     []string
-		rolling         bool
-		totalChunks     int
-		currentChunk    int
-		indexingThreads int
-		parseThreads    int
-		batchSizeBytes  int64
-		internal        []*net.IPNet
-		alwaysIncluded  []*net.IPNet
-		neverIncluded   []*net.IPNet
+		res                  *resources.Resources
+		importFiles          []string
+		rolling              bool
+		totalChunks          int
+		currentChunk         int
+		indexingThreads      int
+		parseThreads         int
+		batchSizeBytes       int64
+		internal             []*net.IPNet
+		alwaysIncluded       []*net.IPNet
+		neverIncluded        []*net.IPNet
+		alwaysIncludedDomain []string
+		neverIncludedDomain  []string
 	}
 
 	trustedAppTiplet struct {
@@ -58,17 +60,19 @@ type (
 func NewFSImporter(res *resources.Resources,
 	indexingThreads int, parseThreads int, importFiles []string) *FSImporter {
 	return &FSImporter{
-		res:             res,
-		importFiles:     importFiles,
-		rolling:         res.Config.S.Rolling.Rolling,
-		totalChunks:     res.Config.S.Rolling.TotalChunks,
-		currentChunk:    res.Config.S.Rolling.CurrentChunk,
-		indexingThreads: indexingThreads,
-		parseThreads:    parseThreads,
-		batchSizeBytes:  2 * (2 << 30), // 2 gigabytes (used to not run out of memory while importing)
-		internal:        util.ParseSubnets(res.Config.S.Filtering.InternalSubnets),
-		alwaysIncluded:  util.ParseSubnets(res.Config.S.Filtering.AlwaysInclude),
-		neverIncluded:   util.ParseSubnets(res.Config.S.Filtering.NeverInclude),
+		res:                  res,
+		importFiles:          importFiles,
+		rolling:              res.Config.S.Rolling.Rolling,
+		totalChunks:          res.Config.S.Rolling.TotalChunks,
+		currentChunk:         res.Config.S.Rolling.CurrentChunk,
+		indexingThreads:      indexingThreads,
+		parseThreads:         parseThreads,
+		batchSizeBytes:       2 * (2 << 30), // 2 gigabytes (used to not run out of memory while importing)
+		internal:             util.ParseSubnets(res.Config.S.Filtering.InternalSubnets),
+		alwaysIncluded:       util.ParseSubnets(res.Config.S.Filtering.AlwaysInclude),
+		neverIncluded:        util.ParseSubnets(res.Config.S.Filtering.NeverInclude),
+		alwaysIncludedDomain: res.Config.S.Filtering.AlwaysIncludeDomain,
+		neverIncludedDomain:  res.Config.S.Filtering.NeverIncludeDomain,
 	}
 }
 
@@ -541,74 +545,81 @@ func (fs *FSImporter) parseFiles(indexedFiles []*fpt.IndexedFile, parsingThreads
 							domain := parseDNS.Query
 							queryTypeName := parseDNS.QTypeName
 
-							// Safely store the number of conns for this uconn
-							mutex.Lock()
-
-							// increment domain map count for exploded dns
-							explodeddnsMap[domain]++
+							// Run domain through filter to filter out certain domains
+							ignore := fs.filterDomain(domain)
 
-							// initialize the hostname input objects for new hostnames
-							if _, ok := hostnameMap[domain]; !ok {
-								hostnameMap[domain] = &hostname.Input{
-									Host: domain,
-								}
-							}
+							// If domain is not subject to filtering, process
+							if !ignore {
 
-							// extract and store the dns client ip address
-							src := parseDNS.Source
-							srcIP := net.ParseIP(src)
-							srcUniqIP := data.NewUniqueIP(srcIP, parseDNS.AgentUUID, parseDNS.AgentHostname)
-							srcKey := srcUniqIP.MapKey()
+								// Safely store the number of conns for this uconn
+								mutex.Lock()
 
-							hostnameMap[domain].ClientIPs.Insert(srcUniqIP)
+								// increment domain map count for exploded dns
+								explodeddnsMap[domain]++
 
-							if queryTypeName == "A" {
-								answers := parseDNS.Answers
-								for _, answer := range answers {
-									answerIP := net.ParseIP(answer)
-									// Check if answer is an IP address and store it if it is
-									if answerIP != nil {
-										answerUniqIP := data.NewUniqueIP(answerIP, parseDNS.AgentUUID, parseDNS.AgentHostname)
-										hostnameMap[domain].ResolvedIPs.Insert(answerUniqIP)
+								// initialize the hostname input objects for new hostnames
+								if _, ok := hostnameMap[domain]; !ok {
+									hostnameMap[domain] = &hostname.Input{
+										Host: domain,
 									}
 								}
-							}
 
-							// We don't filter out the src ips like we do with the conn
-							// section since a c2 channel running over dns could have an
-							// internal ip to internal ip connection and not having that ip
-							// in the host table is limiting
-
-							// in some of these strings, the empty space will get counted as a domain,
-							// don't add host or increment dns query count if queried domain
-							// is blank or ends in 'in-addr.arpa'
-							if (domain != "") && (!strings.HasSuffix(domain, "in-addr.arpa")) {
-								// Check if host map value is set, because this record could
-								// come before a relevant conns record
-								if _, ok := hostMap[srcKey]; !ok {
-									// create new uconn record with src and dst
-									// Set IsLocalSrc and IsLocalDst fields based on InternalSubnets setting
-									// we only need to do this once if the uconn record does not exist
-									hostMap[srcKey] = &host.Input{
-										Host:    srcUniqIP,
-										IsLocal: util.ContainsIP(fs.GetInternalSubnets(), srcIP),
-										IP4:     util.IsIPv4(src),
-										IP4Bin:  util.IPv4ToBinary(srcIP),
+								// extract and store the dns client ip address
+								src := parseDNS.Source
+								srcIP := net.ParseIP(src)
+								srcUniqIP := data.NewUniqueIP(srcIP, parseDNS.AgentUUID, parseDNS.AgentHostname)
+								srcKey := srcUniqIP.MapKey()
+
+								hostnameMap[domain].ClientIPs.Insert(srcUniqIP)
+
+								if queryTypeName == "A" {
+									answers := parseDNS.Answers
+									for _, answer := range answers {
+										answerIP := net.ParseIP(answer)
+										// Check if answer is an IP address and store it if it is
+										if answerIP != nil {
+											answerUniqIP := data.NewUniqueIP(answerIP, parseDNS.AgentUUID, parseDNS.AgentHostname)
+											hostnameMap[domain].ResolvedIPs.Insert(answerUniqIP)
+										}
 									}
 								}
 
-								// if there are no entries in the dnsquerycount map for this
-								// srcKey, initialize map
-								if hostMap[srcKey].DNSQueryCount == nil {
-									hostMap[srcKey].DNSQueryCount = make(map[string]int64)
+								// We don't filter out the src ips like we do with the conn
+								// section since a c2 channel running over dns could have an
+								// internal ip to internal ip connection and not having that ip
+								// in the host table is limiting
+
+								// in some of these strings, the empty space will get counted as a domain,
+								// don't add host or increment dns query count if queried domain
+								// is blank or ends in 'in-addr.arpa'
+								if (domain != "") && (!strings.HasSuffix(domain, "in-addr.arpa")) {
+									// Check if host map value is set, because this record could
+									// come before a relevant conns record
+									if _, ok := hostMap[srcKey]; !ok {
+										// create new uconn record with src and dst
+										// Set IsLocalSrc and IsLocalDst fields based on InternalSubnets setting
+										// we only need to do this once if the uconn record does not exist
+										hostMap[srcKey] = &host.Input{
+											Host:    srcUniqIP,
+											IsLocal: util.ContainsIP(fs.GetInternalSubnets(), srcIP),
+											IP4:     util.IsIPv4(src),
+											IP4Bin:  util.IPv4ToBinary(srcIP),
+										}
+									}
+
+									// if there are no entries in the dnsquerycount map for this
+									// srcKey, initialize map
+									if hostMap[srcKey].DNSQueryCount == nil {
+										hostMap[srcKey].DNSQueryCount = make(map[string]int64)
+									}
+
+									// increment the dns query count for this domain
+									hostMap[srcKey].DNSQueryCount[domain]++
 								}
 
-								// increment the dns query count for this domain
-								hostMap[srcKey].DNSQueryCount[domain]++
+								mutex.Unlock()
 							}
 
-							mutex.Unlock()
-
 							/// *************************************************************///
 							///                             HTTP                             ///
 							/// *************************************************************///

pkg/beaconfqdn/dissector.go

@@ -102,6 +102,7 @@ func (d *dissector) start() {
 					"tbytes": bson.M{"$sum": "$tbytes"},
 					"icerts": bson.M{"$push": "$icerts"},
 				}},
+				{"$match": bson.M{"count": bson.M{"$gt": d.conf.S.BeaconFQDN.DefaultConnectionThresh}}},
 				{"$unwind": bson.M{
 					"path": "$ts",
 					// by default, $unwind does not output a document if the field value is null,

rita.go

@@ -9,7 +9,7 @@ import (
 	"github.com/urfave/cli"
 )
 
-// Entry point of ai-hunt
+// Entry point of ac-hunt
 func main() {
 	app := cli.NewApp()
 	app.Name = "rita"

util/ip.go

@@ -74,6 +74,40 @@ func ContainsIP(subnets []*net.IPNet, ip net.IP) bool {
 	return false
 }
 
+//ContainsDomain checks if a collection of domains contains an IP
+func ContainsDomain(domains []string, host string) bool {
+
+	for _, entry := range domains {
+
+		// check for wildcard
+		if strings.Contains(entry, "*") {
+
+			// trim asterisk from the wildcard domain
+			wildcardDomain := strings.TrimPrefix(entry, "*")
+
+			//This would match a.mydomain.com, b.mydomain.com etc.,
+			if strings.HasSuffix(host, wildcardDomain) {
+				return true
+			}
+
+			// check match of top domain of wildcard
+			// if a user added *.mydomain.com, this will include mydomain.com
+			// in the filtering
+			wildcardDomain = strings.TrimPrefix(wildcardDomain, ".")
+
+			if host == wildcardDomain {
+				return true
+			}
+		} else { // match on exact
+			if host == entry {
+				return true
+			}
+		}
+
+	}
+	return false
+}
+
 // IsIP returns true if string is a valid IP address
 func IsIP(ip string) bool {
 	if net.ParseIP(ip) != nil {