diff --git a/config/static.go b/config/static.go index b86a90cc..69ff29b9 100644 --- a/config/static.go +++ b/config/static.go @@ -105,9 +105,11 @@ type ( //FilteringStaticCfg controls address filtering FilteringStaticCfg struct { - AlwaysInclude []string `yaml:"AlwaysInclude" default:"[]"` - NeverInclude []string `yaml:"NeverInclude" default:"[\"0.0.0.0/32\", \"127.0.0.0/8\", \"169.254.0.0/16\", \"224.0.0.0/4\", \"255.255.255.255/32\", \"::1/128\", \"fe80::/10\", \"ff00::/8\"]"` - InternalSubnets []string `yaml:"InternalSubnets" default:"[\"10.0.0.0/8\", \"172.16.0.0/12\", \"192.168.0.0/16\"]"` + AlwaysInclude []string `yaml:"AlwaysInclude" default:"[]"` + NeverInclude []string `yaml:"NeverInclude" default:"[\"0.0.0.0/32\", \"127.0.0.0/8\", \"169.254.0.0/16\", \"224.0.0.0/4\", \"255.255.255.255/32\", \"::1/128\", \"fe80::/10\", \"ff00::/8\"]"` + InternalSubnets []string `yaml:"InternalSubnets" default:"[\"10.0.0.0/8\", \"172.16.0.0/12\", \"192.168.0.0/16\"]"` + AlwaysIncludeDomain []string `yaml:"AlwaysIncludeDomain" default:"[]"` + NeverIncludeDomain []string `yaml:"NeverIncludeDomain" default:"[]"` } //StrobeStaticCfg controls the maximum number of connections between any two given hosts diff --git a/etc/rita.yaml b/etc/rita.yaml index a3395842..53c3dfd4 100644 --- a/etc/rita.yaml +++ b/etc/rita.yaml @@ -82,7 +82,24 @@ Filtering: - 10.0.0.0/8 # Private-Use Networks RFC 1918 - 172.16.0.0/12 # Private-Use Networks RFC 1918 - 192.168.0.0/16 # Private-Use Networks RFC 1918 - + + # Example: AlwaysIncludeDomain: ["mydomain.com","*.mydomain.com"] + # This functionality overrides the NeverIncludeDomain + # section, making sure that any connection records containing domains + # that match this list are kept and not filtered + # NOTE: When using wildcards, make sure the added entry is in quotes, + # ie, '*.mydomain.com'. Only subdomain wildcarding + # (asterisk as the prefix) is supported + AlwaysIncludeDomain: [] + + # Example: NeverIncludeDomain: ["mydomain.com","*.mydomain.com"] + # This functions as a whitelisting setting, and connections involving + # ranges entered into this section are filtered out at import time + # NOTE: When using wildcards, make sure the added entry is in quotes, + # ie, '*.mydomain.com'. Only subdomain wildcarding + # (asterisk as the prefix) is supported + NeverIncludeDomain: [] + BlackListed: Enabled: true # These are blacklists built into rita-blacklist. Set these to false diff --git a/parser/filter.go b/parser/filter.go index da565a98..242137c0 100644 --- a/parser/filter.go +++ b/parser/filter.go @@ -1,8 +1,9 @@ package parser import ( - "github.com/activecm/rita/util" "net" + + "github.com/activecm/rita/util" ) // filterConnPair returns true if a connection pair is filtered/excluded. @@ -54,3 +55,29 @@ func (fs *FSImporter) filterConnPair(srcIP net.IP, dstIP net.IP) bool { // default to not filter the connection pair return false } + +// filterDomain returns true if a domain is filtered/excluded. +// This is determined by the following rules, in order: +// 1. Not filtered if domain is on the AlwaysInclude list +// 2. Filtered if domain is on the NeverInclude list +// 5. Not filtered in all other cases +func (fs *FSImporter) filterDomain(domain string) bool { + // check if on always included list + isDomainIncluded := util.ContainsDomain(fs.alwaysIncludedDomain, domain) + + // check if on never included list + isDomainExcluded := util.ContainsDomain(fs.neverIncludedDomain, domain) + + // if either IP is on the AlwaysInclude list, filter does not apply + if isDomainIncluded { + return false + } + + // if either IP is on the NeverInclude list, filter applies + if isDomainExcluded { + return true + } + + // default to not filter the connection pair + return false +} diff --git a/parser/filter_test.go b/parser/filter_test.go index 2447df91..97f4f175 100644 --- a/parser/filter_test.go +++ b/parser/filter_test.go @@ -15,6 +15,12 @@ type testCase struct { msg string } +type testCaseDomain struct { + domain string + out bool + msg string +} + func TestFilterConnPairWithInternalSubnets(t *testing.T) { fsTest := &FSImporter{ @@ -106,3 +112,37 @@ func TestFilterConnPairWithoutInternalSubnets(t *testing.T) { assert.Equal(t, test.out, output, test.msg) } } + +func TestFilterDomain(t *testing.T) { + + fsTest := &FSImporter{ + res: nil, + indexingThreads: 1, + parseThreads: 1, + internal: util.ParseSubnets([]string{"10.0.0.0/8"}), + alwaysIncluded: util.ParseSubnets([]string{"10.0.0.1/32", "10.0.0.3/32", "1.1.1.1/32", "1.1.1.3/32"}), + neverIncluded: util.ParseSubnets([]string{"10.0.0.2/32", "10.0.0.3/32", "1.1.1.2/32", "1.1.1.3/32"}), + alwaysIncludedDomain: []string{"bad.com", "google.com", "*.myotherdomain.com"}, + neverIncludedDomain: []string{"good.com", "google.com", "*.mydomain.com"}, + } + + // all permutations of being on internal, always, and never lists + always := "bad.com" + never := "good.com" + alwaysNever := "google.com" + wildcardNever := "a.mydomain.com" + wildcardAlways := "a.myotherdomain.com" + + testCases := []testCaseDomain{ + testCaseDomain{always, false, "AlwaysIncludeDomain should keep this domain from being filtered"}, + testCaseDomain{never, true, "NeverIncludeDomain should filter this domain"}, + testCaseDomain{alwaysNever, false, "NeverIncludeDomain should be ovverriden by AlwaysIncludeDomain"}, + testCaseDomain{wildcardNever, true, "NeverIncludeDomain wildcard should filter this domain"}, + testCaseDomain{wildcardAlways, false, "AlwaysIncludeDomain wildcard should keep this domain from being filtered"}, + } + + for _, test := range testCases { + output := fsTest.filterDomain(test.domain) + assert.Equal(t, test.out, output, test.msg) + } +} diff --git a/parser/fsimporter.go b/parser/fsimporter.go index 03d3d2d7..9929d731 100644 --- a/parser/fsimporter.go +++ b/parser/fsimporter.go @@ -34,17 +34,19 @@ import ( type ( //FSImporter provides the ability to import bro files from the file system FSImporter struct { - res *resources.Resources - importFiles []string - rolling bool - totalChunks int - currentChunk int - indexingThreads int - parseThreads int - batchSizeBytes int64 - internal []*net.IPNet - alwaysIncluded []*net.IPNet - neverIncluded []*net.IPNet + res *resources.Resources + importFiles []string + rolling bool + totalChunks int + currentChunk int + indexingThreads int + parseThreads int + batchSizeBytes int64 + internal []*net.IPNet + alwaysIncluded []*net.IPNet + neverIncluded []*net.IPNet + alwaysIncludedDomain []string + neverIncludedDomain []string } trustedAppTiplet struct { @@ -58,17 +60,19 @@ type ( func NewFSImporter(res *resources.Resources, indexingThreads int, parseThreads int, importFiles []string) *FSImporter { return &FSImporter{ - res: res, - importFiles: importFiles, - rolling: res.Config.S.Rolling.Rolling, - totalChunks: res.Config.S.Rolling.TotalChunks, - currentChunk: res.Config.S.Rolling.CurrentChunk, - indexingThreads: indexingThreads, - parseThreads: parseThreads, - batchSizeBytes: 2 * (2 << 30), // 2 gigabytes (used to not run out of memory while importing) - internal: util.ParseSubnets(res.Config.S.Filtering.InternalSubnets), - alwaysIncluded: util.ParseSubnets(res.Config.S.Filtering.AlwaysInclude), - neverIncluded: util.ParseSubnets(res.Config.S.Filtering.NeverInclude), + res: res, + importFiles: importFiles, + rolling: res.Config.S.Rolling.Rolling, + totalChunks: res.Config.S.Rolling.TotalChunks, + currentChunk: res.Config.S.Rolling.CurrentChunk, + indexingThreads: indexingThreads, + parseThreads: parseThreads, + batchSizeBytes: 2 * (2 << 30), // 2 gigabytes (used to not run out of memory while importing) + internal: util.ParseSubnets(res.Config.S.Filtering.InternalSubnets), + alwaysIncluded: util.ParseSubnets(res.Config.S.Filtering.AlwaysInclude), + neverIncluded: util.ParseSubnets(res.Config.S.Filtering.NeverInclude), + alwaysIncludedDomain: res.Config.S.Filtering.AlwaysIncludeDomain, + neverIncludedDomain: res.Config.S.Filtering.NeverIncludeDomain, } } @@ -541,74 +545,81 @@ func (fs *FSImporter) parseFiles(indexedFiles []*fpt.IndexedFile, parsingThreads domain := parseDNS.Query queryTypeName := parseDNS.QTypeName - // Safely store the number of conns for this uconn - mutex.Lock() - - // increment domain map count for exploded dns - explodeddnsMap[domain]++ + // Run domain through filter to filter out certain domains + ignore := fs.filterDomain(domain) - // initialize the hostname input objects for new hostnames - if _, ok := hostnameMap[domain]; !ok { - hostnameMap[domain] = &hostname.Input{ - Host: domain, - } - } + // If domain is not subject to filtering, process + if !ignore { - // extract and store the dns client ip address - src := parseDNS.Source - srcIP := net.ParseIP(src) - srcUniqIP := data.NewUniqueIP(srcIP, parseDNS.AgentUUID, parseDNS.AgentHostname) - srcKey := srcUniqIP.MapKey() + // Safely store the number of conns for this uconn + mutex.Lock() - hostnameMap[domain].ClientIPs.Insert(srcUniqIP) + // increment domain map count for exploded dns + explodeddnsMap[domain]++ - if queryTypeName == "A" { - answers := parseDNS.Answers - for _, answer := range answers { - answerIP := net.ParseIP(answer) - // Check if answer is an IP address and store it if it is - if answerIP != nil { - answerUniqIP := data.NewUniqueIP(answerIP, parseDNS.AgentUUID, parseDNS.AgentHostname) - hostnameMap[domain].ResolvedIPs.Insert(answerUniqIP) + // initialize the hostname input objects for new hostnames + if _, ok := hostnameMap[domain]; !ok { + hostnameMap[domain] = &hostname.Input{ + Host: domain, } } - } - // We don't filter out the src ips like we do with the conn - // section since a c2 channel running over dns could have an - // internal ip to internal ip connection and not having that ip - // in the host table is limiting - - // in some of these strings, the empty space will get counted as a domain, - // don't add host or increment dns query count if queried domain - // is blank or ends in 'in-addr.arpa' - if (domain != "") && (!strings.HasSuffix(domain, "in-addr.arpa")) { - // Check if host map value is set, because this record could - // come before a relevant conns record - if _, ok := hostMap[srcKey]; !ok { - // create new uconn record with src and dst - // Set IsLocalSrc and IsLocalDst fields based on InternalSubnets setting - // we only need to do this once if the uconn record does not exist - hostMap[srcKey] = &host.Input{ - Host: srcUniqIP, - IsLocal: util.ContainsIP(fs.GetInternalSubnets(), srcIP), - IP4: util.IsIPv4(src), - IP4Bin: util.IPv4ToBinary(srcIP), + // extract and store the dns client ip address + src := parseDNS.Source + srcIP := net.ParseIP(src) + srcUniqIP := data.NewUniqueIP(srcIP, parseDNS.AgentUUID, parseDNS.AgentHostname) + srcKey := srcUniqIP.MapKey() + + hostnameMap[domain].ClientIPs.Insert(srcUniqIP) + + if queryTypeName == "A" { + answers := parseDNS.Answers + for _, answer := range answers { + answerIP := net.ParseIP(answer) + // Check if answer is an IP address and store it if it is + if answerIP != nil { + answerUniqIP := data.NewUniqueIP(answerIP, parseDNS.AgentUUID, parseDNS.AgentHostname) + hostnameMap[domain].ResolvedIPs.Insert(answerUniqIP) + } } } - // if there are no entries in the dnsquerycount map for this - // srcKey, initialize map - if hostMap[srcKey].DNSQueryCount == nil { - hostMap[srcKey].DNSQueryCount = make(map[string]int64) + // We don't filter out the src ips like we do with the conn + // section since a c2 channel running over dns could have an + // internal ip to internal ip connection and not having that ip + // in the host table is limiting + + // in some of these strings, the empty space will get counted as a domain, + // don't add host or increment dns query count if queried domain + // is blank or ends in 'in-addr.arpa' + if (domain != "") && (!strings.HasSuffix(domain, "in-addr.arpa")) { + // Check if host map value is set, because this record could + // come before a relevant conns record + if _, ok := hostMap[srcKey]; !ok { + // create new uconn record with src and dst + // Set IsLocalSrc and IsLocalDst fields based on InternalSubnets setting + // we only need to do this once if the uconn record does not exist + hostMap[srcKey] = &host.Input{ + Host: srcUniqIP, + IsLocal: util.ContainsIP(fs.GetInternalSubnets(), srcIP), + IP4: util.IsIPv4(src), + IP4Bin: util.IPv4ToBinary(srcIP), + } + } + + // if there are no entries in the dnsquerycount map for this + // srcKey, initialize map + if hostMap[srcKey].DNSQueryCount == nil { + hostMap[srcKey].DNSQueryCount = make(map[string]int64) + } + + // increment the dns query count for this domain + hostMap[srcKey].DNSQueryCount[domain]++ } - // increment the dns query count for this domain - hostMap[srcKey].DNSQueryCount[domain]++ + mutex.Unlock() } - mutex.Unlock() - /// *************************************************************/// /// HTTP /// /// *************************************************************/// diff --git a/pkg/beaconfqdn/dissector.go b/pkg/beaconfqdn/dissector.go index ed26672a..58f495e1 100644 --- a/pkg/beaconfqdn/dissector.go +++ b/pkg/beaconfqdn/dissector.go @@ -102,6 +102,7 @@ func (d *dissector) start() { "tbytes": bson.M{"$sum": "$tbytes"}, "icerts": bson.M{"$push": "$icerts"}, }}, + {"$match": bson.M{"count": bson.M{"$gt": d.conf.S.BeaconFQDN.DefaultConnectionThresh}}}, {"$unwind": bson.M{ "path": "$ts", // by default, $unwind does not output a document if the field value is null, diff --git a/rita.go b/rita.go index 29489f57..081ea971 100644 --- a/rita.go +++ b/rita.go @@ -9,7 +9,7 @@ import ( "github.com/urfave/cli" ) -// Entry point of ai-hunt +// Entry point of ac-hunt func main() { app := cli.NewApp() app.Name = "rita" diff --git a/util/ip.go b/util/ip.go index e8b3d4af..13b63686 100644 --- a/util/ip.go +++ b/util/ip.go @@ -74,6 +74,40 @@ func ContainsIP(subnets []*net.IPNet, ip net.IP) bool { return false } +//ContainsDomain checks if a collection of domains contains an IP +func ContainsDomain(domains []string, host string) bool { + + for _, entry := range domains { + + // check for wildcard + if strings.Contains(entry, "*") { + + // trim asterisk from the wildcard domain + wildcardDomain := strings.TrimPrefix(entry, "*") + + //This would match a.mydomain.com, b.mydomain.com etc., + if strings.HasSuffix(host, wildcardDomain) { + return true + } + + // check match of top domain of wildcard + // if a user added *.mydomain.com, this will include mydomain.com + // in the filtering + wildcardDomain = strings.TrimPrefix(wildcardDomain, ".") + + if host == wildcardDomain { + return true + } + } else { // match on exact + if host == entry { + return true + } + } + + } + return false +} + // IsIP returns true if string is a valid IP address func IsIP(ip string) bool { if net.ParseIP(ip) != nil {