From 82f3bc75bfc587ed14b865a5c7b458ddce0066cb Mon Sep 17 00:00:00 2001 From: Xiaming Chen Date: Fri, 25 Dec 2015 01:12:39 +0800 Subject: [PATCH] Add new types to cleansed syslog: CODE_USERROAM, CODE_NEW_DEV --- .gitignore | 1 + .../sjtu/omnilab/odh/rawfilter/WIFICode.java | 2 + .../omnilab/odh/rawfilter/WIFILogFilter.java | 82 ++++++++++++++++--- etlers/WifiToolkit/test500010 | 16 ++++ etlers/WifiToolkit/test522035 | 17 ++++ 5 files changed, 106 insertions(+), 12 deletions(-) create mode 100644 etlers/WifiToolkit/test500010 create mode 100644 etlers/WifiToolkit/test522035 diff --git a/.gitignore b/.gitignore index 25ec11e..5b2e3d3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.pyc *.DS_Store target/ .idea/ diff --git a/etlers/WifiToolkit/src/main/java/cn/edu/sjtu/omnilab/odh/rawfilter/WIFICode.java b/etlers/WifiToolkit/src/main/java/cn/edu/sjtu/omnilab/odh/rawfilter/WIFICode.java index 613f5ce..9116a02 100644 --- a/etlers/WifiToolkit/src/main/java/cn/edu/sjtu/omnilab/odh/rawfilter/WIFICode.java +++ b/etlers/WifiToolkit/src/main/java/cn/edu/sjtu/omnilab/odh/rawfilter/WIFICode.java @@ -9,4 +9,6 @@ public class WIFICode { public static final int UserAuth = 4; public static final int IPAllocation = 5; public static final int IPRecycle = 6; + public static final int UserRoam = 7; + public static final int NewDev = 8; } diff --git a/etlers/WifiToolkit/src/main/java/cn/edu/sjtu/omnilab/odh/rawfilter/WIFILogFilter.java b/etlers/WifiToolkit/src/main/java/cn/edu/sjtu/omnilab/odh/rawfilter/WIFILogFilter.java index fe58fe8..49081aa 100644 --- a/etlers/WifiToolkit/src/main/java/cn/edu/sjtu/omnilab/odh/rawfilter/WIFILogFilter.java +++ b/etlers/WifiToolkit/src/main/java/cn/edu/sjtu/omnilab/odh/rawfilter/WIFILogFilter.java @@ -48,7 +48,7 @@ public class WIFILogFilter { */ public static String filterData(String rawLogEntry) throws IOException { - // Message CODE + // Message codes selected for mobility final int[] CODE_AUTHREQ = {501091, 501092, 501109}; final int[] CODE_AUTHRES = {501093, 501094, 501110}; final int[] CODE_DEAUTH = {501105, 501080, 501098, 501099, 501106, 501107, 501108, 501111}; // from and to @@ -58,39 +58,81 @@ public static String filterData(String rawLogEntry) throws IOException { final int[] CODE_USERAUTH = {522008, 522042, 522038}; // Successful and failed final int[] CODE_USRSTATUS = {522005, 522006, 522026}; // User Entry added, deleted, and user miss final int[] CODE_USERROAM = {500010}; + final int[] CODE_NEW_DEV = {522035}; - final String regPrefix = "(\\w+\\s+\\d+\\s+(?:\\d{1,2}:){2}\\d{1,2}(?:\\s+\\d{4})?)"; + // Regex for timestamp in e.g. "Dec 14 15:45:05 2015" + final String regTime = "(\\w+\\s+\\d+\\s+(?:\\d{1,2}:){2}\\d{1,2}(?:\\s+\\d{4})?)"; + + // Regex for user MAC address like f4:29:81:e3:7c:1f final String regUserMac = "((?:[0-9a-f]{2}:){5}[0-9a-f]{2})"; + + // Regex for IP addresses like 10.188.19.45 + final String regIPAddr = "((?:\\d{1,3}\\.){3}\\d{1,3})"; + + // Regex for IP addresses with specific range as stated in class doc. + final String regIPRange = "((?:111\\.\\d+|10\\.18[4-8])(?:\\.\\d+){2})"; + + // Regex for AP name, e.g. "CL-A-4F-04" + final String regApName = "([\\w-]+)"; + + // Regex for AP info such as "10.192.32.69-00:24:6c:59:b6:33-MH-JZG-10#-OUT" final String regApInfo = "((?:\\d{1,3}\\.){3}\\d{1,3})-((?:[0-9a-f]{2}:){5}[0-9a-f]{2})-([\\w-]+)"; + // Regex for roaming info such as "SJTU-Web/d8:c7:c8:28:ff:b9/a" + final String regRoamInfo = "([\\w-]+)/((?:[0-9a-f]{2}:){5}[0-9a-f]{2})/(\\w+)"; + // time: group(1), usename: group(2), apip: group(3), apmac: group(4), apname: group(5) - final Pattern REG_AUTHREQ = Pattern.compile(String.format("%s(?:.*)Auth\\s+request:\\s+%s:?\\s+(?:.*)AP\\s+%s", regPrefix, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE); + final Pattern REG_AUTHREQ = Pattern.compile(String.format("%s(?:.*)Auth\\s+request:\\s+%s:?\\s+(?:.*)AP\\s+%s", + regTime, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE); + // time: group(1), usename: group(2), apip: group(3), apmac: group(4), apname: group(5) - final Pattern REG_AUTHRES = Pattern.compile(String.format("%s(?:.*)Auth\\s+(success|failure):\\s+%s:?\\s+AP\\s+%s", regPrefix, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE); + final Pattern REG_AUTHRES = Pattern.compile(String.format("%s(?:.*)Auth\\s+(success|failure):\\s+%s:?\\s+AP\\s+%s", + regTime, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE); + // time: group(1), usename: group(2), apip: group(3), apmac: group(4), apname: group(5) - final Pattern REG_DEAUTH = Pattern.compile(String.format("%s(?:.*)Deauth(?:.*):\\s+%s:?\\s+(?:.*)AP\\s+%s", regPrefix, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE); + final Pattern REG_DEAUTH = Pattern.compile(String.format("%s(?:.*)Deauth(?:.*):\\s+%s:?\\s+(?:.*)AP\\s+%s", + regTime, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE); + // time: group(1), usename: group(2), apip: group(3), apmac: group(4), apname: group(5) - final Pattern REG_ASSOCREQ = Pattern.compile(String.format("%s(?:.*)Assoc(?:.*):\\s+%s(?:.*):?\\s+(?:.*)AP\\s+%s", regPrefix, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE); + final Pattern REG_ASSOCREQ = Pattern.compile(String.format("%s(?:.*)Assoc(?:.*):\\s+%s(?:.*):?\\s+(?:.*)AP\\s+%s", + regTime, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE); + // time: group(1), usename: group(2), apip: group(3), apmac: group(4), apname: group(5) - final Pattern REG_DISASSOCFROM = Pattern.compile(String.format("%s(?:.*)Disassoc(?:.*):\\s+%s:?\\s+AP\\s+%s", regPrefix, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE); + final Pattern REG_DISASSOCFROM = Pattern.compile(String.format("%s(?:.*)Disassoc(?:.*):\\s+%s:?\\s+AP\\s+%s", + regTime, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE); + // time: group(1), usename: group(2), usermac: group(3), userip: group(4), apname: group(5) - final Pattern REG_USERAUTH = Pattern.compile(String.format("%s(?:.*)\\s+username=([^\\s]+)\\s+MAC=%s\\s+IP=((?:\\d{1,3}\\.){3}\\d{1,3})(?:.+)(?:AP=([^\\s]+))?", regPrefix, regUserMac), Pattern.CASE_INSENSITIVE); + final Pattern REG_USERAUTH = Pattern.compile(String.format("%s(?:.*)\\s+username=([^\\s]+)\\s+MAC=%s\\s+IP=%s(?:.+)(?:AP=([^\\s]+))?", + regTime, regUserMac, regIPAddr), Pattern.CASE_INSENSITIVE); + // time: group(1), usermac: group(2), userip: group(3) - final Pattern REG_USRSTATUS = Pattern.compile(String.format("%s(?:.*)MAC=%s\\s+IP=((?:111\\.\\d+|10\\.18[4-8])(?:\\.\\d+){2})", regPrefix, regUserMac), Pattern.CASE_INSENSITIVE); + final Pattern REG_USRSTATUS = Pattern.compile(String.format("%s(?:.*)MAC=%s\\s+IP=%s", + regTime, regUserMac, regIPRange), Pattern.CASE_INSENSITIVE); + + // time: group(1), usermac: group(2), userip: group(3), apname: group(4), essid: group(5), bssid: group(6), phy: group(7) + final Pattern REG_USERROAM = Pattern.compile(String.format("%s(?:.*)Station\\s+%s,\\s+(?:%s)?:\\s+(?:.*)\\s+AP\\s+%s,\\s+%s", + regTime, regUserMac, regIPAddr, regApName, regRoamInfo)); + + // time: group(1), usermac: group(2), bssid: group(3), essid: group(4), apname: group(5) + final Pattern REG_NEW_DEV = Pattern.compile(String.format("%s(?:.*)MAC=%s Station UP: BSSID=%s ESSID=%s (?:.*)AP-name=%s", + regTime, regUserMac, regUserMac, regApName, regApName)); + String cleanLog = null; String[] chops = new String[0]; try { chops = rawLogEntry.split("<", 3); } catch (Exception e) { + // invalid syslog that is incomplete. return cleanLog; } - if (chops.length < 3 || chops[2].length() == 0 || chops[2].charAt(0) != '5') + if (chops.length < 3 || chops[2].length() == 0 || chops[2].charAt(0) != '5') { + // invalid syslog that does not convey user's mobility info. return cleanLog; + } int messageCode = Integer.valueOf(chops[2].split(">", 2)[0]); - System.out.println(messageCode); if (hasCodes(messageCode, CODE_AUTHREQ)) { // Auth request Matcher matcher = REG_AUTHREQ.matcher(rawLogEntry); @@ -102,7 +144,6 @@ public static String filterData(String rawLogEntry) throws IOException { } } else if (hasCodes(messageCode, CODE_DEAUTH)) { // Deauth from and to Matcher matcher = REG_DEAUTH.matcher(rawLogEntry); - System.out.println(matcher.find()); if (matcher.find()) { String time = formattrans(matcher.group(1)); String usermac = matcher.group(2).replaceAll(":", ""); @@ -155,6 +196,23 @@ public static String filterData(String rawLogEntry) throws IOException { */ cleanLog = String.format("%s,%s,%s,%s", usermac, time, action, userip); } + } else if (hasCodes(messageCode, CODE_USERROAM)) { + Matcher matcher = REG_USERROAM.matcher(rawLogEntry); + if (matcher.find()) { + String time = formattrans(matcher.group(1)); + String usermac = matcher.group(2).replaceAll(":", ""); + String userip = matcher.group(3); + String apname = matcher.group(4); + cleanLog = String.format("%s,%s,%s,%s,%s", usermac, time, WIFICode.UserRoam, apname, userip); + } + } else if (hasCodes(messageCode, CODE_NEW_DEV)) { + Matcher matcher = REG_NEW_DEV.matcher(rawLogEntry); + if (matcher.find()) { + String time = formattrans(matcher.group(1)); + String usermac = matcher.group(2).replaceAll(":", ""); + String apname = matcher.group(5); + cleanLog = String.format("%s,%s,%s,%s", usermac, time, WIFICode.NewDev, apname); + } } return cleanLog; diff --git a/etlers/WifiToolkit/test500010 b/etlers/WifiToolkit/test500010 new file mode 100644 index 0000000..5b682d1 --- /dev/null +++ b/etlers/WifiToolkit/test500010 @@ -0,0 +1,16 @@ +1449963445646 <141>Dec 13 07:29:18 2015 SJTU-Local3 mobileip[2209]: <500010> Station 60:fe:c5:6b:fa:4b, 10.188.71.9: Mobility trail, on switch 10.190.3.1, VLAN 1003, AP XH-ZY-3F-04, SJTU/6c:f3:7f:34:9f:18/a +1449963445741 <141>Dec 13 07:18:42 2015 SJTU-Local5 mobileip[2161]: <500010> Station 00:ee:bd:88:8c:c3, : Mobility trail, on switch 10.190.5.1, VLAN 1005, AP D3ST-1F-01, SJTU/6c:f3:7f:5a:cc:61/g +1449963446031 <141>Dec 13 07:20:29 2015 SJTU-Local2 mobileip[2151]: <500010> Station ac:7f:3e:78:7f:c4, 10.186.208.113: Mobility trail, on switch 10.190.2.1, VLAN 1002, AP YXL-2-1F-04, SJTU/d8:c7:c8:29:00:10/g +1449963446100 <141>Dec 13 07:18:42 2015 SJTU-Local5 mobileip[2161]: <500010> Station b0:c5:59:b9:11:75, : Mobility trail, on switch 10.190.5.1, VLAN 1005, AP DEST-1F-04, SJTU/6c:f3:7f:56:be:c1/g +1449963446362 <141>Dec 13 07:29:19 2015 SJTU-Local3 mobileip[2209]: <500010> Station 84:85:06:7f:3f:50, : Mobility trail, on switch 10.190.3.1, VLAN 1003, AP XH-ST-1F-10, SJTU/ac:a3:1e:f2:41:a0/g +1449963446799 <141>Dec 13 07:29:19 2015 SJTU-Local3 mobileip[2209]: <500010> Station f4:29:81:e3:7c:1f, 10.188.176.164: Mobility trail, on switch 10.190.3.1, VLAN 1004, AP DWST-1F-01, SJTU/6c:f3:7f:5a:cd:a1/g +1449963447111 <141>Dec 13 07:18:43 2015 SJTU-Local5 mobileip[2161]: <500010> Station 94:e9:6a:c4:03:68, 10.184.176.105: Mobility trail, on switch 10.190.5.1, VLAN 1005, AP DEST-1F-01, SJTU/6c:f3:7f:57:38:01/g +1449963447536 <141>Dec 13 07:29:20 2015 SJTU-Local3 mobileip[2209]: <500010> Station 84:db:ac:73:0b:3d, : Mobility trail, on switch 10.190.3.1, VLAN 1001, AP DSST-1F-04, SJTU/6c:f3:7f:5a:d1:40/g +1449963447607 <141>Dec 13 07:29:20 2015 SJTU-Local3 mobileip[2209]: <500010> Station 84:db:ac:54:ac:24, : Mobility trail, on switch 10.190.3.1, VLAN 1003, AP LXZL-4F-03, SJTU/6c:f3:7f:36:5a:40/g +1449963447821 <141>Dec 13 07:18:44 2015 SJTU-Local5 mobileip[2161]: <500010> Station e4:58:e7:55:94:b9, 10.184.176.66: Mobility trail, on switch 10.190.5.1, VLAN 1005, AP D3ST-1F-01, SJTU-Web/6c:f3:7f:5a:cc:62/g +1449963447949 <141>Dec 13 07:29:20 2015 SJTU-Local3 mobileip[2209]: <500010> Station d4:97:0b:49:3d:91, 10.188.112.153: Mobility trail, on switch 10.190.3.1, VLAN 1001, AP CL-A-4F-04, SJTU-Web/d8:c7:c8:28:ff:b9/a +1449963448535 <141>Dec 13 07:18:45 2015 SJTU-Local5 mobileip[2161]: <500010> Station 5c:97:f3:4d:87:86, 10.184.176.106: Mobility trail, on switch 10.190.5.1, VLAN 1005, AP DEST-1F-04, SJTU/6c:f3:7f:56:be:c1/g +1449963448803 <141>Dec 13 07:29:21 2015 SJTU-Local3 mobileip[2209]: <500010> Station 24:24:0e:8e:3b:f1, 10.188.112.151: Mobility trail, on switch 10.190.3.1, VLAN 1003, AP LXZL-2F-06, SJTU/6c:f3:7f:36:32:e8/a +1449963449337 <141>Dec 13 07:18:46 2015 SJTU-Local5 mobileip[2161]: <500010> Station 58:44:98:e8:75:ef, : Mobility trail, on switch 10.190.5.1, VLAN 1005, AP DZY-1-1F-08, SJTU/6c:f3:7f:5a:df:c0/g +1449963449358 <141>Dec 13 07:20:32 2015 SJTU-Local2 mobileip[2151]: <500010> Station e4:ce:8f:8f:91:71, : Mobility trail, on switch 10.190.2.1, VLAN 1002, AP JXDLXY-A-Z-8F-06, SJTU/d8:c7:c8:28:f6:10/g +1449963449564 <141>Dec 13 07:18:46 2015 SJTU-Local5 mobileip[2161]: <500010> Station d8:1d:72:b4:07:b6, 10.184.88.74: Mobility trail, on switch 10.190.5.1, VLAN 1005, AP XSFWZX-1F-01, SJTU-Web/6c:f3:7f:34:9a:41/g \ No newline at end of file diff --git a/etlers/WifiToolkit/test522035 b/etlers/WifiToolkit/test522035 new file mode 100644 index 0000000..0f0f02a --- /dev/null +++ b/etlers/WifiToolkit/test522035 @@ -0,0 +1,17 @@ +1449958022765 <142>Dec 13 05:50:06 2015 SJTU-Local2 authmgr[2421]: <522035> MAC=90:3c:92:cb:39:8f Station UP: BSSID=d8:c7:c8:29:05:90 ESSID=SJTU VLAN=1002 AP-name=JXDLXY-C-3F-01 +1449958022841 <142>Dec 13 05:58:55 2015 SJTU-Local3 authmgr[2480]: <522035> MAC=2c:1f:23:99:dc:c3 Station UP: BSSID=6c:f3:7f:36:36:20 ESSID=SJTU VLAN=1003 AP-name=XH-FXL-4F-11 +1449958027167 <142>Dec 13 05:48:24 2015 SJTU-Local5 authmgr[2434]: <522035> MAC=b8:03:05:80:ae:08 Station UP: BSSID=d8:c7:c8:29:04:80 ESSID=SJTU VLAN=1005 AP-name=ZNDW-3F-05 +1449958029030 <142>Dec 13 05:50:12 2015 SJTU-Local2 authmgr[2421]: <522035> MAC=f4:8b:32:a7:72:33 Station UP: BSSID=d8:c7:c8:26:c2:30 ESSID=SJTU VLAN=1002 AP-name=MLXY-N-1F-02 +1449958034085 <142>Dec 13 05:48:31 2015 SJTU-Local5 authmgr[2434]: <522035> MAC=9c:99:a0:c2:d9:75 Station UP: BSSID=6c:f3:7f:56:be:c2 ESSID=SJTU-Web VLAN=1005 AP-name=DEST-1F-04 +1449958035155 <142>Dec 13 05:48:32 2015 SJTU-Local5 authmgr[2434]: <522035> MAC=b8:03:05:80:ae:08 Station UP: BSSID=d8:c7:c8:29:04:80 ESSID=SJTU VLAN=1005 AP-name=ZNDW-3F-05 +1449958035713 <142>Dec 13 05:50:19 2015 SJTU-Local2 authmgr[2421]: <522035> MAC=7c:d1:c3:e2:4a:cd Station UP: BSSID=d8:c7:c8:26:e3:00 ESSID=SJTU VLAN=1002 AP-name=NXSWXY-F3-4F-04 +1449958037833 <142>Dec 13 05:50:21 2015 SJTU-Local2 authmgr[2421]: <522035> MAC=e4:25:e7:2e:fb:a6 Station UP: BSSID=d8:c7:c8:26:d8:10 ESSID=SJTU VLAN=1002 AP-name=JXDLXY-B-E-2F-03 +1449958038606 <142>Dec 13 05:50:22 2015 SJTU-Local2 authmgr[2421]: <522035> MAC=bc:85:56:11:c4:00 Station UP: BSSID=6c:f3:7f:37:e3:80 ESSID=SJTU VLAN=1002 AP-name=LLDCYJS-2F-02 +1449958039916 <142>Dec 13 05:50:23 2015 SJTU-Local2 authmgr[2421]: <522035> MAC=f4:8b:32:a7:72:33 Station UP: BSSID=d8:c7:c8:26:c2:30 ESSID=SJTU VLAN=1002 AP-name=MLXY-N-1F-02 +1449958040087 <142>Dec 13 05:50:23 2015 SJTU-Local2 authmgr[2421]: <522035> MAC=e4:25:e7:2e:fb:a6 Station UP: BSSID=d8:c7:c8:26:d8:10 ESSID=SJTU VLAN=1002 AP-name=JXDLXY-B-E-2F-03 +1449958044685 <142>Dec 13 05:48:41 2015 SJTU-Local5 authmgr[2434]: <522035> MAC=b8:03:05:80:ae:08 Station UP: BSSID=d8:c7:c8:29:04:80 ESSID=SJTU VLAN=1005 AP-name=ZNDW-3F-05 +1449958046907 <142>Dec 13 05:50:30 2015 SJTU-Local2 authmgr[2421]: <522035> MAC=bc:85:56:11:c4:00 Station UP: BSSID=6c:f3:7f:37:e3:80 ESSID=SJTU VLAN=1002 AP-name=LLDCYJS-2F-02 +1449958048074 <142>Dec 13 05:50:31 2015 SJTU-Local2 authmgr[2421]: <522035> MAC=c8:e7:d8:f9:51:65 Station UP: BSSID=d8:c7:c8:26:9b:11 ESSID=SJTU-Web VLAN=1002 AP-name=NXSWXY-F1-4F-03 +1449958049343 <142>Dec 13 05:59:22 2015 SJTU-Local3 authmgr[2480]: <522035> MAC=84:8e:0c:9c:93:42 Station UP: BSSID=6c:f3:7f:36:3d:e8 ESSID=SJTU VLAN=1003 AP-name=XYY-S-1F-03 +1449958051744 <142>Dec 13 05:48:48 2015 SJTU-Local5 authmgr[2434]: <522035> MAC=b8:03:05:80:ae:08 Station UP: BSSID=d8:c7:c8:29:04:80 ESSID=SJTU VLAN=1005 AP-name=ZNDW-3F-05 +1449958055264 <142>Dec 13 05:59:28 2015 SJTU-Local3 authmgr[2480]: <522035> MAC=74:81:14:b6:28:6c Station UP: BSSID=6c:f3:7f:37:b4:c1 ESSID=SJTU VLAN=1003 AP-name=XTSWYJY-C-4F-04