Skip to content

Commit

Permalink
Add new types to cleansed syslog: CODE_USERROAM, CODE_NEW_DEV
Browse files Browse the repository at this point in the history
  • Loading branch information
@caesar0301
caesar0301 committed Dec 24, 2015
1 parent 987e9d9 commit 82f3bc7
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 12 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
*.pyc
*.DS_Store
target/
.idea/
Expand Down
2 changes: 2 additions & 0 deletions etlers/WifiToolkit/src/main/java/cn/edu/sjtu/omnilab/odh/rawfilter/WIFICode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,6 @@ public class WIFICode {
public static final int UserAuth = 4;
public static final int IPAllocation = 5;
public static final int IPRecycle = 6;
public static final int UserRoam = 7;
public static final int NewDev = 8;
}
82 changes: 70 additions & 12 deletions etlers/WifiToolkit/src/main/java/cn/edu/sjtu/omnilab/odh/rawfilter/WIFILogFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public class WIFILogFilter {
*/
public static String filterData(String rawLogEntry) throws IOException {

// Message CODE
// Message codes selected for mobility
final int[] CODE_AUTHREQ = {501091, 501092, 501109};
final int[] CODE_AUTHRES = {501093, 501094, 501110};
final int[] CODE_DEAUTH = {501105, 501080, 501098, 501099, 501106, 501107, 501108, 501111}; // from and to
Expand All @@ -58,39 +58,81 @@ public static String filterData(String rawLogEntry) throws IOException {
final int[] CODE_USERAUTH = {522008, 522042, 522038}; // Successful and failed
final int[] CODE_USRSTATUS = {522005, 522006, 522026}; // User Entry added, deleted, and user miss
final int[] CODE_USERROAM = {500010};
final int[] CODE_NEW_DEV = {522035};

final String regPrefix = "(\\w+\\s+\\d+\\s+(?:\\d{1,2}:){2}\\d{1,2}(?:\\s+\\d{4})?)";
// Regex for timestamp in e.g. "Dec 14 15:45:05 2015"
final String regTime = "(\\w+\\s+\\d+\\s+(?:\\d{1,2}:){2}\\d{1,2}(?:\\s+\\d{4})?)";

// Regex for user MAC address like f4:29:81:e3:7c:1f
final String regUserMac = "((?:[0-9a-f]{2}:){5}[0-9a-f]{2})";

// Regex for IP addresses like 10.188.19.45
final String regIPAddr = "((?:\\d{1,3}\\.){3}\\d{1,3})";

// Regex for IP addresses with specific range as stated in class doc.
final String regIPRange = "((?:111\\.\\d+|10\\.18[4-8])(?:\\.\\d+){2})";

// Regex for AP name, e.g. "CL-A-4F-04"
final String regApName = "([\\w-]+)";

// Regex for AP info such as "10.192.32.69-00:24:6c:59:b6:33-MH-JZG-10#-OUT"
final String regApInfo = "((?:\\d{1,3}\\.){3}\\d{1,3})-((?:[0-9a-f]{2}:){5}[0-9a-f]{2})-([\\w-]+)";

// Regex for roaming info such as "SJTU-Web/d8:c7:c8:28:ff:b9/a"
final String regRoamInfo = "([\\w-]+)/((?:[0-9a-f]{2}:){5}[0-9a-f]{2})/(\\w+)";

// time: group(1), usename: group(2), apip: group(3), apmac: group(4), apname: group(5)
final Pattern REG_AUTHREQ = Pattern.compile(String.format("%s(?:.*)Auth\\s+request:\\s+%s:?\\s+(?:.*)AP\\s+%s", regPrefix, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE);
final Pattern REG_AUTHREQ = Pattern.compile(String.format("%s(?:.*)Auth\\s+request:\\s+%s:?\\s+(?:.*)AP\\s+%s",
regTime, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE);

// time: group(1), usename: group(2), apip: group(3), apmac: group(4), apname: group(5)
final Pattern REG_AUTHRES = Pattern.compile(String.format("%s(?:.*)Auth\\s+(success|failure):\\s+%s:?\\s+AP\\s+%s", regPrefix, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE);
final Pattern REG_AUTHRES = Pattern.compile(String.format("%s(?:.*)Auth\\s+(success|failure):\\s+%s:?\\s+AP\\s+%s",
regTime, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE);

// time: group(1), usename: group(2), apip: group(3), apmac: group(4), apname: group(5)
final Pattern REG_DEAUTH = Pattern.compile(String.format("%s(?:.*)Deauth(?:.*):\\s+%s:?\\s+(?:.*)AP\\s+%s", regPrefix, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE);
final Pattern REG_DEAUTH = Pattern.compile(String.format("%s(?:.*)Deauth(?:.*):\\s+%s:?\\s+(?:.*)AP\\s+%s",
regTime, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE);

// time: group(1), usename: group(2), apip: group(3), apmac: group(4), apname: group(5)
final Pattern REG_ASSOCREQ = Pattern.compile(String.format("%s(?:.*)Assoc(?:.*):\\s+%s(?:.*):?\\s+(?:.*)AP\\s+%s", regPrefix, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE);
final Pattern REG_ASSOCREQ = Pattern.compile(String.format("%s(?:.*)Assoc(?:.*):\\s+%s(?:.*):?\\s+(?:.*)AP\\s+%s",
regTime, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE);

// time: group(1), usename: group(2), apip: group(3), apmac: group(4), apname: group(5)
final Pattern REG_DISASSOCFROM = Pattern.compile(String.format("%s(?:.*)Disassoc(?:.*):\\s+%s:?\\s+AP\\s+%s", regPrefix, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE);
final Pattern REG_DISASSOCFROM = Pattern.compile(String.format("%s(?:.*)Disassoc(?:.*):\\s+%s:?\\s+AP\\s+%s",
regTime, regUserMac, regApInfo), Pattern.CASE_INSENSITIVE);

// time: group(1), usename: group(2), usermac: group(3), userip: group(4), apname: group(5)
final Pattern REG_USERAUTH = Pattern.compile(String.format("%s(?:.*)\\s+username=([^\\s]+)\\s+MAC=%s\\s+IP=((?:\\d{1,3}\\.){3}\\d{1,3})(?:.+)(?:AP=([^\\s]+))?", regPrefix, regUserMac), Pattern.CASE_INSENSITIVE);
final Pattern REG_USERAUTH = Pattern.compile(String.format("%s(?:.*)\\s+username=([^\\s]+)\\s+MAC=%s\\s+IP=%s(?:.+)(?:AP=([^\\s]+))?",
regTime, regUserMac, regIPAddr), Pattern.CASE_INSENSITIVE);

// time: group(1), usermac: group(2), userip: group(3)
final Pattern REG_USRSTATUS = Pattern.compile(String.format("%s(?:.*)MAC=%s\\s+IP=((?:111\\.\\d+|10\\.18[4-8])(?:\\.\\d+){2})", regPrefix, regUserMac), Pattern.CASE_INSENSITIVE);
final Pattern REG_USRSTATUS = Pattern.compile(String.format("%s(?:.*)MAC=%s\\s+IP=%s",
regTime, regUserMac, regIPRange), Pattern.CASE_INSENSITIVE);

// time: group(1), usermac: group(2), userip: group(3), apname: group(4), essid: group(5), bssid: group(6), phy: group(7)
final Pattern REG_USERROAM = Pattern.compile(String.format("%s(?:.*)Station\\s+%s,\\s+(?:%s)?:\\s+(?:.*)\\s+AP\\s+%s,\\s+%s",
regTime, regUserMac, regIPAddr, regApName, regRoamInfo));

// time: group(1), usermac: group(2), bssid: group(3), essid: group(4), apname: group(5)
final Pattern REG_NEW_DEV = Pattern.compile(String.format("%s(?:.*)MAC=%s Station UP: BSSID=%s ESSID=%s (?:.*)AP-name=%s",
regTime, regUserMac, regUserMac, regApName, regApName));


String cleanLog = null;
String[] chops = new String[0];
try {
chops = rawLogEntry.split("<", 3);
} catch (Exception e) {
// invalid syslog that is incomplete.
return cleanLog;
}

if (chops.length < 3 || chops[2].length() == 0 || chops[2].charAt(0) != '5')
if (chops.length < 3 || chops[2].length() == 0 || chops[2].charAt(0) != '5') {
// invalid syslog that does not convey user's mobility info.
return cleanLog;
}

int messageCode = Integer.valueOf(chops[2].split(">", 2)[0]);
System.out.println(messageCode);

if (hasCodes(messageCode, CODE_AUTHREQ)) { // Auth request
Matcher matcher = REG_AUTHREQ.matcher(rawLogEntry);
Expand All @@ -102,7 +144,6 @@ public static String filterData(String rawLogEntry) throws IOException {
}
} else if (hasCodes(messageCode, CODE_DEAUTH)) { // Deauth from and to
Matcher matcher = REG_DEAUTH.matcher(rawLogEntry);
System.out.println(matcher.find());
if (matcher.find()) {
String time = formattrans(matcher.group(1));
String usermac = matcher.group(2).replaceAll(":", "");
Expand Down Expand Up @@ -155,6 +196,23 @@ public static String filterData(String rawLogEntry) throws IOException {
*/
cleanLog = String.format("%s,%s,%s,%s", usermac, time, action, userip);
}
} else if (hasCodes(messageCode, CODE_USERROAM)) {
Matcher matcher = REG_USERROAM.matcher(rawLogEntry);
if (matcher.find()) {
String time = formattrans(matcher.group(1));
String usermac = matcher.group(2).replaceAll(":", "");
String userip = matcher.group(3);
String apname = matcher.group(4);
cleanLog = String.format("%s,%s,%s,%s,%s", usermac, time, WIFICode.UserRoam, apname, userip);
}
} else if (hasCodes(messageCode, CODE_NEW_DEV)) {
Matcher matcher = REG_NEW_DEV.matcher(rawLogEntry);
if (matcher.find()) {
String time = formattrans(matcher.group(1));
String usermac = matcher.group(2).replaceAll(":", "");
String apname = matcher.group(5);
cleanLog = String.format("%s,%s,%s,%s", usermac, time, WIFICode.NewDev, apname);
}
}

return cleanLog;
Expand Down
16 changes: 16 additions & 0 deletions etlers/WifiToolkit/test500010
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
1449963445646 <141>Dec 13 07:29:18 2015 SJTU-Local3 mobileip[2209]: <500010> <NOTI> <SJTU-Local3 10.190.3.1> Station 60:fe:c5:6b:fa:4b, 10.188.71.9: Mobility trail, on switch 10.190.3.1, VLAN 1003, AP XH-ZY-3F-04, SJTU/6c:f3:7f:34:9f:18/a
1449963445741 <141>Dec 13 07:18:42 2015 SJTU-Local5 mobileip[2161]: <500010> <NOTI> <SJTU-Local5 10.190.5.1> Station 00:ee:bd:88:8c:c3, : Mobility trail, on switch 10.190.5.1, VLAN 1005, AP D3ST-1F-01, SJTU/6c:f3:7f:5a:cc:61/g
1449963446031 <141>Dec 13 07:20:29 2015 SJTU-Local2 mobileip[2151]: <500010> <NOTI> <SJTU-Local2 10.190.2.1> Station ac:7f:3e:78:7f:c4, 10.186.208.113: Mobility trail, on switch 10.190.2.1, VLAN 1002, AP YXL-2-1F-04, SJTU/d8:c7:c8:29:00:10/g
1449963446100 <141>Dec 13 07:18:42 2015 SJTU-Local5 mobileip[2161]: <500010> <NOTI> <SJTU-Local5 10.190.5.1> Station b0:c5:59:b9:11:75, : Mobility trail, on switch 10.190.5.1, VLAN 1005, AP DEST-1F-04, SJTU/6c:f3:7f:56:be:c1/g
1449963446362 <141>Dec 13 07:29:19 2015 SJTU-Local3 mobileip[2209]: <500010> <NOTI> <SJTU-Local3 10.190.3.1> Station 84:85:06:7f:3f:50, : Mobility trail, on switch 10.190.3.1, VLAN 1003, AP XH-ST-1F-10, SJTU/ac:a3:1e:f2:41:a0/g
1449963446799 <141>Dec 13 07:29:19 2015 SJTU-Local3 mobileip[2209]: <500010> <NOTI> <SJTU-Local3 10.190.3.1> Station f4:29:81:e3:7c:1f, 10.188.176.164: Mobility trail, on switch 10.190.3.1, VLAN 1004, AP DWST-1F-01, SJTU/6c:f3:7f:5a:cd:a1/g
1449963447111 <141>Dec 13 07:18:43 2015 SJTU-Local5 mobileip[2161]: <500010> <NOTI> <SJTU-Local5 10.190.5.1> Station 94:e9:6a:c4:03:68, 10.184.176.105: Mobility trail, on switch 10.190.5.1, VLAN 1005, AP DEST-1F-01, SJTU/6c:f3:7f:57:38:01/g
1449963447536 <141>Dec 13 07:29:20 2015 SJTU-Local3 mobileip[2209]: <500010> <NOTI> <SJTU-Local3 10.190.3.1> Station 84:db:ac:73:0b:3d, : Mobility trail, on switch 10.190.3.1, VLAN 1001, AP DSST-1F-04, SJTU/6c:f3:7f:5a:d1:40/g
1449963447607 <141>Dec 13 07:29:20 2015 SJTU-Local3 mobileip[2209]: <500010> <NOTI> <SJTU-Local3 10.190.3.1> Station 84:db:ac:54:ac:24, : Mobility trail, on switch 10.190.3.1, VLAN 1003, AP LXZL-4F-03, SJTU/6c:f3:7f:36:5a:40/g
1449963447821 <141>Dec 13 07:18:44 2015 SJTU-Local5 mobileip[2161]: <500010> <NOTI> <SJTU-Local5 10.190.5.1> Station e4:58:e7:55:94:b9, 10.184.176.66: Mobility trail, on switch 10.190.5.1, VLAN 1005, AP D3ST-1F-01, SJTU-Web/6c:f3:7f:5a:cc:62/g
1449963447949 <141>Dec 13 07:29:20 2015 SJTU-Local3 mobileip[2209]: <500010> <NOTI> <SJTU-Local3 10.190.3.1> Station d4:97:0b:49:3d:91, 10.188.112.153: Mobility trail, on switch 10.190.3.1, VLAN 1001, AP CL-A-4F-04, SJTU-Web/d8:c7:c8:28:ff:b9/a
1449963448535 <141>Dec 13 07:18:45 2015 SJTU-Local5 mobileip[2161]: <500010> <NOTI> <SJTU-Local5 10.190.5.1> Station 5c:97:f3:4d:87:86, 10.184.176.106: Mobility trail, on switch 10.190.5.1, VLAN 1005, AP DEST-1F-04, SJTU/6c:f3:7f:56:be:c1/g
1449963448803 <141>Dec 13 07:29:21 2015 SJTU-Local3 mobileip[2209]: <500010> <NOTI> <SJTU-Local3 10.190.3.1> Station 24:24:0e:8e:3b:f1, 10.188.112.151: Mobility trail, on switch 10.190.3.1, VLAN 1003, AP LXZL-2F-06, SJTU/6c:f3:7f:36:32:e8/a
1449963449337 <141>Dec 13 07:18:46 2015 SJTU-Local5 mobileip[2161]: <500010> <NOTI> <SJTU-Local5 10.190.5.1> Station 58:44:98:e8:75:ef, : Mobility trail, on switch 10.190.5.1, VLAN 1005, AP DZY-1-1F-08, SJTU/6c:f3:7f:5a:df:c0/g
1449963449358 <141>Dec 13 07:20:32 2015 SJTU-Local2 mobileip[2151]: <500010> <NOTI> <SJTU-Local2 10.190.2.1> Station e4:ce:8f:8f:91:71, : Mobility trail, on switch 10.190.2.1, VLAN 1002, AP JXDLXY-A-Z-8F-06, SJTU/d8:c7:c8:28:f6:10/g
1449963449564 <141>Dec 13 07:18:46 2015 SJTU-Local5 mobileip[2161]: <500010> <NOTI> <SJTU-Local5 10.190.5.1> Station d8:1d:72:b4:07:b6, 10.184.88.74: Mobility trail, on switch 10.190.5.1, VLAN 1005, AP XSFWZX-1F-01, SJTU-Web/6c:f3:7f:34:9a:41/g
17 changes: 17 additions & 0 deletions etlers/WifiToolkit/test522035
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
1449958022765 <142>Dec 13 05:50:06 2015 SJTU-Local2 authmgr[2421]: <522035> <INFO> <SJTU-Local2 10.190.2.1> MAC=90:3c:92:cb:39:8f Station UP: BSSID=d8:c7:c8:29:05:90 ESSID=SJTU VLAN=1002 AP-name=JXDLXY-C-3F-01
1449958022841 <142>Dec 13 05:58:55 2015 SJTU-Local3 authmgr[2480]: <522035> <INFO> <SJTU-Local3 10.190.3.1> MAC=2c:1f:23:99:dc:c3 Station UP: BSSID=6c:f3:7f:36:36:20 ESSID=SJTU VLAN=1003 AP-name=XH-FXL-4F-11
1449958027167 <142>Dec 13 05:48:24 2015 SJTU-Local5 authmgr[2434]: <522035> <INFO> <SJTU-Local5 10.190.5.1> MAC=b8:03:05:80:ae:08 Station UP: BSSID=d8:c7:c8:29:04:80 ESSID=SJTU VLAN=1005 AP-name=ZNDW-3F-05
1449958029030 <142>Dec 13 05:50:12 2015 SJTU-Local2 authmgr[2421]: <522035> <INFO> <SJTU-Local2 10.190.2.1> MAC=f4:8b:32:a7:72:33 Station UP: BSSID=d8:c7:c8:26:c2:30 ESSID=SJTU VLAN=1002 AP-name=MLXY-N-1F-02
1449958034085 <142>Dec 13 05:48:31 2015 SJTU-Local5 authmgr[2434]: <522035> <INFO> <SJTU-Local5 10.190.5.1> MAC=9c:99:a0:c2:d9:75 Station UP: BSSID=6c:f3:7f:56:be:c2 ESSID=SJTU-Web VLAN=1005 AP-name=DEST-1F-04
1449958035155 <142>Dec 13 05:48:32 2015 SJTU-Local5 authmgr[2434]: <522035> <INFO> <SJTU-Local5 10.190.5.1> MAC=b8:03:05:80:ae:08 Station UP: BSSID=d8:c7:c8:29:04:80 ESSID=SJTU VLAN=1005 AP-name=ZNDW-3F-05
1449958035713 <142>Dec 13 05:50:19 2015 SJTU-Local2 authmgr[2421]: <522035> <INFO> <SJTU-Local2 10.190.2.1> MAC=7c:d1:c3:e2:4a:cd Station UP: BSSID=d8:c7:c8:26:e3:00 ESSID=SJTU VLAN=1002 AP-name=NXSWXY-F3-4F-04
1449958037833 <142>Dec 13 05:50:21 2015 SJTU-Local2 authmgr[2421]: <522035> <INFO> <SJTU-Local2 10.190.2.1> MAC=e4:25:e7:2e:fb:a6 Station UP: BSSID=d8:c7:c8:26:d8:10 ESSID=SJTU VLAN=1002 AP-name=JXDLXY-B-E-2F-03
1449958038606 <142>Dec 13 05:50:22 2015 SJTU-Local2 authmgr[2421]: <522035> <INFO> <SJTU-Local2 10.190.2.1> MAC=bc:85:56:11:c4:00 Station UP: BSSID=6c:f3:7f:37:e3:80 ESSID=SJTU VLAN=1002 AP-name=LLDCYJS-2F-02
1449958039916 <142>Dec 13 05:50:23 2015 SJTU-Local2 authmgr[2421]: <522035> <INFO> <SJTU-Local2 10.190.2.1> MAC=f4:8b:32:a7:72:33 Station UP: BSSID=d8:c7:c8:26:c2:30 ESSID=SJTU VLAN=1002 AP-name=MLXY-N-1F-02
1449958040087 <142>Dec 13 05:50:23 2015 SJTU-Local2 authmgr[2421]: <522035> <INFO> <SJTU-Local2 10.190.2.1> MAC=e4:25:e7:2e:fb:a6 Station UP: BSSID=d8:c7:c8:26:d8:10 ESSID=SJTU VLAN=1002 AP-name=JXDLXY-B-E-2F-03
1449958044685 <142>Dec 13 05:48:41 2015 SJTU-Local5 authmgr[2434]: <522035> <INFO> <SJTU-Local5 10.190.5.1> MAC=b8:03:05:80:ae:08 Station UP: BSSID=d8:c7:c8:29:04:80 ESSID=SJTU VLAN=1005 AP-name=ZNDW-3F-05
1449958046907 <142>Dec 13 05:50:30 2015 SJTU-Local2 authmgr[2421]: <522035> <INFO> <SJTU-Local2 10.190.2.1> MAC=bc:85:56:11:c4:00 Station UP: BSSID=6c:f3:7f:37:e3:80 ESSID=SJTU VLAN=1002 AP-name=LLDCYJS-2F-02
1449958048074 <142>Dec 13 05:50:31 2015 SJTU-Local2 authmgr[2421]: <522035> <INFO> <SJTU-Local2 10.190.2.1> MAC=c8:e7:d8:f9:51:65 Station UP: BSSID=d8:c7:c8:26:9b:11 ESSID=SJTU-Web VLAN=1002 AP-name=NXSWXY-F1-4F-03
1449958049343 <142>Dec 13 05:59:22 2015 SJTU-Local3 authmgr[2480]: <522035> <INFO> <SJTU-Local3 10.190.3.1> MAC=84:8e:0c:9c:93:42 Station UP: BSSID=6c:f3:7f:36:3d:e8 ESSID=SJTU VLAN=1003 AP-name=XYY-S-1F-03
1449958051744 <142>Dec 13 05:48:48 2015 SJTU-Local5 authmgr[2434]: <522035> <INFO> <SJTU-Local5 10.190.5.1> MAC=b8:03:05:80:ae:08 Station UP: BSSID=d8:c7:c8:29:04:80 ESSID=SJTU VLAN=1005 AP-name=ZNDW-3F-05
1449958055264 <142>Dec 13 05:59:28 2015 SJTU-Local3 authmgr[2480]: <522035> <INFO> <SJTU-Local3 10.190.3.1> MAC=74:81:14:b6:28:6c Station UP: BSSID=6c:f3:7f:37:b4:c1 ESSID=SJTU VLAN=1003 AP-name=XTSWYJY-C-4F-04

0 comments on commit 82f3bc7

Please sign in to comment.