event2timeline.py

########################################################################################################
#                                                                                                      #
#   Event2Timeline by @tomchop_ for CERT Societe Generale (@CertSG). Some features added by @Jipe_     #
#                                                                                                      #
#   This work is licensed under the GPL License                                                        #
#   http://www.gnu.org/licenses/gpl.txt                                                                #
#                                                                                                      #
#   https://cert.societegenerale.com/ - https://github.com/certsocietegenerale/event2timeline/         #
#                                                                                                      #
########################################################################################################

### /!\ You must modify the USERNAME_STRING to match the "username" strings in your langage! /!\ ###

import csv, sys, re
import optparse
import datetime
import calendar
import mmap
import contextlib
from dateutil.parser import parse
from Evtx.Evtx import FileHeader
from Evtx.Views import evtx_file_xml_view

__description__ = "Event2Timeline"
__version__ = "0.0.2"

eid_regex = re.compile('<EventID Qualifiers="(?P<qualifiers>.*)">(?P<eid>\d+)</EventID>')
sessid_regex = re.compile('<Data Name="TargetLogonId">(?P<session_id>0x[0-9a-fA-F]+)</Data>')
time_regex = re.compile('SystemTime="(?P<time>.*)"')

EVTX_LOGIN = [
				4624, 	# An account was successfully logged on
				4778,	# RDP - Session connceted / reconnected
				]	

EVTX_LOGOFF = [
				4647, 	# User initiated logoff
				4634,	# An account was logged off
				4779, 	# RDP - Session disconnected
				]

EVT_LOGIN = [i-4096 for i in EVTX_LOGIN] + [540] # Successful network logon (=4624 in EVTX)
EVT_LOGOFF = [i-4096 for i in EVTX_LOGOFF]

USERNAME_STRING = 'tilisateur' ###### MODIFY THIS TO WHICHEVER LANGUAGE YOUR CSV FILE IS ######

def get_data(xml, name):
	rex = re.compile('<Data Name="%s">(?P<%s>.*)</Data>' % (name, name))
	try:
		return rex.search(xml).group(name)
	except Exception, e:
		return None

def import_xml(filename):

	# 4624 - Login 		528
	# 4647 - Logoff 	551

	#[*] Keys: Category, Description, Data, Domain\User, Date&Time, Source, Computer, Time, Date, Type, Event

	sessions = {}
	user_sessions = {}
	count = 0

	with open(filename, 'r') as f:
		print "[*] Reading EVTX file %s" % filename
		with contextlib.closing(mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)) as buf:
			fh = FileHeader(buf, 0x0)
			count = 0
			for xml, record in evtx_file_xml_view(fh):
				if (count % 2000) == 0:
					print "%s records read" % count
				count +=1
				
				match = eid_regex.search(xml)
				eid = int(match.group('eid'))
				session_id = get_data(xml, 'TargetLogonId')
				
				# Insert new session in dictionary
				if sessions.get(session_id, None) == None:
					sessions[session_id] = {}

				if eid in EVTX_LOGIN:

					if session_id:
						sessions[session_id] = {}
					else:
						continue

					info = {}
					info['logon_type'] = get_data(xml, 'LogonType')
					info['eid'] = str(eid)
					info['ip'] = get_data(xml, 'IpAddress') + ':' + get_data(xml, 'IpPort')
					info['datetime'] = parse(time_regex.search(xml).group('time')[:-7])

					sessions[session_id][str(eid)] = info
					username = get_data(xml, 'TargetDomainName') + '\\' + get_data(xml, 'TargetUserName')
					sessions[session_id]['username'] = username
				
				elif eid in EVTX_LOGOFF:
					# Ignore if orphan session
					if not sessions.get(session_id, None) == None:
						continue

					info = {}
					info['eid'] = str(eid)
					info['datetime'] = parse(time_regex.search(xml).group('time')[:-7])
					sessions[session_id][str(eid)] = info


	return sessions

def import_csv(filename, delimiter=';', quotechar='"'):

	with open (filename, 'rU') as csvfile:
		logs = []
		print "[*] Reading CSV file %s" % filename
		logreader = csv.DictReader(csvfile, delimiter=delimiter, quotechar=quotechar)

# Audit Success;23/05/2013;09:00:00;23/05/2013 09:00:00;538;Security;Ouverture/Fermeture de session;\S-1-5-21-2052699199-3915784498-1582209984-43253;USER01;"Fermeture de la session utilisateur : Utilisateur :        username Domaine :        userdomain Id. de la session :        (0x0,0xB38D21AB) Type de session :        4"; 

		for log in logreader:
			logs.append(log)

		print "[*] %s lines imported" % (len(logs))
		print '[*] Keys: %s' % ", ".join([i for i in logs[0]])

		# return logs[::-1]

		sessions = {}
		user_sessions = {}

		count = 0

		for log in logs:

			count += 1

			try:
				session_id = re.search('(?P<session_id>0x([0-9a-fA-F]{2,}))', log['Description']).group('session_id')
			except Exception, e:
				continue

			if sessions.get(session_id, None) == None:
				sessions[session_id] = {}

			if int(log['Event']) in EVT_LOGIN:
				info = {}
				info['eid'] = log['Event']
				info['datetime'] = parse(log['Date&Time'])
				
				sessions[session_id][log['Event']] = info
				username = re.search("%s\W+(?P<username>[\w\.\-$]+)\n" %USERNAME_STRING, log['Description']).group('username')
				sessions[session_id]['username'] = username

			elif int(log['Event']) in EVT_LOGOFF:
				if sessions.get(session_id, None) == None: # Avoid orphan sessions
					continue
				info = {}
				info['eid'] = log['Event']
				info['datetime'] = parse(log['Date&Time'])

				sessions[session_id][log['Event']] = info

		print "[*] %s sessions found (%s lines parsed)" % (len(sessions), count)
		return sessions

def sessions2timeline(sessions):

		# generate session list by user
		user_sessions = {}
		for sid in [s for s in sessions if sessions[s].get('username', None) != None]:
			s = sessions[sid]
			
			if user_sessions.get(s['username'], None) == None:
				user_sessions[s['username']] = {}
			
			user_sessions[s['username']][sid] = s
			
		print "[*] Unique users: %s" % len(user_sessions)
				
		lanes = [u for u in user_sessions]

		items = []
		for i, username in enumerate(user_sessions):

			for user_session in get_sessions(user_sessions[username]):
				if user_session['end'] - user_session['start'] < datetime.timedelta(seconds=10):
					user_session['end'] = user_session['start'] + datetime.timedelta(seconds=10)

				items.append({'info': user_session['info'],'lane': i, 'start': str(user_session['start']), 'end': str(user_session['end'])})
			
		time_begin = min([i['start'] for i in items])
		time_end = max([i['end'] for i in items])

		return {'time_begin': time_begin, 'time_end': time_end, 'items': items, 'lanes': lanes}




def get_sessions(user_sessions):
	
	sessions = []

	for sid in user_sessions:
		s = user_sessions[sid]
		start, end = None, None
		for evt in s:
			if evt != 'username':
				if int(evt) in EVT_LOGIN or int(evt) in EVTX_LOGIN: # deal with a login event- look for the smallest date for session start
					if start == None:
						start = s[evt]['datetime']
					elif s[evt]['datetime'] < start:
						start = s[evt]['datetime']
				if int(evt) in EVT_LOGOFF or int(evt) in EVTX_LOGOFF: # deal with a logoff event- look for the biggest date for session start
					if end == None:
						end = s[evt]['datetime']
					elif s[evt]['datetime'] > end:
						end = s[evt]['datetime']

		if end == None:
			end = start
		if start == None:
			start = end

		# remove datetime object, which does not parse well to JS
		for i in s:
			try:
				s[i].pop('datetime')
			except Exception, e:
				pass
		sessions.append({'start': start, 'end': end, 'info': s})
	
	return sessions
	

def print_log(log):
	for key in log:
		for entry in log[key]:
			for k in entry:
				print "%s:\n%s\n" % (k, entry[k])	
		else:
			print "%s:\n%s\n" % (key, log[key])

if __name__ == '__main__':

	Parser = optparse.OptionParser(usage='usage: %prog [-c | -e] -f eventlogfile')
	Parser.add_option('-f', '--filename', dest="eventlogfile", help='path to the evenlog file')
	Parser.add_option('-c', '--csv', action="store_true", default=False, help='Specify the events are in CSV format (for an exported .evt)')
	Parser.add_option('-e', '--evtx', action="store_true", default=False, help='Specify the events are in EVTX format (for a native .evtx)')

	(options, args) = Parser.parse_args()
	
	if not options.eventlogfile:
		Parser.error("You must specify a filename")

	if options.csv:
		sessions = import_csv(options.eventlogfile)
	elif options.evtx:
		sessions = import_xml(options.eventlogfile)
	else:
		Parser.error("You must specify a file format format (csv or xml)")

	timeline = sessions2timeline(sessions)

	print "[*] Mapped %s sessions from %s to %s" % (len(timeline['items']), timeline['time_begin'], timeline['time_end'])


	js = open('timeline/evtdata.js','w+')

	js.write("var lanes = %s,\n" % str(timeline['lanes']))
	js.write("laneLength = lanes.length,\n")
	js.write("items = %s,\n" % timeline['items'])
	js.write("timeBegin = \'%s\',\n" % timeline['time_begin'])
	js.write("timeEnd = \'%s\',\n" % timeline['time_end'])
	js.write("filename = \'%s\';\n" % options.eventlogfile)


	js.close()