SHISO_benchmark.py

#!/usr/bin/env python

import sys
sys.path.append('../')
from logparser import SHISO, evaluator
import os
import pandas as pd


input_dir = '../logs/' # The input directory of log file
output_dir = 'SHISO_result/' # The output directory of parsing results

benchmark_settings = {
    'HDFS': {
        'log_file': 'HDFS/HDFS_2k.log',
        'log_format': '<Date> <Time> <Pid> <Level> <Component>: <Content>',
        'regex': [r'blk_-?\d+', r'(\d+\.){3}\d+(:\d+)?'],
        'maxChildNum': 4,
        'mergeThreshold': 0.1,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85
        },

    'Hadoop': {
        'log_file': 'Hadoop/Hadoop_2k.log',
        'log_format': '<Date> <Time> <Level> \[<Process>\] <Component>: <Content>', 
        'regex': [r'(\d+\.){3}\d+'],
        'maxChildNum': 4,
        'mergeThreshold': 0.002,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85
        },

    'Spark': {
        'log_file': 'Spark/Spark_2k.log',
        'log_format': '<Date> <Time> <Level> <Component>: <Content>', 
        'regex': [r'(\d+\.){3}\d+', r'\b[KGTM]?B\b', r'([\w-]+\.){2,}[\w-]+'],
        'maxChildNum': 4,
        'mergeThreshold': 0.002,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85
        },

    'Zookeeper': {
        'log_file': 'Zookeeper/Zookeeper_2k.log',
        'log_format': '<Date> <Time> - <Level>  \[<Node>:<Component>@<Id>\] - <Content>',
        'regex': [r'(/|)(\d+\.){3}\d+(:\d+)?'],
        'maxChildNum': 4,
        'mergeThreshold': 0.003,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85
        },

    'BGL': {
        'log_file': 'BGL/BGL_2k.log',
        'log_format': '<Label> <Timestamp> <Date> <Node> <Time> <NodeRepeat> <Type> <Component> <Level> <Content>',
        'regex': [r'core\.\d+'],
        'maxChildNum': 4,
        'mergeThreshold': 0.005,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85
        },

    'HPC': {
        'log_file': 'HPC/HPC_2k.log',
        'log_format': '<LogId> <Node> <Component> <State> <Time> <Flag> <Content>',
        'regex': [r'=\d+'],
        'maxChildNum': 3,
        'mergeThreshold': 0.003,
        'formatLookupThreshold': 0.6,
        'superFormatThreshold': 0.4
        },

    'Thunderbird': {
        'log_file': 'Thunderbird/Thunderbird_2k.log',
        'log_format': '<Label> <Timestamp> <Date> <User> <Month> <Day> <Time> <Location> <Component>(\[<PID>\])?: <Content>',
        'regex': [r'(\d+\.){3}\d+'],
        'maxChildNum': 4,
        'mergeThreshold': 0.0002,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85
        },

    'Windows': {
        'log_file': 'Windows/Windows_2k.log',
        'log_format': '<Date> <Time>, <Level>                  <Component>    <Content>',
        'regex': [r'0x.*?\s'],
        'maxChildNum': 3,
        'mergeThreshold': 0.002,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85
        },

    'Linux': {
        'log_file': 'Linux/Linux_2k.log',
        'log_format': '<Month> <Date> <Time> <Level> <Component>(\[<PID>\])?: <Content>',
        'regex': [r'(\d+\.){3}\d+', r'\d{2}:\d{2}:\d{2}'],
        'maxChildNum': 3,
        'mergeThreshold': 0.005,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.4
        },

    'Andriod': {
        'log_file': 'Andriod/Andriod_2k.log',
        'log_format': '<Date> <Time>  <Pid>  <Tid> <Level> <Component>: <Content>',
        'regex': [r'(/[\w-]+)+', r'([\w-]+\.){2,}[\w-]+', r'\b(\-?\+?\d+)\b|\b0[Xx][a-fA-F\d]+\b|\b[a-fA-F\d]{4,}\b'],
        'maxChildNum': 4,
        'mergeThreshold': 0.002,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85        
        },

    'HealthApp': {
        'log_file': 'HealthApp/HealthApp_2k.log',
        'log_format': '<Time>\|<Component>\|<Pid>\|<Content>',
        'regex': [],
        'maxChildNum': 4,
        'mergeThreshold': 0.0001,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85        
        },

    'Apache': {
        'log_file': 'Apache/Apache_2k.log',
        'log_format': '\[<Time>\] \[<Level>\] <Content>',
        'regex': [r'(\d+\.){3}\d+'],
        'maxChildNum': 4,
        'mergeThreshold': 0.002,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85
        },

    'Proxifier': {
        'log_file': 'Proxifier/Proxifier_2k.log',
        'log_format': '\[<Time>\] <Program> - <Content>',
        'regex': [r'<\d+\ssec', r'([\w-]+\.)+[\w-]+(:\d+)?', r'\d{2}:\d{2}(:\d{2})*', r'\b[KGTM]?B\b'],
        'maxChildNum': 4,
        'mergeThreshold': 0.002,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85
        },

    'OpenSSH': {
        'log_file': 'OpenSSH/OpenSSH_2k.log',
        'log_format': '<Date> <Day> <Time> <Component> sshd\[<Pid>\]: <Content>',
        'regex': [r'(\d+\.){3}\d+', r'([\w-]+\.){2,}[\w-]+'],
        'maxChildNum': 4,
        'mergeThreshold': 0.002,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85
        },

    'OpenStack': {
        'log_file': 'OpenStack/OpenStack_2k.log',
        'log_format': '<Logrecord> <Date> <Time> <Pid> <Level> <Component> \[<ADDR>\] <Content>',
        'regex': [r'((\d+\.){3}\d+,?)+', r'/.+?\s', r'\d+'],
        'maxChildNum': 4,
        'mergeThreshold': 0.002,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85
        },

    'Mac': {
        'log_file': 'Mac/Mac_2k.log',
        'log_format': '<Month>  <Date> <Time> <User> <Component>\[<PID>\]( \(<Address>\))?: <Content>',
        'regex': [r'([\w-]+\.){2,}[\w-]+'],
        'maxChildNum': 4,
        'mergeThreshold': 0.002,
        'formatLookupThreshold': 0.3,
        'superFormatThreshold': 0.85
    },
}

bechmark_result = []
for dataset, setting in benchmark_settings.iteritems():
    print('\n=== Evaluation on %s ==='%dataset)
    indir = os.path.join(input_dir, os.path.dirname(setting['log_file']))
    log_file = os.path.basename(setting['log_file'])

    parser = SHISO.LogParser(log_format=setting['log_format'], indir=indir, outdir=output_dir, rex=setting['regex'],
                            maxChildNum=setting['maxChildNum'], mergeThreshold=setting['mergeThreshold'],
                            formatLookupThreshold=setting['formatLookupThreshold'], superFormatThreshold=setting['superFormatThreshold'])
    parser.parse(log_file)
    
    F1_measure, accuracy = evaluator.evaluate(
                           groundtruth=os.path.join(indir, log_file + '_structured.csv'),
                           parsedresult=os.path.join(output_dir, log_file + '_structured.csv')
                           )
    bechmark_result.append([dataset, F1_measure, accuracy])


print('\n=== Overall evaluation results ===')
df_result = pd.DataFrame(bechmark_result, columns=['Dataset', 'F1_measure', 'Accuracy'])
df_result.set_index('Dataset', inplace=True)
print(df_result)
df_result.T.to_csv('SHISO_bechmark_result.csv')