Move dataloder into seperate module and add BGL and Thunderbird dataloader

benchmarks/HDFS_bechmark.py

@@ -5,27 +5,28 @@
 sys.path.append('../')
 import pandas as pd
 from loglizer.models import *
-from loglizer import dataloader, preprocessing
+from loglizer import preprocessing
+from loglizer.dataloader import HDFS
 
-run_models = ['PCA', 'InvariantsMiner', 'LogClustering', 'IsolationForest', 'LR', 
+run_models = ['PCA', 'InvariantsMiner', 'LogClustering', 'IsolationForest', 'LR',
               'SVM', 'DecisionTree']
 struct_log = '../data/HDFS/HDFS.npz' # The benchmark dataset
 
 if __name__ == '__main__':
-    (x_tr, y_train), (x_te, y_test) = dataloader.load_HDFS(struct_log,
-                                                           window='session', 
-                                                           train_ratio=0.5,
-                                                           split_type='uniform')
+    (x_tr, y_train), (x_te, y_test) = HDFS.loadDataset(struct_log,
+                                                window='session',
+                                                train_ratio=0.5,
+                                                split_type='uniform')
     benchmark_results = []
     for _model in run_models:
         print('Evaluating {} on HDFS:'.format(_model))
         if _model == 'PCA':
             feature_extractor = preprocessing.FeatureExtractor()
-            x_train = feature_extractor.fit_transform(x_tr, term_weighting='tf-idf', 
+            x_train = feature_extractor.fit_transform(x_tr, term_weighting='tf-idf',
                                                       normalization='zero-mean')
             model = PCA()
             model.fit(x_train)
-        
+
         elif _model == 'InvariantsMiner':
             feature_extractor = preprocessing.FeatureExtractor()
             x_train = feature_extractor.fit_transform(x_tr)
@@ -41,7 +42,7 @@
         elif _model == 'IsolationForest':
             feature_extractor = preprocessing.FeatureExtractor()
             x_train = feature_extractor.fit_transform(x_tr)
-            model = IsolationForest(random_state=2019, max_samples=0.9999, contamination=0.03, 
+            model = IsolationForest(random_state=2019, max_samples=0.9999, contamination=0.03,
                                     n_jobs=4)
             model.fit(x_train)
 
@@ -62,7 +63,7 @@
             x_train = feature_extractor.fit_transform(x_tr, term_weighting='tf-idf')
             model = DecisionTree()
             model.fit(x_train, y_train)
-        
+
         x_test = feature_extractor.transform(x_te)
         print('Train accuracy:')
         precision, recall, f1 = model.evaluate(x_train, y_train)

demo/DecisionTree_demo.py

@@ -4,15 +4,17 @@
 import sys
 sys.path.append('../')
 from loglizer.models import DecisionTree
-from loglizer import dataloader, preprocessing
+from loglizer import preprocessing
+from loglizer import preprocessing
+from loglizer.dataloader import HDFS
 
 struct_log = '../data/HDFS/HDFS_100k.log_structured.csv' # The structured log file
 label_file = '../data/HDFS/anomaly_label.csv' # The anomaly label file
 
 if __name__ == '__main__':
-    (x_train, y_train), (x_test, y_test) = dataloader.load_HDFS(struct_log,
+    (x_train, y_train), (x_test, y_test) = HDFS.loadDataset(struct_log,
                                                                 label_file=label_file,
-                                                                window='session', 
+                                                                window='session',
                                                                 train_ratio=0.5,
                                                                 split_type='uniform')
 

demo/DeepLog_demo.py

@@ -6,7 +6,8 @@
 from loglizer import dataloader
 from loglizer.models import DeepLog
 from loglizer.preprocessing import Vectorizer, Iterator
-
+from loglizer import preprocessing
+from loglizer.dataloader import HDFS
 
 batch_size = 32
 hidden_size = 32
@@ -16,14 +17,14 @@
 window_size = 10
 epoches = 2
 num_workers = 2
-device = 0 
+device = 0
 
 struct_log = '../data/HDFS/HDFS_100k.log_structured.csv' # The structured log file
 label_file = '../data/HDFS/anomaly_label.csv' # The anomaly label file
 
 if __name__ == '__main__':
-    (x_train, window_y_train, y_train), (x_test, window_y_test, y_test) = dataloader.load_HDFS(struct_log, label_file=label_file, window='session', window_size=window_size, train_ratio=train_ratio, split_type='uniform')
-    
+    (x_train, window_y_train, y_train), (x_test, window_y_test, y_test) = HDFS.loadDataset(struct_log, label_file=label_file, window='session', window_size=window_size, train_ratio=train_ratio, split_type='uniform')
+
     feature_extractor = Vectorizer()
     train_dataset = feature_extractor.fit_transform(x_train, window_y_train, y_train)
     test_dataset = feature_extractor.transform(x_test, window_y_test, y_test)

demo/InvariantsMiner_demo.py

@@ -4,16 +4,18 @@
 import sys
 sys.path.append('../')
 from loglizer.models import InvariantsMiner
-from loglizer import dataloader, preprocessing
+from loglizer import preprocessing
+from loglizer import preprocessing
+from loglizer.dataloader import HDFS
 
 struct_log = '../data/HDFS/HDFS_100k.log_structured.csv' # The structured log file
 label_file = '../data/HDFS/anomaly_label.csv' # The anomaly label file
 epsilon = 0.5 # threshold for estimating invariant space
 
 if __name__ == '__main__':
-    (x_train, y_train), (x_test, y_test) = dataloader.load_HDFS(struct_log,
+    (x_train, y_train), (x_test, y_test) = HDFS.loadDataset(struct_log,
                                                                 label_file=label_file,
-                                                                window='session', 
+                                                                window='session',
                                                                 train_ratio=0.5,
                                                                 split_type='sequential')
     feature_extractor = preprocessing.FeatureExtractor()
@@ -25,7 +27,7 @@
 
     print('Train validation:')
     precision, recall, f1 = model.evaluate(x_train, y_train)
-    
+
     print('Test validation:')
     precision, recall, f1 = model.evaluate(x_test, y_test)
 

demo/InvariantsMiner_demo_without_labels.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 ''' This is a demo file for the Invariants Mining model.
     API usage:
-        dataloader.load_HDFS(): load HDFS dataset
+        HDFS.loadDataset(): load HDFS dataset
         feature_extractor.fit_transform(): fit and transform features
         feature_extractor.transform(): feature transform after fitting
         model.fit(): fit the model
@@ -13,16 +13,18 @@
 import sys
 sys.path.append('../')
 from loglizer.models import InvariantsMiner
-from loglizer import dataloader, preprocessing
+from loglizer import preprocessing
+from loglizer import preprocessing
+from loglizer.dataloader import HDFS
 
 struct_log = '../data/HDFS/HDFS_100k.log_structured.csv' # The structured log file
 label_file = '../data/HDFS/anomaly_label.csv' # The anomaly label file
 epsilon = 0.5 # threshold for estimating invariant space
 
 if __name__ == '__main__':
     # Load structured log without label info
-    (x_train, _), (x_test, _) = dataloader.load_HDFS(struct_log,
-                                                     window='session', 
+    (x_train, _), (x_test, _) = HDFS.loadDataset(struct_log,
+                                                     window='session',
                                                      train_ratio=0.5,
                                                      split_type='sequential')
     # Feature extraction
@@ -43,11 +45,11 @@
 
     # If you have labeled data, you can evaluate the accuracy of the model as well.
     # Load structured log with label info
-    (x_train, y_train), (x_test, y_test) = dataloader.load_HDFS(struct_log,
+    (x_train, y_train), (x_test, y_test) = HDFS.loadDataset(struct_log,
                                                                label_file=label_file,
-                                                               window='session', 
+                                                               window='session',
                                                                train_ratio=0.5,
-                                                               split_type='sequential')   
+                                                               split_type='sequential')
     x_test = feature_extractor.transform(x_test)
     precision, recall, f1 = model.evaluate(x_test, y_test)
 

demo/IsolationForest_demo.py

@@ -4,16 +4,17 @@
 import sys
 sys.path.append('../')
 from loglizer.models import IsolationForest
-from loglizer import dataloader, preprocessing
+from loglizer import preprocessing
+from loglizer.dataloader import HDFS
 
 struct_log = '../data/HDFS/HDFS_100k.log_structured.csv' # The structured log file
 label_file = '../data/HDFS/anomaly_label.csv' # The anomaly label file
 anomaly_ratio = 0.03 # Estimate the ratio of anomaly samples in the data
 
 if __name__ == '__main__':
-    (x_train, y_train), (x_test, y_test) = dataloader.load_HDFS(struct_log,
+    (x_train, y_train), (x_test, y_test) = HDFS.loadDataset(struct_log,
                                                                 label_file=label_file,
-                                                                window='session', 
+                                                                window='session',
                                                                 train_ratio=0.5,
                                                                 split_type='uniform')
     feature_extractor = preprocessing.FeatureExtractor()
@@ -25,7 +26,7 @@
 
     print('Train validation:')
     precision, recall, f1 = model.evaluate(x_train, y_train)
-    
+
     print('Test validation:')
     precision, recall, f1 = model.evaluate(x_test, y_test)
 

demo/LR_demo.py

@@ -4,15 +4,17 @@
 import sys
 sys.path.append('../')
 from loglizer.models import LR
-from loglizer import dataloader, preprocessing
+from loglizer import preprocessing
+from loglizer import preprocessing
+from loglizer.dataloader import HDFS
 
 struct_log = '../data/HDFS/HDFS_100k.log_structured.csv' # The structured log file
 label_file = '../data/HDFS/anomaly_label.csv' # The anomaly label file
 
 if __name__ == '__main__':
-    (x_train, y_train), (x_test, y_test) = dataloader.load_HDFS(struct_log,
+    (x_train, y_train), (x_test, y_test) = HDFS.loadDataset(struct_log,
                                                                 label_file=label_file,
-                                                                window='session', 
+                                                                window='session',
                                                                 train_ratio=0.5,
                                                                 split_type='uniform')
 

demo/LogClustering_demo.py

@@ -4,17 +4,19 @@
 import sys
 sys.path.append('../')
 from loglizer.models import LogClustering
-from loglizer import dataloader, preprocessing
+from loglizer import preprocessing
+from loglizer import preprocessing
+from loglizer.dataloader import HDFS
 
 struct_log = '../data/HDFS/HDFS_100k.log_structured.csv' # The structured log file
 label_file = '../data/HDFS/anomaly_label.csv' # The anomaly label file
 max_dist = 0.3 # the threshold to stop the clustering process
 anomaly_threshold = 0.3 # the threshold for anomaly detection
 
 if __name__ == '__main__':
-    (x_train, y_train), (x_test, y_test) = dataloader.load_HDFS(struct_log,
+    (x_train, y_train), (x_test, y_test) = HDFS.loadDataset(struct_log,
                                                                 label_file=label_file,
-                                                                window='session', 
+                                                                window='session',
                                                                 train_ratio=0.5,
                                                                 split_type='uniform')
     feature_extractor = preprocessing.FeatureExtractor()
@@ -26,6 +28,6 @@
 
     print('Train validation:')
     precision, recall, f1 = model.evaluate(x_train, y_train)
-    
+
     print('Test validation:')
     precision, recall, f1 = model.evaluate(x_test, y_test)

demo/PCA_demo.py

@@ -4,19 +4,21 @@
 import sys
 sys.path.append('../')
 from loglizer.models import PCA
-from loglizer import dataloader, preprocessing
+from loglizer import preprocessing
+from loglizer import preprocessing
+from loglizer.dataloader import HDFS
 
 struct_log = '../data/HDFS/HDFS_100k.log_structured.csv' # The structured log file
 label_file = '../data/HDFS/anomaly_label.csv' # The anomaly label file
 
 if __name__ == '__main__':
-    (x_train, y_train), (x_test, y_test) = dataloader.load_HDFS(struct_log,
+    (x_train, y_train), (x_test, y_test) = HDFS.loadDataset(struct_log,
                                                                 label_file=label_file,
-                                                                window='session', 
+                                                                window='session',
                                                                 train_ratio=0.5,
                                                                 split_type='uniform')
     feature_extractor = preprocessing.FeatureExtractor()
-    x_train = feature_extractor.fit_transform(x_train, term_weighting='tf-idf', 
+    x_train = feature_extractor.fit_transform(x_train, term_weighting='tf-idf',
                                               normalization='zero-mean')
     x_test = feature_extractor.transform(x_test)
 
@@ -25,6 +27,6 @@
 
     print('Train validation:')
     precision, recall, f1 = model.evaluate(x_train, y_train)
-    
+
     print('Test validation:')
     precision, recall, f1 = model.evaluate(x_test, y_test)

demo/PCA_demo_without_labels.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 ''' This is a demo file for the PCA model.
     API usage:
-        dataloader.load_HDFS(): load HDFS dataset
+        HDFS.loadDataset(): load HDFS dataset
         feature_extractor.fit_transform(): fit and transform features
         feature_extractor.transform(): feature transform after fitting
         model.fit(): fit the model
@@ -13,36 +13,38 @@
 import sys
 sys.path.append('../')
 from loglizer.models import PCA
-from loglizer import dataloader, preprocessing
+from loglizer import preprocessing
+from loglizer import preprocessing
+from loglizer.dataloader import HDFS
 
 struct_log = '../data/HDFS/HDFS_100k.log_structured.csv' # The structured log file
 
 if __name__ == '__main__':
     ## 1. Load strutured log file and extract feature vectors
     # Save the raw event sequence file by setting save_csv=True
-    (x_train, _), (_, _) = dataloader.load_HDFS(struct_log, window='session', 
+    (x_train, _), (_, _) = HDFS.loadDataset(struct_log, window='session',
                                                 split_type='sequential', save_csv=True)
     feature_extractor = preprocessing.FeatureExtractor()
-    x_train = feature_extractor.fit_transform(x_train, term_weighting='tf-idf', 
+    x_train = feature_extractor.fit_transform(x_train, term_weighting='tf-idf',
                                               normalization='zero-mean')
-    
+
     ## 2. Train an unsupervised model
     print('Train phase:')
     # Initialize PCA, or other unsupervised models, LogClustering, InvariantsMiner
-    model = PCA() 
+    model = PCA()
     # Model hyper-parameters may be sensitive to log data, here we use the default for demo
     model.fit(x_train)
     # Make predictions and manually check for correctness. Details may need to go into the raw logs
-    y_train = model.predict(x_train) 
+    y_train = model.predict(x_train)
 
     ## 3. Use the trained model for online anomaly detection
     print('Test phase:')
     # Load another new log file. Here we use struct_log for demo only
-    (x_test, _), (_, _) = dataloader.load_HDFS(struct_log, window='session', split_type='sequential')
+    (x_test, _), (_, _) = HDFS.loadDataset(struct_log, window='session', split_type='sequential')
     # Go through the same feature extraction process with training, using transform() instead
-    x_test = feature_extractor.transform(x_test) 
+    x_test = feature_extractor.transform(x_test)
     # Finally make predictions and alter on anomaly cases
     y_test = model.predict(x_test)
-    
+
 
 

demo/SVM_demo.py

@@ -4,15 +4,17 @@
 import sys
 sys.path.append('../')
 from loglizer.models import SVM
-from loglizer import dataloader, preprocessing
+from loglizer import preprocessing
+from loglizer import preprocessing
+from loglizer.dataloader import HDFS
 
 struct_log = '../data/HDFS/HDFS_100k.log_structured.csv' # The structured log file
 label_file = '../data/HDFS/anomaly_label.csv' # The anomaly label file
 
 if __name__ == '__main__':
-    (x_train, y_train), (x_test, y_test) = dataloader.load_HDFS(struct_log,
+    (x_train, y_train), (x_test, y_test) = HDFS.loadDataset(struct_log,
                                                                 label_file=label_file,
-                                                                window='session', 
+                                                                window='session',
                                                                 train_ratio=0.5,
                                                                 split_type='uniform')