enhance ngxtop ability

ngxtop/config_parser.py

@@ -4,6 +4,8 @@
 import os
 import re
 import subprocess
+import glob
+
 
 from pyparsing import Literal, Word, ZeroOrMore, OneOrMore, Group, \
     printables, quotedString, pythonStyleComment, removeQuotes
@@ -59,17 +61,21 @@ def get_access_logs(config):
     access_log = Literal("access_log") + ZeroOrMore(parameter) + semicolon
     access_log.ignore(pythonStyleComment)
 
+    access_logs_dict = {}
     for directive in access_log.searchString(config).asList():
         path = directive[1]
         if path == 'off' or path.startswith('syslog:'):
             # nothing to process here
             continue
-
-        format_name = 'combined'
+        access_logs_dict[path] = ['combined']
         if len(directive) > 2 and '=' not in directive[2]:
-            format_name = directive[2]
+            if directive[2] not in access_logs_dict[path]:
+                if 'combined' in access_logs_dict[path]:
+                    access_logs_dict[path] = [directive[2]]
+                else:
+                    (access_logs_dict[path]).append(directive[2])
+    return access_logs_dict
 
-        yield path, format_name
 
 
 def get_log_formats(config):
@@ -86,6 +92,32 @@ def get_log_formats(config):
         format_string = ''.join(directive[2])
         yield name, format_string
 
+def get_config_str(config):
+    """
+    Parse config for include_format directives
+    """
+    # include path
+    include = Literal("include") + ZeroOrMore(parameter) + semicolon
+    include.ignore(pythonStyleComment)
+
+    config_str = ''
+    config_str_list = []  
+    path_list = [config] 
+    with open(config) as f1:
+        config_str_list.append(f1.read())
+        for directive in include.searchString(config_str_list[0]).asList():
+            path = directive[1]
+            path_list.append (path)
+        for path in path_list: 
+            if path == config:
+                continue
+            file_list = glob.glob(path)  
+            for file_path in file_list:
+                with open(file_path) as f2: 
+                    config_str_list.append(f2.read())
+    config_str = '\n'.join(config_str_list)
+    return config_str
+
 
 def detect_log_config(arguments):
     """
@@ -98,53 +130,65 @@ def detect_log_config(arguments):
     if not os.path.exists(config):
         error_exit('Nginx config file not found: %s' % config)
 
-    with open(config) as f:
-        config_str = f.read()
-    access_logs = dict(get_access_logs(config_str))
-    if not access_logs:
+    config_str = get_config_str(config) 
+    access_logs_dict = get_access_logs(config_str)
+    if len(access_logs_dict) == 0:
         error_exit('Access log file is not provided and ngxtop cannot detect it from your config file (%s).' % config)
 
-    log_formats = dict(get_log_formats(config_str))
-    if len(access_logs) == 1:
-        log_path, format_name = list(access_logs.items())[0]
-        if format_name == 'combined':
-            return log_path, LOG_FORMAT_COMBINED
-        if format_name not in log_formats:
-            error_exit('Incorrect format name set in config for access log file "%s"' % log_path)
-        return log_path, log_formats[format_name]
+    log_formats_dict = dict(get_log_formats(config_str))
+    if len(access_logs_dict) == 1:
+        for log_path in access_logs_dict:
+            if access_logs_dict[log_path] == ['combined']:
+                log_formats_dict.clear()
+                log_formats_dict['combined'] = LOG_FORMAT_COMBINED
+                return log_path, log_formats_dict
+            for format_name in access_logs_dict[log_path]:
+                if format_name not in log_formats_dict:
+                    error_exit('Incorrect format name set in config for access log file "%s"' % log_path)
+            return log_path, log_formats_dict
 
     # multiple access logs configured, offer to select one
     print('Multiple access logs detected in configuration:')
-    log_path = choose_one(list(access_logs.keys()), 'Select access log file to process: ')
-    format_name = access_logs[log_path]
-    if format_name not in log_formats:
-        error_exit('Incorrect format name set in config for access log file "%s"' % log_path)
-    return log_path, log_formats[format_name]
+    log_path = choose_one(list(access_logs_dict.keys()), 'Select access log file to process: ')
+    format_name_list = access_logs_dict[log_path]
+    for format_name in format_name_list:
+        if format_name not in log_formats_dict:
+            error_exit('Incorrect format name set in config for access log file "%s"' % log_path)
+    return log_path, dict(log_formats_dict[log_path])
 
 
-def build_pattern(log_format):
+def build_pattern(log_formats_dict, arguments):
     """
     Build regular expression to parse given format.
     :param log_format: format string to parse
     :return: regular expression to parse given format
     """
-    if log_format == 'combined':
-        log_format = LOG_FORMAT_COMBINED
-    elif log_format == 'common':
-        log_format = LOG_FORMAT_COMMON
-    pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_format)
-    pattern = re.sub(REGEX_LOG_FORMAT_VARIABLE, '(?P<\\1>.*)', pattern)
-    return re.compile(pattern)
-
-
-def extract_variables(log_format):
+    log_format = arguments['--log-format']
+    if len(log_formats_dict) == 0:
+        if log_format == 'combined':
+            log_format = LOG_FORMAT_COMBINED
+        elif log_format == 'common':
+            log_format = LOG_FORMAT_COMMON
+        pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_format)
+        pattern = re.sub(REGEX_LOG_FORMAT_VARIABLE, '(?P<\\1>.*)', pattern)
+        return re.compile(pattern)
+    else:
+        pattern_list = []
+        for key in log_formats_dict:
+            pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_formats_dict[key])
+            pattern = re.sub(REGEX_LOG_FORMAT_VARIABLE, '(?P<\\1>.*)', pattern)
+            pattern_list.append(re.compile(pattern))
+        return pattern_list
+
+def extract_variables(log_formats_dict):
     """
     Extract all variables from a log format string.
     :param log_format: format string to extract
     :return: iterator over all variables in given format string
     """
-    if log_format == 'combined':
-        log_format = LOG_FORMAT_COMBINED
-    for match in re.findall(REGEX_LOG_FORMAT_VARIABLE, log_format):
-        yield match
+    for log_format in log_formats_dict:
+        if log_format == 'combined':
+            log_format = LOG_FORMAT_COMBINED
+        for match in re.findall(REGEX_LOG_FORMAT_VARIABLE, log_format):
+            yield match
 

ngxtop/ngxtop.py

@@ -183,7 +183,10 @@ def to_float(value):
 
 
 def parse_log(lines, pattern):
-    matches = (pattern.match(l) for l in lines)
+    if isinstance(pattern, list): 
+        matches = (pattern_value.match(l) for l in lines for pattern_value in pattern)
+    else:
+        matches = (pattern.match(l) for l in lines )
     records = (m.groupdict() for m in matches if m is not None)
     records = map_field('status', to_int, records)
     records = add_field('status_type', parse_status_type, records)
@@ -257,7 +260,6 @@ def process_log(lines, pattern, processor, arguments):
     pre_filer_exp = arguments['--pre-filter']
     if pre_filer_exp:
         lines = (line for line in lines if eval(pre_filer_exp, {}, dict(line=line)))
-
     records = parse_log(lines, pattern)
 
     filter_exp = arguments['--filter']
@@ -344,27 +346,27 @@ def print_report(sig, frame):
 
 def process(arguments):
     access_log = arguments['--access-log']
-    log_format = arguments['--log-format']
+    log_formats_dict = {}
     if access_log is None and not sys.stdin.isatty():
         # assume logs can be fetched directly from stdin when piped
         access_log = 'stdin'
     if access_log is None:
-        access_log, log_format = detect_log_config(arguments)
+        access_log, log_formats_dict = detect_log_config(arguments)
+        logging.info('log_format: %s', log_formats_dict)
 
     logging.info('access_log: %s', access_log)
-    logging.info('log_format: %s', log_format)
     if access_log != 'stdin' and not os.path.exists(access_log):
         error_exit('access log file "%s" does not exist' % access_log)
 
     if arguments['info']:
         print('nginx configuration file:\n ', detect_config_path())
         print('access log file:\n ', access_log)
-        print('access log format:\n ', log_format)
-        print('available variables:\n ', ', '.join(sorted(extract_variables(log_format))))
+        print('access log format:\n ', log_formats_dict)
+        print('available variables:\n ', ', '.join(sorted(extract_variables(log_formats_dict))))
         return
 
     source = build_source(access_log, arguments)
-    pattern = build_pattern(log_format)
+    pattern = build_pattern(log_formats_dict, arguments)
     processor = build_processor(arguments)
     setup_reporter(processor, arguments)
     process_log(source, pattern, processor, arguments)