raven-coredump

#!/usr/bin/gdb --python
# Copyright 2017 Thomas Sondergaard <tsondergaard@vitalimages.com>
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials provided
# with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import argparse
import datetime
import gdb
import glob
import gzip
import sys
if  sys.version_info[0] < 3:
    import imp
else:
    import importlib
import io
import json
import os
import platform
import re
import requests
import shutil
import socket
import subprocess
import syslog
import tempfile
import traceback
import uuid
import zipfile
from gdb.FrameIterator import FrameIterator
try:
    from urllib.parse import urlparse # Python 3
except:
    from urlparse import urlparse # Python 2

sdk_name = 'raven-coredump'
sdk_version = '0.2.0'

# Variables can be shadowed
def insert_var(dict, symbol, frame):
    name = symbol.name
    i = 0
    while name in dict:
        name = "%s (shadowed%s)" % (symbol.name, '*' * i)
        i = i + 1
    try:
        var_value = str(symbol.value(frame))
    except gdb.MemoryError as mem_error:
        var_value = str(mem_error)
    dict[name] = var_value

# returns local variables and function arguments
def frame_vars_json(frame):
    try:
        block = frame.block()
    except:
        return None
    vars = {}
    # Inspired by https://stackoverflow.com/questions/30013252
    while block:
        for symbol in block:
            if (symbol.is_variable or symbol.is_argument):
                insert_var(vars, symbol, frame)
        # Don't search upwards past the function scope
        if block.function:
            break
        else:
            block = block.superblock
    return vars

def frame_json(frame):
    sal = frame.find_sal()
    js = {
        'filename': sal.symtab.fullname() if (sal and sal.symtab) else '<unknown>',
        'function': str(frame.function() or frame.name() or 'unknown'),
        'module': gdb.solib_name(frame.pc()),
        'instruction_addr': ('0x%x' % frame.pc())
    }
    if (sal and sal.line > 0):
        js['lineno'] = sal.line
    vars = frame_vars_json(frame)
    if (vars):
        js['vars'] = vars
    return js

def backtrace_json(top_frame):
    frames = [ frame_json(frame) for frame in FrameIterator(top_frame) ]
    # Sentry format oddly wants oldest frame first.
    frames.reverse()
    return { 'frames': frames }

def thread_json(thread):
    thread.switch()
    js = {
        'id': thread.ptid[1],
        'name': thread.name,
        'crashed': thread == crashing_thread,
        'stacktrace': backtrace_json(gdb.newest_frame())
    }
    return js

def threads_json(threads):
    threadsjs = [ thread_json(thread) for thread in threads ]
    return { 'values': threadsjs }

def prune_vars_from_stacktrace(stacktrace):
    for frame in stacktrace['frames']:
        frame.pop('vars', None)

def prune_vars_from_threads(threads):
    threads_list = threads['values']
    if threads_list:
        for thread in threads_list:
            stacktrace = thread['stacktrace']
            if stacktrace:
                prune_vars_from_stacktrace(stacktrace)

def prune_vars_from_sentry_event(sentry_event):
    stacktrace = sentry_event['stacktrace']
    if stacktrace:
        prune_vars_from_stacktrace(stacktrace)
    threads = sentry_event['threads']
    if threads:
        prune_vars_from_threads(threads)

def install_program(dest_dir = '/'):
    dest_path = 'opt/raven-coredump/bin'
    dest_file = os.path.join(dest_path, 'raven-coredump')
    print('Installing %s' % dest_file)
    if not os.path.isdir(os.path.join(dest_dir, dest_path)):
        os.makedirs(os.path.join(dest_dir, dest_path))
    shutil.copy(os.path.realpath(__file__), os.path.join(dest_dir, dest_file))
    os.chmod(os.path.join(dest_dir, dest_file), 0o755)
    etc_dir = 'etc'
    conf_file = os.path.join(etc_dir, 'raven-coredump.conf')
    if os.path.exists(os.path.join(dest_dir, conf_file)):
        print('%s exists - leaving it unchanged' % os.path.join(dest_dir, conf_file))
    else:
        print('Installing %s' % os.path.join(dest_dir, conf_file))
        if not os.path.isdir(os.path.join(dest_dir, etc_dir)):
            os.makedirs(os.path.join(dest_dir, etc_dir))
        with open(os.path.join(dest_dir, conf_file), 'w') as outfile:
            outfile.write(json.dumps({
                '#dsn': 'https://sentry_key:sentry_secret@sentry.io/project-number',
                '#set-opaque-type-resolution-off': True,
                '#include-variables': False,
                '#local-crash-folder': '/var/opt/raven-coredump'
            }, indent=4))
            outfile.write('\n')
    core_pattern_setting = 'kernel.core_pattern=|%s -e=%%e -E=%%E -p=%%p -P=%%P' % os.path.join('/', dest_file)

    sysctl_dir = os.path.join(etc_dir, 'sysctl.d')
    if not os.path.isdir(os.path.join(dest_dir, sysctl_dir)):
        os.makedirs(os.path.join(dest_dir, sysctl_dir))
    sysctl_conf_file = os.path.join(sysctl_dir, '80-raven-coredump.conf')
    print('Installing %s' % os.path.join(dest_dir, sysctl_conf_file))
    with open(os.path.join(dest_dir, sysctl_conf_file), 'w') as outfile:
        outfile.write(core_pattern_setting)
        outfile.write('\n')
    if dest_dir == '/':
        sysctl_cmd = 'sysctl "%s"' % core_pattern_setting
        print('Running %s' % sysctl_cmd)
        subprocess.call(sysctl_cmd, shell=True)

def load_python_plugin(filename):
    if  sys.version_info[0] < 3:
        plugin_module = imp.load_source('ravenplugin', filename)
    else:
        spec = importlib.util.spec_from_file_location('ravenplugin', filename)
        plugin_module = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(plugin_module)
    return plugin_module

def gzip_compress(data):
    out = io.BytesIO()
    # Cannot use "with" because GZipFile from Python 2.6 (shipped with CentOS 6) doesn't
    # support the context manager protocol
    f = gzip.GzipFile(fileobj=out, mode="w");
    try:
        f.write(data)
    finally:
        f.close()
    return out.getvalue()

class RavenClient:
    def __init__(self, dsn):
        self.dsn = dsn
        url = urlparse(self.dsn)
        self.sentry_key = url.username
        self.sentry_secret = url.password
        self.project = re.sub(r'^/', '', url.path)
        self.scheme = url.scheme
        self.hostname = url.hostname
        self.port = url.port

    def send_event(self, event, timestamp):
        url = self.__api_url('store/')
        headers = {
            'User-Agent': ('%s/%s' % (sdk_name, sdk_version)),
            'X-Sentry-Auth': ('Sentry sentry_version=7, sentry_timestamp=%s, sentry_key=%s, sentry_secret=%s, sentry_client=%s/%s' % (timestamp.strftime('%s'), self.sentry_key, self.sentry_secret, sdk_name, sdk_version)),
            'Content-Type': 'application/json',
            'Content-Encoding': 'gzip'
        }
        js = json.dumps(event)
        r = requests.post(url, headers=headers, data=gzip_compress(js.encode('utf-8')))
        r.raise_for_status()

    def __api_url(self, op):
        port_part = self.port if (self.port) else ''
        return '%s://%s%s/api/%s/%s' % (self.scheme, self.hostname, port_part, self.project, op)

def parse_commandline_args():
    parser = argparse.ArgumentParser(description='raven-coredump - sentry.io coredump agent')
    parser.add_argument('-c', metavar='limit', dest='core_file_size_soft_limit', help='core file size soft resource limit of crashing process')
    parser.add_argument('-e', metavar='filename', dest='executable_filename', help='executable filename (without path prefix)')
    parser.add_argument('-E', metavar='filepath', dest='executable_filepath', help="pathname of executable, with slashes ('/') replaced by exclamation marks ('!')")
    parser.add_argument('-I', metavar='tid', dest='tid', help='TID of thread that triggered core dump, as seen in the initial PID namespace')
    parser.add_argument('-p', metavar='pid', dest='pid_own_ns', help='PID of dumped process, as seen in the PID namespace in which the process resides')
    parser.add_argument('-P', metavar='pid', dest='pid', help='PID of dumped process, as seen in the initial PID namespace')
    parser.add_argument('--install', action='store_true', help='Install this program in /opt/raven-coredump/bin and configures is as coredump handler')
    parser.add_argument('--rpm-install', metavar='dest_dir', dest='rpm_install', help='Install this program in {dest_dir}/opt/raven-coredump/bin')
    parser.add_argument('--version', action='store_true', help='Print the version of the program')
    args = parser.parse_args(sys.argv)
    if not args.pid:
       args.pid = args.pid_own_ns
    return args

syslog.openlog('raven-coredump')
syslog.syslog('Started with argument: %s' % sys.argv)

try:
    args = parse_commandline_args()
    
    if args.install:
        install_program()
        exit(0)
    
    if args.rpm_install:
        install_program(args.rpm_install)
        exit(0)

    if args.version:
        print(sdk_version)
        exit(0)

    with open('/etc/raven-coredump.conf', 'r') as config_file:
        config = json.load(config_file)
    
    proc_info = {}

    with open('/proc/%s/environ' % args.pid) as file:
        proc_info['environ'] = file.read()

    with open('/proc/%s/status' % args.pid) as file:
        proc_info['status'] = file.read()

    with open('/proc/%s/cmdline' % args.pid) as file:
        proc_info['cmdline'] = file.read()

    with tempfile.NamedTemporaryFile(prefix='raven-coredump-core', delete=True) as f:
        # Using stdin.buffer will suffice with python-3: shutil.copyfileobj(sys.stdin.buffer, f)
        with os.fdopen(sys.stdin.fileno(), 'rb') as input_file:
            shutil.copyfileobj(input_file, f)
        f.flush()
        args.core_filename = f.name
        args.executable_filepath = args.executable_filepath.replace('!', '/')
    
        if config.get('set-opaque-type-resolution-off', True):
            # Workaround gdb performance bug https://bugzilla.redhat.com/show_bug.cgi?id=1401091
            gdb.execute('set opaque-type-resolution off', False, True)
    
        gdb.execute('file %s' % args.executable_filepath, False, True)
    
        gdb.execute('target core %s' % args.core_filename, False, True)
    
        # The initial thread when analyzing a crash is the crashing thread
        crashing_thread = gdb.selected_thread()
    
        # Extract signal information ($_siginfo is thread specific and shall be extracted
        # while the crashing thread is the gdb.selected_thread())
        exitsignal = str(gdb.parse_and_eval('$_exitsignal'))
        exitsignal = exitsignal if (exitsignal != "void") else None
        try:
            signalinfo = str(gdb.parse_and_eval("$_siginfo")) if (exitsignal) else None
        except:
            signalinfo = None
        registers = gdb.execute('info registers', False, True)
    
        timestamp = datetime.datetime.utcnow()
        executable_filename = os.path.basename(args.executable_filepath)
        sentry_event = {
            'event_id': uuid.uuid4().hex,
            'timestamp': timestamp.isoformat(),
            'message': ('Signal %s in %s' % (exitsignal, executable_filename)) if (exitsignal) else ('Core from %s' % executable_filename),
            'platform': 'c',
            'sdk': { 'name': sdk_name, 'version': sdk_version },
            'level': 'fatal',
            'server_name': socket.getfqdn(),
            'contexts': {
                'os': {
                    'name': platform.system(),
                    'version': '-'.join(platform.linux_distribution()[0:2]),
                    'kernel_version': platform.release()
                }
            },
            'extra': {
                'registers': registers
            },
            # FIXME: Include stacktrace of crashing thread directly in event - this works
            # around crashing thread not contributing to grouping. Issue reported:
            # https://github.com/getsentry/sentry/issues/6398
            'stacktrace': thread_json(crashing_thread)['stacktrace'],
            'threads': threads_json(gdb.selected_inferior().threads())
        }
    
        if (exitsignal):
            extra = sentry_event['extra']
            extra['exitsignal'] = exitsignal
            extra['signalinfo'] = signalinfo
    
        print('json: %s' % json.dumps(sentry_event, indent=4))

        crash_folder = config.get('local-crash-folder', '/var/opt/raven-coredump')
        crash_info_filename = '%s-%s-%s.zip' % (executable_filename, args.pid, timestamp.strftime('%Y-%m-%dT%H-%M-%S'))
        crash_info_archive_filepath = os.path.join(crash_folder, crash_info_filename)
        if not os.path.isdir(crash_folder):
            os.makedirs(crash_folder)
        extra['crash_archive'] = crash_info_archive_filepath
        # ZipFile is not a context manager until Python 2.7 so we can't use "with" here
        # because RHEL/CentOS 6 uses gdb with python 2.6.
        archive = zipfile.ZipFile(crash_info_archive_filepath, 'w', zipfile.ZIP_DEFLATED)
        archive.writestr('proc-pid-environ.txt', proc_info['environ'])
        archive.writestr('proc-pid-status.txt', proc_info['status'])
        archive.writestr('proc-pid-cmdline.txt', proc_info['cmdline'])
        if 'extension-script' in config:
            plugin_module = load_python_plugin(config['extension-script'])
            plugin_module.process_core(sentry_event, args, proc_info, archive)
        archive.writestr('sentry-event.json', json.dumps(sentry_event, indent=4))
        archive.close()

        if 'dsn' in config:
            raven_client = RavenClient(config['dsn'])
            if not config.get('include-variables', False):
                prune_vars_from_sentry_event(sentry_event)
            raven_client.send_event(sentry_event, timestamp)
except Exception:
    syslog.syslog(syslog.LOG_ERR | syslog.LOG_USER, 'Exception: %s ' % traceback.format_exception(*sys.exc_info()))
    print('Exception: %s ' % traceback.format_exception(*sys.exc_info()))
    gdb.execute('quit 1', False, True)


syslog.syslog('crash analysis completed')