multilb-sanity-check

#!/usr/bin/env python3
# multilb-sanity-check (part of ossobv/vcutil) // wdoekes/2022 // Public Domain
#
# Internal tool. Used for sanity checks on load balancer machines.
#
# Checks:
# - that all running nginx/haproxy have a systemd enabled symlink
# - no processes are (still) running that should have a config
# - that the processed appear reloaded after new certs have been uploaded
#
# Usage on machine with one or more nginx/haproxy instances configured
# in a certain way:
#
#    $ multilb-sanity-check
#
from base64 import b16decode
from os import listdir, path, readlink
from socket import AF_INET, AF_INET6, inet_ntop, ntohl
from time import time


class ProcNetBase:
    def __init__(self):
        if ntohl(0x12345678) == 0x78563412:
            self._bswap = (lambda x: x[::-1])
        else:
            self._bswap = (lambda x: x)

        with open(path.join('/proc/net', self.FILE)) as fp:
            self.data = fp.read()

    def get_listen(self, ino):
        # # cat /proc/net/tcp | head
        #   sl  local_address rem_address   st ... timeout inode
        #    0: 00000000:00B3 00000000:0000 0A ...       0 19819 1 ffff90...
        #    1: 3500007F:0035 00000000:0000 0A ...       0 34963 1 ffff90...
        if not hasattr(self, '_get_listen'):
            listen = {}
            lines = self.data.split('\n')
            lo_addr_pos = lines[0].index('local_address')
            try:
                rem_addr_pos = lines[0].index('rem_address')
            except ValueError:
                rem_addr_pos = lines[0].index('remote_address')
            inode_pos = lines[0].index('inode')
            for line in lines[1:]:
                if line.startswith(self.REMOTE_LISTEN, rem_addr_pos):
                    local = line[lo_addr_pos:rem_addr_pos - 1]
                    inode = int(line[inode_pos:].split(' ', 1)[0])
                    assert inode not in listen, (inode, local, listen)
                    ip, port = local.split(':')
                    ip = inet_ntop(
                        self.ADDRESS_FAMILY,
                        self._bswap(b16decode(ip)))
                    port = int(port, 16)
                    listen[inode] = (ip, port)
            self._get_listen = listen

        if ino in self._get_listen:
            return [self._get_listen[ino]]
        return []


class ProcNetTcp(ProcNetBase):
    FILE = 'tcp'
    ADDRESS_FAMILY = AF_INET
    REMOTE_LISTEN = '00000000:0000'


class ProcNetTcp6(ProcNetBase):
    FILE = 'tcp6'
    ADDRESS_FAMILY = AF_INET6
    REMOTE_LISTEN = '00000000000000000000000000000000:0000'


class ProcNetUdp(ProcNetBase):
    FILE = 'udp'
    ADDRESS_FAMILY = AF_INET
    REMOTE_LISTEN = '00000000:0000'


class ProcNetUdp6(ProcNetBase):
    FILE = 'udp6'
    ADDRESS_FAMILY = AF_INET6
    REMOTE_LISTEN = '00000000000000000000000000000000:0000'


class NetManager:
    @property
    def tcp(self):
        if not hasattr(self, '_tcp'):
            self._tcp = ProcNetTcp()
        return self._tcp

    @property
    def tcp6(self):
        if not hasattr(self, '_tcp6'):
            self._tcp6 = ProcNetTcp6()
        return self._tcp6

    @property
    def udp(self):
        if not hasattr(self, '_udp'):
            self._udp = ProcNetUdp()
        return self._udp

    @property
    def udp6(self):
        if not hasattr(self, '_udp6'):
            self._udp6 = ProcNetUdp6()
        return self._udp6

    def get_listen(self, ino):
        return (
            self.tcp.get_listen(ino) +
            self.tcp6.get_listen(ino) +
            self.udp.get_listen(ino) +
            self.udp6.get_listen(ino))


class ProcManager:
    def __init__(self):
        self._procs = {}

    def add(self, proc):
        self._procs[proc.pid] = proc

    def get(self, pid):
        if pid is None:
            return None
        assert isinstance(pid, int), pid
        return self._procs[pid]

    def has_children(self, pid):
        for proc in self._procs.values():
            if proc.ppid == pid:
                return True
        return False


class Proc:
    def __init__(self, procdir, procmgr, netmgr):
        self.procdir = procdir
        self.procmgr = procmgr
        self.netmgr = netmgr
        self.spid = procdir.rsplit('/', 1)[-1]
        self.pid = int(self.spid)
        try:
            self.exe = readlink(path.join(procdir, 'exe'))
        except FileNotFoundError:
            self.exe = None

        self.procmgr.add(self)

    @property
    def cmdline(self):
        if not hasattr(self, '_cmdline'):
            with open(path.join(self.procdir, 'cmdline')) as fp:
                self._cmdline = fp.read().split('\0')
        return self._cmdline

    @property
    def config(self):
        if not hasattr(self, '_config'):
            self._config = self.get_config()
        return self._config

    @property
    def ssl_time(self):
        """
        Get the newest SSL cert time from the config

        So we can compare that to when the process was started. And if the
        process is older, we have likely forgot a reload.
        """
        if hasattr(self, '_ssl_time'):
            return self._ssl_time

        config = self.config
        if not config:
            self._ssl_time = None
            return None

        with open(config) as fp:
            config = fp.read()

        possible_certs = []

        if self.exe == '/usr/sbin/haproxy':
            lines = [
                line.strip() for line in config.split('\n')
                if not line.lstrip().startswith('#')]
            for line in lines:
                parts = line.rstrip().split()
                for optkey, optval in zip(parts[:-1], parts[1:]):
                    if optkey == 'crt':
                        possible_certs.append(optval)
        elif self.exe == '/usr/sbin/nginx':
            lines = [
                line.strip() for line in config.split('\n')
                if not line.lstrip().startswith('#')]
            for line in lines:
                if line.startswith('ssl_certificate'):
                    parts = line.split()
                    if parts[0] == 'ssl_certificate':
                        possible_certs.append(parts[1].rstrip(';'))
        else:
            raise NotImplementedError((self.exe, self.config))

        if not possible_certs:
            return None

        # Take newest stat from all files.
        self._ssl_time = max(path.getmtime(i) for i in possible_certs)
        return self._ssl_time

    @property
    def stat(self):
        if not hasattr(self, '_stat'):
            with open(path.join(self.procdir, 'stat')) as fp:
                stat = fp.read().split(' ')
            # Pad, so we can count 1-based, like the proc(5) manual does.
            self._stat = tuple(['0-pad'] + stat)
            assert self._stat[1] == self.spid, (self._stat, self.spid)
        return self._stat

    @property
    def ppid(self):
        """
        proc(5): (4) ppid %d The PID of the parent of this process.
        """
        return int(self.stat[4])

    @property
    def start_time(self):
        """
        proc(5): (22) starttime  %llu

        The time the process started after system boot. The value is
        expressed in clock ticks (divide by sysconf(_SC_CLK_TCK)).
        """
        if not hasattr(self, '_start_time'):
            starttime = int(self.stat[22]) / _SC_CLK_TICK  # seconds after boot
            self._start_time = SYSTEM_STARTED_AT + starttime
        return self._start_time

    @property
    def systemd_enabled_file(self):
        return '/etc/systemd/system/multi-user.target.wants/{}'.format(
            self.systemd_service)

    @property
    def systemd_service(self):
        config = self.config.split('/')
        lastdir = config[-2]
        file = config[-1].rsplit('.', 1)[0]
        suffix = '{}-{}'.format(lastdir, file)
        service = self.exe.rsplit('/', 1)[-1]
        return '{}@{}.service'.format(service, suffix)

    @property
    def listenip_any(self):
        ips = self.listenip
        if len(ips) >= 1:
            return ips[0]
        # MISSING_IP = IRRELEVANT_IP = 'IP.ADD.RE.SS'
        return '-'

    @property
    def listenip(self):
        if not hasattr(self, '_listenip'):
            self._listenip = self.get_listenip()
        return self._listenip

    @property
    def open_inodes(self):
        if not hasattr(self, '_open_inodes'):
            self._open_inodes = self.get_open_inodes()
        return self._open_inodes

    def is_parent(self):
        return self.procmgr.has_children(self.pid)

    def get_open_inodes(self):
        """
        Read /proc/PID/fdinfo/* and get ino:HEX.
        """
        fdinfo = path.join(self.procdir, 'fdinfo')
        fds = [fd for fd in listdir(fdinfo) if fd.isdigit()]
        ret = set()
        for fd in fds:
            # $ cat /proc/711079/fdinfo/4
            # pos:     0
            # flags:  02
            # mnt_id: 15
            # tfd:     9 events: 2019 data: 9 pos:0 ino:47e0beac sdev:9
            # tfd:    10 events: 2019 data: a pos:0 ino:47e0bead sdev:9
            with open(path.join(fdinfo, fd)) as fp:
                data = fp.read()
            if 'ino:' not in data:
                continue
            lines = data.split('\n')
            for line in lines:
                if line.startswith('tfd:'):
                    hex_ino = line.split('ino:', 1)[1].lstrip().split()[0]
                    ret.add(int(hex_ino, 16))
        return ret

    def get_listenip(self):
        listens = []
        for ino in self.open_inodes:
            listens.extend(self.netmgr.get_listen(ino))
        return tuple(sorted(set(i[0] for i in listens)))

        # This is the olden way, which was slow:
        # try:
        #     lsof = check_output([
        #         'lsof', '-Pan', '-i', '-p', self.spid]).decode()
        # except Exception:
        #     return ()
        # lsof = lsof.split('\n')[1:]
        # # COMMAND   PID    USER   FD   TYPE DEVICE S/O NODE NAME
        # # haproxy 21174 haproxy    5u  IPv4 302504 0t0  TCP <ip>:80 (LISTEN)
        # # haproxy 21174 haproxy    6u  IPv4 302505 0t0  TCP <ip>:443 (LISTEN)
        # # haproxy 21174 haproxy    7u  IPv4 302506 0t0  TCP <ip>:1984 (LISTE.
        # listens = [
        #     val[8] for val in [line.split() for line in lsof]
        #     if len(val) >= 10 and val[9] == '(LISTEN)']
        # ips = set([val.split(':', 1)[0] for val in listens])
        # return tuple(sorted(ips))

    def get_config(self):
        if self.exe == '/usr/sbin/haproxy':
            for optkey, optval in zip(self.cmdline[:-1], self.cmdline[1:]):
                if optkey == '-f':
                    return optval
            assert False, self.cmdline
        elif self.exe == '/usr/sbin/nginx':
            # nginx cmdline is a string, not NUL-joined
            cmdline = ' '.join(self.cmdline).split(' ')
            for optkey, optval in zip(cmdline[:-1], cmdline[1:]):
                if optkey == '-c':
                    return optval
            # nginx has workers that do not list their info: get the parent
            parent = self.get_same_parent()
            if parent:
                return parent.config
            assert False, cmdline
        return None

    def get_parent(self):
        return self.procmgr.get(self.ppid)

    def get_same_parent(self):
        parent = self.get_parent()
        if parent and parent.exe == self.exe:
            return parent
        return None


def system_start_time():
    now = time()
    with open('/proc/uptime') as fp:
        uptime, idletime = [float(i) for i in fp.read().strip().split()]
    return (now - uptime)


SYSTEM_STARTED_AT = system_start_time()
_SC_CLK_TICK = 100  # getconf CLK_TCK


proc_manager = ProcManager()
net_manager = NetManager()

all_procs = [
    Proc(
        '/proc/{pid}'.format(pid=pid), procmgr=proc_manager,
        netmgr=net_manager)
    for pid in listdir('/proc') if pid.isdigit()]

procs_with_config = [proc for proc in all_procs if proc.config]


for proc in sorted(procs_with_config, key=(lambda x: (x.config, x.pid))):
    # Check for processes that were started BEFORE the newest SSL certificate
    # found in config was updated. For nginx and haproxy, the children of the
    # main process are re-executed, so they should always be younger than the
    # certificate file.
    if proc.ssl_time and proc.start_time < proc.ssl_time:
        if not proc.is_parent():
            print(
                '{ip:15s} {proc.exe} (certificates) {proc.systemd_service!r} '
                'certificate file newer than running process'.format(
                    proc=proc, ip=proc.listenip_any))

    # Check that a configuration exists. IF there is no mention in
    # systemd_enabled_file, then we're not auto-starting on boot.
    if path.exists(proc.config):
        if not path.exists(proc.systemd_enabled_file):
            print(
                '{ip:15s} {proc.exe} (systemd) {proc.systemd_enabled_file!r} '
                'missing'.format(proc=proc, ip=proc.listenip_any))

    # If we have a listenip, but no configuration file, then we're likely
    # running a stale process.
    if not path.exists(proc.config) and proc.listenip:  # listenip is expensive
        assert len(proc.listenip) == 1, (proc.config, proc.pid, proc.listenip)
        print(
            '{ip:15s} {proc.exe} (config) {proc.config!r} '
            'missing: kill {proc.pid}'.format(proc=proc, ip=proc.listenip_any))
        if path.exists(proc.systemd_enabled_file):
            print(
                '{ip:15s} {proc.exe} (systemd) {proc.systemd_enabled_file!r} '
                'exists'.format(proc=proc, ip=proc.listenip_any))