From 5a8b68a8a2affd15cdba22df54e146f5afe12750 Mon Sep 17 00:00:00 2001 From: Chris Snijder Date: Thu, 29 Mar 2018 16:09:11 +0200 Subject: [PATCH 1/7] Fix issue 48 parse file extensions correctly from config file. --- stapled/core/daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stapled/core/daemon.py b/stapled/core/daemon.py index e64d260..65a2c01 100644 --- a/stapled/core/daemon.py +++ b/stapled/core/daemon.py @@ -93,7 +93,7 @@ def __init__(self, **kwargs): 'haproxy_socket_mapping', None ) self.file_extensions = kwargs.pop('file_extensions') - self.file_extensions.replace(" ", "").split(",") + self.file_extensions = self.file_extensions.replace(" ", "").split(",") self.renewal_threads = kwargs.pop('renewal_threads') self.refresh_interval = kwargs.pop('refresh_interval') self.minimum_validity = kwargs.pop('minimum_validity') From 3d8d52123c6864372d425ef76163ae3142c75161 Mon Sep 17 00:00:00 2001 From: Chris Snijder Date: Thu, 29 Mar 2018 16:16:34 +0200 Subject: [PATCH 2/7] Fix broken exception handling in python 2, version script still contains some instances but it doesn't run in production. --- stapled/core/certfinder.py | 7 +++++-- stapled/core/excepthandler.py | 14 ++++++++++++-- stapled/core/stapleadder.py | 2 +- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/stapled/core/certfinder.py b/stapled/core/certfinder.py index 474716d..842bf45 100644 --- a/stapled/core/certfinder.py +++ b/stapled/core/certfinder.py @@ -28,6 +28,7 @@ import fnmatch import os import stapled +import errno from stapled.core.excepthandler import stapled_except_handle from stapled.core.taskcontext import StapleTaskContext from stapled.core.certmodel import CertModel @@ -165,13 +166,15 @@ def _find_new_certs(self, paths, cert_path=None): dirs = [] try: dirs = os.listdir(path) - except NotADirectoryError: + except (OSError, IOError) as exc: # If a path is actually a file we can still use it.. - if os.path.isfile(path): + if exc.errno == errno.ENOTDIR and os.path.isfile(path): LOG.debug("%s may be a single file", path) # This will allow us to use our usual iteration. dirs = [os.path.basename(path)] path = os.path.dirname(path) + else: + raise exc for entry in dirs: entry = os.path.join(path, entry) if os.path.isdir(entry): diff --git a/stapled/core/excepthandler.py b/stapled/core/excepthandler.py index 47b4269..0120fe6 100755 --- a/stapled/core/excepthandler.py +++ b/stapled/core/excepthandler.py @@ -35,6 +35,7 @@ import datetime import logging import os +import errno import traceback import configargparse from stapled.core.exceptions import OCSPBadResponse @@ -150,8 +151,17 @@ def stapled_except_handle(ctx=None): "entries", err_count, len_ocsp_urls ) - except PermissionError as exc: - LOG.critical(exc) + except (IOError, OSError) as exc: + if exc.errno==errno.EPERM or exc.errno==errno.EACCES: + reason = "Permission error" + elif exc.errno==errno.ENOENT: + reason = "File not found error" + elif isinstance(exc, IOError): + reason = "I/O Error" + else: + reason = "OS Error" + + LOG.critical("{}: {}".format(reason, str(exc))) # the show must go on.. except Exception as exc: # pylint: disable=broad-except diff --git a/stapled/core/stapleadder.py b/stapled/core/stapleadder.py index b0f20a9..1103ec0 100644 --- a/stapled/core/stapleadder.py +++ b/stapled/core/stapleadder.py @@ -97,7 +97,7 @@ def _open_socket(self, path): ) try: sock.connect(path) - except (FileNotFoundError, PermissionError) as exc: + except (OSError, IOError) as exc: raise stapled.core.exceptions.SocketError( "Could not initialize StapleAdder with socket {}: {}".format( path, From 0613183e09db0e64dcdd48e63a68763b244e24f3 Mon Sep 17 00:00:00 2001 From: Chris Snijder Date: Thu, 29 Mar 2018 18:11:00 +0200 Subject: [PATCH 3/7] Handle opening, re-opening and sending of commands to sockets better. --- stapled/core/stapleadder.py | 154 +++++++++++++++++++++++------------- 1 file changed, 98 insertions(+), 56 deletions(-) diff --git a/stapled/core/stapleadder.py b/stapled/core/stapleadder.py index b0f20a9..f1461d8 100644 --- a/stapled/core/stapleadder.py +++ b/stapled/core/stapleadder.py @@ -46,6 +46,12 @@ class StapleAdder(threading.Thread): #: the base64 encoded OCSP staple OCSP_ADD = 'set ssl ocsp-response {}' + #: Predefines commands to send to sockets just after opening them. + CONNECT_COMMANDS = [ + "prompt", + "set timeout cli {}".format(SOCKET_TIMEOUT) + ] + def __init__(self, *args, **kwargs): """ Initialise the thread and its parent :class:`threading.Thread`. @@ -74,10 +80,33 @@ def __init__(self, *args, **kwargs): for paths in self.haproxy_socket_mapping.values(): for path in paths: with stapled_except_handle(): - self.socks[path] = self._open_socket(path) + self._open_socket(path) super(StapleAdder, self).__init__(*args, **kwargs) + def _re_open_socket(self, path): + """ + Re-open socket located at path, and return the socket. + Closes open sockets and wraps appropriate logging arount the + ``_open_socket`` method. + + :param str path: A valid HAProxy socket path. + :return socket.socket: An open socket. + :raises :exc:stapled.core.exceptions.SocketError: when the socket can + not be opened. + """ + # Try to re-open the socket. If that doesn't work, that + # will raise a :exc:`~stapled.core.exceptions.SocketError` + LOG.info("Re-opening socket %s", path) + try: + sock = self.socks[path] + sock.close() + except (KeyError, UnboundLocalError): + # Socket not openend, no need to close anything. + pass + # Open socket again.. + return self._open_socket(path) + def _open_socket(self, path): """ Open socket located at path, and return the socket. @@ -87,27 +116,30 @@ def _open_socket(self, path): socket. :param str path: A valid HAProxy socket path. - + :return socket.socket: An open socket. :raises :exc:stapled.core.exceptions.SocketError: when the socket can not be opened. """ - sock = socket.socket( - socket.AF_UNIX, - socket.SOCK_STREAM - ) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: sock.connect(path) - except (FileNotFoundError, PermissionError) as exc: + result = [] + for command in self.CONNECT_COMMANDS: + result.extend(self._send(sock, command) + # Results (index 1) come per path (index 0), we need only results + result = [res[1] for res in result] + # Indented and on separate lines or None if an empty list + result = "\n\t{}".format("\n\t".join(result)) if result else "None" + LOG.debug("Opened prompt with result: %s", result) + self.socks[path] = sock + return sock + except (BrokenPipeError, OSError, IOError) as exc: raise stapled.core.exceptions.SocketError( "Could not initialize StapleAdder with socket {}: {}".format( path, exc ) ) - result = self.send(path, "prompt") - LOG.debug("Opened prompt with result: '%s'", result) - result = self.send(path, "set timeout cli {}".format(SOCKET_TIMEOUT)) - return sock def __del__(self): """Close the sockets on exit.""" @@ -146,12 +178,12 @@ def add_staple(self, model): """ command = self.OCSP_ADD.format(model.ocsp_staple.base64) LOG.debug("Setting OCSP staple with command '%s'", command) - path = self.haproxy_socket_mapping[model.cert_path] - if path is None: + paths = self.haproxy_socket_mapping[model.cert_path] + if not paths: LOG.debug("No socket set for %s", model.filename) return - responses = self.send(path, command) - for response in responses: + responses = self.send(paths, command) + for path, response in responses: if response != 'OCSP Response updated!': raise stapled.core.exceptions.StapleAdderBadResponse( "Bad HAProxy response: '{}' from socket {}".format( @@ -160,15 +192,54 @@ def add_staple(self, model): ) ) - def send(self, paths, command): + def _send(self, sock, command): """ - Send the command through the socket at ``path``. + Send the command through the ``socket`` and handle response. - :param list paths: The path(s) to the socket(s) which should already - be open. + :param list sock: An already opened socket. :param str command: String with the HAProxy command. For a list of possible commands, see the `haproxy documentation`_ + :return list: List of tuples containing path and response from HAProxy. + :raises IOError if an error occurs and it's not errno.EAGAIN or + errno.EINTR + + .. _haproxy documentation: + http://haproxy.tech-notes.net/9-2-unix-socket-commands/ + + """ + sock.sendall((command + "\n").encode()) + buff = StringIO() + # Get new response. + while True: + try: + chunk = sock.recv(SOCKET_BUFFER_SIZE) + if chunk: + decoded_chunk = chunk.decode('ascii') + buff.write(decoded_chunk) + # TODO: Find out what happens if several threads + # are talking to HAProxy on this socket + if '> ' in decoded_chunk: + break + else: + break + except IOError as err: + if err.errno not in (errno.EAGAIN, errno.EINTR): + raise + + # Strip *all* \n, > and space characters from the end + response = buff.getvalue().strip('\n> ') + buff.close() + return response + def send(self, paths, command): + """ + Send the command through the sockets at ``paths``. + + :param str|list paths: The path(s) to the socket(s) which should + already be open. + :param str command: String with the HAProxy command. For a list of + possible commands, see the `haproxy documentation`_ + :return list: List of tuples containing path and response from HAProxy. :raises IOError if an error occurs and it's not errno.EAGAIN or errno.EINTR @@ -194,47 +265,18 @@ def send(self, paths, command): # raise # else: # break - - # Send command with stapled_except_handle(): responses = [] + if not isinstance(paths, (list, tuple)): + paths = [paths] for path in paths: try: sock = self.socks[path] - sock.sendall((command + "\n").encode()) + response = self._send(sock, "{}\n".format(command)) except (BrokenPipeError, KeyError): - # Try to re-open the socket. If that doesn't work, that - # will raise a :exc:`~stapled.core.exceptions.SocketError` - LOG.info("Re-opening socket %s", path) - try: - sock.close() - except UnboundLocalError: # sock is not yet defined. - pass - sock = self.socks[path] = self._open_socket(path) - # Try again, if this results in a BrokenPipeError *again*, - # it will be caught by stapled_except_handle - sock.sendall((command + "\n").encode()) - buff = StringIO() - # Get new response. - while True: - try: - chunk = sock.recv(SOCKET_BUFFER_SIZE) - if chunk: - d_chunk = chunk.decode('ascii') - buff.write(d_chunk) - # TODO: Find out what happens if several threads - # are talking to HAProxy on this socket - if '> ' in d_chunk: - break - else: - break - except IOError as err: - if err.errno not in (errno.EAGAIN, errno.EINTR): - raise - - # Strip *all* \n, > and space characters from the end - response = buff.getvalue().strip('\n> ') + sock = self._re_open_socket(path) + response = self._send(sock, "{}\n".format(command)) + LOG.debug("Received HAProxy response '%s'", response) - responses.append() - buff.close() - return responses + responses.append((path, response)) + return responses From 955cc73f51d2ef38c3629b309f22d108d11db7bd Mon Sep 17 00:00:00 2001 From: Chris Snijder Date: Thu, 29 Mar 2018 18:21:15 +0200 Subject: [PATCH 4/7] Add missing paranthesis. --- stapled/core/stapleadder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stapled/core/stapleadder.py b/stapled/core/stapleadder.py index f1461d8..1cba7e7 100644 --- a/stapled/core/stapleadder.py +++ b/stapled/core/stapleadder.py @@ -125,7 +125,7 @@ def _open_socket(self, path): sock.connect(path) result = [] for command in self.CONNECT_COMMANDS: - result.extend(self._send(sock, command) + result.extend(self._send(sock, command)) # Results (index 1) come per path (index 0), we need only results result = [res[1] for res in result] # Indented and on separate lines or None if an empty list From acbe6afb154058d3f4074ad90d21a577fb6010ed Mon Sep 17 00:00:00 2001 From: Chris Snijder Date: Thu, 29 Mar 2018 18:48:39 +0200 Subject: [PATCH 5/7] Add changelog and bump version number to 0.6. --- debian/changelog | 11 ++++++++++- stapled/version.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 358f8cd..8d5ecf6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +stapled (0.6) stretch; urgency=low + + * Improvement: Remove some spaces from log lines. + * FIX: Broken exception handling in python 2 version. + * FIX: Parsing file extensions incorrectly from config file. + * FIX: Opening, re-opening and sending of commands to sockets broken.n + + -- Chris Snijder Thu, 29 Mar 2018 18:46:44 +0200 + stapled (0.5) stretch; urgency=low * Improved consistency in source code. @@ -6,7 +15,7 @@ stapled (0.5) stretch; urgency=low sockets and certificate directories. * It is now possible to specify single certificate aside from entire directories. - * DEPRECATED: --directories was superseded by --cert-paths. + * DEPRECATED: --directories was superseded by --cert-paths. -- Chris Snijder Wed, 7 Mar 2018 16:35:22 +0100 diff --git a/stapled/version.py b/stapled/version.py index 8e5be70..9973d86 100644 --- a/stapled/version.py +++ b/stapled/version.py @@ -1,3 +1,3 @@ -__version__ = '0.5' +__version__ = '0.6' __app_name__ = 'stapled' __debian_version__ = 'stretch' From 94e51da50b412f1b52c8edfe25bb0e07b9417493 Mon Sep 17 00:00:00 2001 From: Chris Snijder Date: Fri, 30 Mar 2018 11:08:35 +0200 Subject: [PATCH 6/7] Fix typo --- stapled/core/stapleadder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stapled/core/stapleadder.py b/stapled/core/stapleadder.py index 1cba7e7..bece607 100644 --- a/stapled/core/stapleadder.py +++ b/stapled/core/stapleadder.py @@ -87,7 +87,7 @@ def __init__(self, *args, **kwargs): def _re_open_socket(self, path): """ Re-open socket located at path, and return the socket. - Closes open sockets and wraps appropriate logging arount the + Closes open sockets and wraps appropriate logging around the ``_open_socket`` method. :param str path: A valid HAProxy socket path. From 7721b53bd3303ccd665c4389fb7e4bed4dd82d2b Mon Sep 17 00:00:00 2001 From: Chris Snijder Date: Fri, 30 Mar 2018 11:47:53 +0200 Subject: [PATCH 7/7] BrokenPipeError does not exist in Python2.7 --- stapled/core/stapleadder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stapled/core/stapleadder.py b/stapled/core/stapleadder.py index bece607..44ba004 100644 --- a/stapled/core/stapleadder.py +++ b/stapled/core/stapleadder.py @@ -133,7 +133,7 @@ def _open_socket(self, path): LOG.debug("Opened prompt with result: %s", result) self.socks[path] = sock return sock - except (BrokenPipeError, OSError, IOError) as exc: + except (OSError, IOError) as exc: raise stapled.core.exceptions.SocketError( "Could not initialize StapleAdder with socket {}: {}".format( path,