diff --git a/README.rst b/README.rst index 5e0eed9f..ff727b1a 100644 --- a/README.rst +++ b/README.rst @@ -29,7 +29,7 @@ Installation pip install parallel-ssh -As of version ``0.93.0`` pip version >= ``6.0.0`` is required for Python 2.6 compatibility with newer versions of gevent which have dropped 2.6 support. This limitation will be removed post ``1.0.0`` releases which will deprecate ``2.6`` support. +As of version ``0.93.0`` pip version >= ``6.0.0`` is required for Python 2.6 compatibility with latest versions of gevent which have dropped 2.6 support. This limitation will be removed post ``1.0.0`` releases which will deprecate ``2.6`` support. To upgrade ``pip`` run the following - use of ``virtualenv`` is recommended so as not to override system provided packages:: @@ -77,11 +77,11 @@ Exit codes become available once stdout/stderr is iterated on or ``client.join(o 0 0 -The client's join function can be used to block and wait for all parallel commands to finish *if output is not needed*. :: +The client's ``join`` function can be used to block and wait for all parallel commands to finish:: client.join(output) -Similarly, if only exit codes are needed but not output :: +Similarly, exit codes are available after ``client.join`` is called:: output = client.run_command('exit 0') # Block and gather exit codes. Output variable is updated in-place @@ -89,6 +89,10 @@ Similarly, if only exit codes are needed but not output :: print(output[client.hosts[0]]['exit_code']) 0 +.. note:: + + In versions prior to ``1.0.0`` only, ``client.join`` would consume standard output. + There is also a built in host logger that can be enabled to log output from remote hosts. The helper function ``pssh.utils.enable_host_logger`` will enable host logging to stdout, for example :: import pssh.utils @@ -128,6 +132,30 @@ On the other end of the spectrum, long lived remote commands that generate *no* Output *generation* is done remotely and has no effect on the event loop until output is gathered - output buffers are iterated on. Only at that point does the event loop need to be held. +******** +SFTP/SCP +******** + +SFTP is supported (SCP version 2) natively, no ``scp`` command required. + +For example to copy a local file to remote hosts in parallel:: + + from pssh import ParallelSSHClient, utils + from gevent import joinall + + utils.enable_logger(utils.logger) + hosts = ['myhost1', 'myhost2'] + client = ParallelSSHClient(hosts) + greenlets = client.copy_file('../test', 'test_dir/test') + joinall(greenlets, raise_error=True) + + Copied local file ../test to remote destination myhost1:test_dir/test + Copied local file ../test to remote destination myhost2:test_dir/test + +There is similar capability to copy remote files to local ones suffixed with the host's name with the ``copy_remote_file`` function. + +Directory recursion is supported in both cases via the ``recurse`` parameter - defaults to off. + ************************** Frequently asked questions ************************** @@ -138,15 +166,15 @@ Frequently asked questions :A: In short, the tools are intended for different use cases. - ``ParallelSSH`` satisfies uses cases for a parallel SSH client library that scales well over hundreds to hundreds of thousands of hosts - per `Design And Goals`_ - a use case that is very common on cloud platforms and virtual machine automation . It would be best used where it is a good fit for the use case. + ``ParallelSSH`` satisfies uses cases for a parallel SSH client library that scales well over hundreds to hundreds of thousands of hosts - per `Design And Goals`_ - a use case that is very common on cloud platforms and virtual machine automation. It would be best used where it is a good fit for the use case at hand. - Fabric and tools like it on the other hand are not well suited to such use cases, for many reasons, performance and differing design goals in particular. The similarity is only that these tools also make use of SSH to run their commands. + Fabric and tools like it on the other hand are not well suited to such use cases, for many reasons, performance and differing design goals in particular. The similarity is only that these tools also make use of SSH to run commands. ``ParallelSSH`` is in other words well suited to be the SSH client tools like Fabric and Ansible and others use to run their commands rather than a direct replacement for. By focusing on providing a well defined, lightweight - actual code is a few hundred lines - library, ``ParallelSSH`` is far better suited for *run this command on X number of hosts* tasks for which frameworks like Fabric, Capistrano and others are overkill and unsuprisignly, as it is not what they are for, ill-suited to and do not perform particularly well with. - Fabric and tools like it are high level deployment frameworks - as opposed to general purpose libraries - for building deployment tasks to perform on hosts matching a role with task chaining and a DSL like syntax and are primarily intended for command line use for which the framework is a good fit for - very far removed from an SSH client library. + Fabric and tools like it are high level deployment frameworks - as opposed to general purpose libraries - for building deployment tasks to perform on hosts matching a role with task chaining, a DSL like syntax and are primarily intended for command line use for which the framework is a good fit for - very far removed from an SSH client *library*. Fabric in particular is a port of `Capistrano `_ from Ruby to Python. Its design goals are to provide a faithful port of Capistrano with its `tasks` and `roles` framework to python with interactive command line being the intended usage. @@ -205,28 +233,3 @@ Frequently asked questions :A: There is a public `ParallelSSH Google group `_ setup for this purpose - both posting and viewing are open to the public. - - -******** -SFTP/SCP -******** - -SFTP is supported (SCP version 2) natively, no ``scp`` command required. - -For example to copy a local file to remote hosts in parallel:: - - from pssh import ParallelSSHClient, utils - from gevent import joinall - - utils.enable_logger(utils.logger) - hosts = ['myhost1', 'myhost2'] - client = ParallelSSHClient(hosts) - greenlets = client.copy_file('../test', 'test_dir/test') - joinall(greenlets, raise_error=True) - - Copied local file ../test to remote destination myhost1:test_dir/test - Copied local file ../test to remote destination myhost2:test_dir/test - -There is similar capability to copy remote files to local ones suffixed with the host's name with the ``copy_remote_file`` function. - -Directory recursion is supported in both cases - defaults to off. diff --git a/pssh/__init__.py b/pssh/__init__.py index 8e133bfe..83baec21 100644 --- a/pssh/__init__.py +++ b/pssh/__init__.py @@ -17,17 +17,16 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -"""Asynchronous parallel SSH library +"""Asynchronous parallel SSH client library. -parallel-ssh uses asychronous network requests - there is *no* multi-threading nor multi-processing used. +Run SSH commands over many - hundreds/hundreds of thousands - number of servers asynchronously and with minimal system load on the client host. -This is a *requirement* for commands on many (hundreds/thousands/hundreds of thousands) of hosts which would grind a system to a halt simply by having so many processes/threads all wanting to execute if done with multi-threading/processing. +New users should start with :py:func:`pssh.pssh_client.ParallelSSHClient.run_command` -The `libev event loop library `_ is utilised on nix systems. Windows is not supported. - -See :mod:`pssh.ParallelSSHClient` and :mod:`pssh.SSHClient` for class documentation. +See also :py:class:`pssh.ParallelSSHClient` and :py:class:mod:`pssh.SSHClient` for class documentation. """ +import logging from ._version import get_versions __version__ = get_versions()['version'] del get_versions @@ -36,7 +35,6 @@ from .utils import enable_host_logger from .exceptions import UnknownHostException, \ AuthenticationException, ConnectionErrorException, SSHException -import logging host_logger = logging.getLogger('pssh.host_logger') logger = logging.getLogger('pssh') diff --git a/pssh/agent.py b/pssh/agent.py index 869b12f9..9e42e4a3 100644 --- a/pssh/agent.py +++ b/pssh/agent.py @@ -22,6 +22,7 @@ class SSHAgent(paramiko.agent.AgentSSH): supplying an SSH agent""" def __init__(self): + paramiko.agent.AgentSSH.__init__(self) self._conn = None self.keys = [] diff --git a/pssh/constants.py b/pssh/constants.py index b6f20196..d6bb9609 100644 --- a/pssh/constants.py +++ b/pssh/constants.py @@ -15,5 +15,6 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +"""Constants definitions for pssh package""" DEFAULT_RETRIES = 3 diff --git a/pssh/pssh_client.py b/pssh/pssh_client.py index aa08ae2a..b66e3b87 100644 --- a/pssh/pssh_client.py +++ b/pssh/pssh_client.py @@ -23,13 +23,13 @@ del sys.modules['threading'] from gevent import monkey monkey.patch_all() +import string +import random import logging + import gevent.pool import gevent.hub gevent.hub.Hub.NOT_ERROR = (Exception,) -import warnings -import string -import random from .exceptions import HostArgumentException from .constants import DEFAULT_RETRIES @@ -106,8 +106,10 @@ def __init__(self, hosts, :param host_config: (Optional) Per-host configuration for cases where \ not all hosts use the same configuration values. :type host_config: dict - :param channel_timeout: (Optional) Time in seconds before an SSH operation \ - times out. + :param channel_timeout: (Optional) Time in seconds before reading from \ + an SSH channel times out. For example with channel timeout set to one, \ + trying to immediately gather output from a command producing no output \ + for more than one second will timeout. :type channel_timeout: int :param allow_agent: (Optional) set to False to disable connecting to \ the SSH agent @@ -156,7 +158,8 @@ def __init__(self, hosts, from remote commands on hosts as it comes in. This allows for stdout to be automatically logged without having to - print it serially per host. + print it serially per host. :mod:`pssh.utils.host_logger` is a standard + library logger and may be configured to log to anywhere else. .. code-block:: python @@ -185,7 +188,7 @@ def __init__(self, hosts, * Iterating over stdout/stderr to completion * Calling ``client.join(output)`` - is necessary to cause `parallel-ssh` to wait for commands to finish and + is necessary to cause ``parallel-ssh`` to wait for commands to finish and be able to gather exit codes. .. note :: @@ -211,13 +214,13 @@ def __init__(self, hosts, which returns ``True`` if command has finished. - Either iterating over stdout/stderr or `client.join(output)` will cause exit + Either iterating over stdout/stderr or ``client.join(output)`` will cause exit codes to become available in output without explicitly calling `get_exit_codes`. Use ``client.join(output)`` to block until all commands have finished and gather exit codes at same time. - However, note that ``client.join(output)`` will consume stdout/stderr. + In versions prior to ``1.0.0`` only, ``client.join`` would consume output. **Exit code retrieval** @@ -296,7 +299,7 @@ def __init__(self, hosts, output = client.run_command('ls -ltrh /tmp/aasdfasdf') client.join(output) - :netstat: ``tcp 0 0 127.0.0.1:53054 127.0.0.1:22 ESTABLISHED`` + :netstat: ``tcp 0 0 127.0.0.1:53054 127.0.0.1:22 ESTABLISHED`` Connection remains active after commands have finished executing. Any \ additional commands will use the same connection. @@ -362,9 +365,12 @@ def run_command(self, *args, **kwargs): to True - use shell defined in user login to run command string :type use_shell: bool :param use_pty: (Optional) Enable/Disable use of pseudo terminal \ - emulation. This is required in vast majority of cases, exception \ - being where a shell is not used and/or stdout/stderr/stdin buffers \ - are not required. Defaults to ``True`` + emulation. Disabling it will prohibit capturing standard input/output. \ + This is required in majority of cases, exception being where a shell is \ + not used and/or input/output is not required. In particular \ + when running a command which deliberately closes input/output pipes, \ + such as a daemon process, you may want to disable ``use_pty``. \ + Defaults to ``True`` :type use_pty: bool :param host_args: (Optional) Format command string with per-host \ arguments in ``host_args``. ``host_args`` length must equal length of \ @@ -414,7 +420,7 @@ def run_command(self, *args, **kwargs): 0 0 - *Wait for completion, update output with exit codes* + *Wait for completion, print exit codes* .. code-block:: python @@ -435,9 +441,10 @@ def run_command(self, *args, **kwargs): into memory and may exhaust available memory if command output is large enough. - Iterating over stdout/stderr by definition implies blocking until - command has finished. To only log output as it comes in without blocking - the host logger can be enabled - see `Enabling Host Logger` above. + Iterating over stdout/stderr to completion by definition implies + blocking until command has finished. To only log output as it comes in + without blocking the host logger can be enabled - see + `Enabling Host Logger` above. .. code-block:: python @@ -502,13 +509,13 @@ def run_command(self, *args, **kwargs): Since generators by design only iterate over a sequence once then stop, `client.hosts` should be re-assigned after each call to `run_command` - when using iterators as target of `client.hosts`. + when using generators as target of `client.hosts`. **Overriding host list** Host list can be modified in place. Call to `run_command` will create new connections as necessary and output will only contain output for - hosts command ran on. + the hosts ``run_command`` executed on. .. code-block:: python @@ -781,15 +788,16 @@ def copy_file(self, local_file, remote_file, recurse=False): This function returns a list of greenlets which can be `join`-ed on to wait for completion. - :py:func:`gevent.joinall` function may be used to join on all greenlets and - will also raise exceptions if called with ``raise_error=True`` - default - is `False`. + :py:func:`gevent.joinall` function may be used to join on all greenlets + and will also raise exceptions from them if called with + ``raise_error=True`` - default is `False`. Alternatively call `.get` on each greenlet to raise any exceptions from it. - Exceptions listed here are raised when `.get` is called on each - greenlet, not this function itself. + Exceptions listed here are raised when + ``gevent.joinall(, raise_error=True)`` or ``.get`` is called on + each greenlet, not this function itself. :param local_file: Local filepath to copy to remote host :type local_file: str @@ -802,7 +810,7 @@ def copy_file(self, local_file, remote_file, recurse=False): and recurse is not set :raises: :py:class:`IOError` on I/O errors writing files :raises: :py:class:`OSError` on OS errors like permission denied - + .. note :: Remote directories in `remote_file` that do not exist will be diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 0933c48e..703f9219 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -18,17 +18,18 @@ """Package containing SSHClient class.""" -import sys +import os +import logging +from socket import gaierror as sock_gaierror, error as sock_error + from gevent import sleep import paramiko from paramiko.ssh_exception import ChannelException -import os -from socket import gaierror as sock_gaierror, error as sock_error + from .exceptions import UnknownHostException, AuthenticationException, \ ConnectionErrorException, SSHException from .constants import DEFAULT_RETRIES from .utils import read_openssh_config -import logging host_logger = logging.getLogger('pssh.host_logger') logger = logging.getLogger(__name__) @@ -143,15 +144,15 @@ def _connect_tunnel(self): logger.info("Connecting via SSH proxy %s:%s -> %s:%s", self.proxy_host, self.proxy_port, self.host, self.port,) try: - proxy_channel = self.proxy_client.get_transport().open_channel( - 'direct-tcpip', (self.host, self.port,), ('127.0.0.1', 0)) - sleep(0) - return self._connect(self.client, self.host, self.port, sock=proxy_channel) + proxy_channel = self.proxy_client.get_transport().open_channel( + 'direct-tcpip', (self.host, self.port,), ('127.0.0.1', 0)) + sleep(0) + return self._connect(self.client, self.host, self.port, sock=proxy_channel) except ChannelException as ex: - error_type = ex.args[1] if len(ex.args) > 1 else ex.args[0] - raise ConnectionErrorException("Error connecting to host '%s:%s' - %s", - self.host, self.port, - str(error_type)) + error_type = ex.args[1] if len(ex.args) > 1 else ex.args[0] + raise ConnectionErrorException("Error connecting to host '%s:%s' - %s", + self.host, self.port, + str(error_type)) def _connect(self, client, host, port, sock=None, retries=1, user=None, password=None, pkey=None): @@ -243,7 +244,7 @@ def exec_command(self, command, sudo=False, user=None, stdout, stderr, stdin = channel.makefile('rb'), channel.makefile_stderr('rb'), \ channel.makefile('wb') for _char in ['\\', '"', '$', '`']: - command = command.replace(_char, '\%s' % (_char,)) + command = command.replace(_char, r'\%s' % (_char,)) shell = '$SHELL -c' if not shell else shell _command = '' if sudo and not user: diff --git a/pssh/utils.py b/pssh/utils.py index 5bef6226..80ac2089 100644 --- a/pssh/utils.py +++ b/pssh/utils.py @@ -20,8 +20,8 @@ import logging -import gevent import os + from paramiko.rsakey import RSAKey from paramiko.dsskey import DSSKey from paramiko.ecdsakey import ECDSAKey diff --git a/tests/test_pssh_client.py b/tests/test_pssh_client.py index 4b44b4f4..75848d42 100644 --- a/tests/test_pssh_client.py +++ b/tests/test_pssh_client.py @@ -28,6 +28,7 @@ import warnings import shutil import sys +from socket import timeout as socket_timeout import gevent from pssh import ParallelSSHClient, UnknownHostException, \ @@ -930,5 +931,21 @@ def test_ssh_client_utf_encoding(self): msg="Got unexpected unicode output %s - expected %s" % ( stdout, expected,)) + def test_pty(self): + cmd = "exit 0" + output = self.client.run_command(cmd, use_pty=False) + self.client.join(output) + stdout = list(output[self.host]['stdout']) + exit_code = output[self.host]['exit_code'] + expected = [] + self.assertEqual(expected, stdout) + self.assertTrue(exit_code == 0) + + def test_channel_timeout(self): + cmd = "sleep 2; echo me" + self.client = ParallelSSHClient([self.host], channel_timeout=.1) + output = self.client.run_command(cmd) + self.assertRaises(socket_timeout, list, output[self.host]['stdout']) + if __name__ == '__main__': unittest.main()