X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fpython%2Fssh.py;h=4bed173bbb359005c71294c5d427a8e1328d3dcd;hp=f35b925d106ee2b0bd38b60497a4f22c338a00b2;hb=cc85afad85339d2cc8f035ce63ff21220d8e6b96;hpb=ef67d0dbd0aca3ea0c24800763f6da79f3671727 diff --git a/resources/libraries/python/ssh.py b/resources/libraries/python/ssh.py index f35b925d10..4bed173bbb 100644 --- a/resources/libraries/python/ssh.py +++ b/resources/libraries/python/ssh.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -11,6 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Library for SSH connection management.""" + import StringIO from time import time, sleep @@ -19,7 +21,6 @@ import paramiko from paramiko import RSAKey from paramiko.ssh_exception import SSHException from scp import SCPClient -from interruptingcow import timeout from robot.api import logger from robot.utils.asserts import assert_equal @@ -28,7 +29,13 @@ __all__ = ["exec_cmd", "exec_cmd_no_error"] # TODO: load priv key +class SSHTimeout(Exception): + """This exception is raised when a timeout occurs.""" + pass + + class SSH(object): + """Contains methods for managing and using SSH connections.""" __MAX_RECV_BUF = 10*1024*1024 __existing_connections = {} @@ -39,40 +46,65 @@ class SSH(object): @staticmethod def _node_hash(node): + """Get IP address and port hash from node dictionary. + + :param node: Node in topology. + :type node: dict + :returns: IP address and port for the specified node. + :rtype: int + """ + return hash(frozenset([node['host'], node['port']])) - def connect(self, node): + def connect(self, node, attempts=5): """Connect to node prior to running exec_command or scp. If there already is a connection to the node, this method reuses it. + + :param node: Node in topology. + :param attempts: Number of reconnect attempts. + :type node: dict + :type attempts: int + :raises IOError: If cannot connect to host. """ self._node = node node_hash = self._node_hash(node) if node_hash in SSH.__existing_connections: self._ssh = SSH.__existing_connections[node_hash] - logger.debug('reusing ssh: {0}'.format(self._ssh)) + if self._ssh.get_transport().is_active(): + logger.debug('Reusing SSH: {ssh}'.format(ssh=self._ssh)) + else: + if attempts > 0: + self._reconnect(attempts-1) + else: + raise IOError('Cannot connect to {host}'. + format(host=node['host'])) else: - start = time() - pkey = None - if 'priv_key' in node: - pkey = RSAKey.from_private_key( + try: + start = time() + pkey = None + if 'priv_key' in node: + pkey = RSAKey.from_private_key( StringIO.StringIO(node['priv_key'])) - self._ssh = paramiko.SSHClient() - self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - self._ssh.connect(node['host'], username=node['username'], - password=node.get('password'), pkey=pkey, - port=node['port']) + self._ssh = paramiko.SSHClient() + self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - SSH.__existing_connections[node_hash] = self._ssh + self._ssh.connect(node['host'], username=node['username'], + password=node.get('password'), pkey=pkey, + port=node['port']) - logger.trace('connect took {} seconds'.format(time() - start)) - logger.debug('new ssh: {0}'.format(self._ssh)) + self._ssh.get_transport().set_keepalive(10) - logger.debug('Connect peer: {0}'. - format(self._ssh.get_transport().getpeername())) - logger.debug('Connections: {0}'.format(str(SSH.__existing_connections))) + SSH.__existing_connections[node_hash] = self._ssh + logger.debug('New SSH to {peer} took {total} seconds: {ssh}'. + format( + peer=self._ssh.get_transport().getpeername(), + total=(time() - start), + ssh=self._ssh)) + except SSHException: + raise IOError('Cannot connect to {host}'. + format(host=node['host'])) def disconnect(self, node): """Close SSH connection to the node. @@ -82,45 +114,54 @@ class SSH(object): """ node_hash = self._node_hash(node) if node_hash in SSH.__existing_connections: - logger.debug('Disconnecting peer: {}, {}'. - format(node['host'], node['port'])) + logger.debug('Disconnecting peer: {host}, {port}'. + format(host=node['host'], port=node['port'])) ssh = SSH.__existing_connections.pop(node_hash) ssh.close() - def _reconnect(self): + def _reconnect(self, attempts=0): + """Close the SSH connection and open it again. + + :param attempts: Number of reconnect attempts. + :type attempts: int + """ node = self._node self.disconnect(node) - self.connect(node) - logger.debug('Reconnecting peer done: {}'. - format(self._ssh.get_transport().getpeername())) + self.connect(node, attempts) + logger.debug('Reconnecting peer done: {host}, {port}'. + format(host=node['host'], port=node['port'])) def exec_command(self, cmd, timeout=10): """Execute SSH command on a new channel on the connected Node. :param cmd: Command to run on the Node. - :param timeout: Maximal time in seconds to wait while the command is - done. If is None then wait forever. + :param timeout: Maximal time in seconds to wait until the command is + done. If set to None then wait forever. :type cmd: str :type timeout: int :return return_code, stdout, stderr :rtype: tuple(int, str, str) - :raise socket.timeout: If command is not finished in timeout time. + :raise SSHTimeout: If command is not finished in timeout time. """ - start = time() stdout = StringIO.StringIO() stderr = StringIO.StringIO() try: chan = self._ssh.get_transport().open_session(timeout=5) + peer = self._ssh.get_transport().getpeername() except AttributeError: self._reconnect() chan = self._ssh.get_transport().open_session(timeout=5) + peer = self._ssh.get_transport().getpeername() except SSHException: self._reconnect() chan = self._ssh.get_transport().open_session(timeout=5) + peer = self._ssh.get_transport().getpeername() chan.settimeout(timeout) - logger.trace('exec_command on {0}: {1}' - .format(self._ssh.get_transport().getpeername(), cmd)) + logger.trace('exec_command on {peer} with timeout {timeout}: {cmd}' + .format(peer=peer, timeout=timeout, cmd=cmd)) + + start = time() chan.exec_command(cmd) while not chan.exit_status_ready() and timeout is not None: if chan.recv_ready(): @@ -130,11 +171,12 @@ class SSH(object): stderr.write(chan.recv_stderr(self.__MAX_RECV_BUF)) if time() - start > timeout: - raise socket.timeout( - 'Timeout exception.\n' - 'Current contents of stdout buffer: {0}\n' - 'Current contents of stderr buffer: {1}\n' - .format(stdout.getvalue(), stderr.getvalue()) + raise SSHTimeout( + 'Timeout exception during execution of command: {cmd}\n' + 'Current contents of stdout buffer: {stdout}\n' + 'Current contents of stderr buffer: {stderr}\n' + .format(cmd=cmd, stdout=stdout.getvalue(), + stderr=stderr.getvalue()) ) sleep(0.1) @@ -147,14 +189,12 @@ class SSH(object): stderr.write(chan.recv_stderr(self.__MAX_RECV_BUF)) end = time() - logger.trace('exec_command on {0} took {1} seconds'.format( - self._ssh.get_transport().getpeername(), end-start)) - - logger.trace('chan_recv/_stderr took {} seconds'.format(time()-end)) + logger.trace('exec_command on {peer} took {total} seconds'. + format(peer=peer, total=end-start)) - logger.trace('return RC {}'.format(return_code)) - logger.trace('return STDOUT {}'.format(stdout.getvalue())) - logger.trace('return STDERR {}'.format(stderr.getvalue())) + logger.trace('return RC {rc}'.format(rc=return_code)) + logger.trace('return STDOUT {stdout}'.format(stdout=stdout.getvalue())) + logger.trace('return STDERR {stderr}'.format(stderr=stderr.getvalue())) return return_code, stdout.getvalue(), stderr.getvalue() def exec_command_sudo(self, cmd, cmd_input=None, timeout=30): @@ -163,7 +203,7 @@ class SSH(object): :param cmd: Command to be executed. :param cmd_input: Input redirected to the command. :param timeout: Timeout. - :return: return_code, stdout, stderr + :returns: return_code, stdout, stderr :Example: @@ -181,11 +221,34 @@ class SSH(object): command = 'sudo -S {c} <<< "{i}"'.format(c=cmd, i=cmd_input) return self.exec_command(command, timeout) - def interactive_terminal_open(self, time_out=10): + def exec_command_lxc(self, lxc_cmd, lxc_name, lxc_params='', sudo=True, + timeout=30): + """Execute command in LXC on a new SSH channel on the connected Node. + + :param lxc_cmd: Command to be executed. + :param lxc_name: LXC name. + :param lxc_params: Additional parameters for LXC attach. + :param sudo: Run in privileged LXC mode. Default: privileged + :param timeout: Timeout. + :type lxc_cmd: str + :type lxc_name: str + :type lxc_params: str + :type sudo: bool + :type timeout: int + :returns: return_code, stdout, stderr + """ + command = "lxc-attach {p} --name {n} -- /bin/sh -c '{c}'"\ + .format(p=lxc_params, n=lxc_name, c=lxc_cmd) + + if sudo: + command = 'sudo -S {c}'.format(c=command) + return self.exec_command(command, timeout) + + def interactive_terminal_open(self, time_out=30): """Open interactive terminal on a new channel on the connected Node. :param time_out: Timeout in seconds. - :return: SSH channel with opened terminal. + :returns: SSH channel with opened terminal. .. warning:: Interruptingcow is used here, and it uses signal(SIGALRM) to let the operating system interrupt program @@ -198,20 +261,24 @@ class SSH(object): chan.get_pty() chan.invoke_shell() chan.settimeout(int(time_out)) + chan.set_combine_stderr(True) buf = '' - try: - with timeout(time_out, exception=RuntimeError): - while not buf.endswith(':~$ '): - if chan.recv_ready(): - buf = chan.recv(4096) - except RuntimeError: - raise Exception('Open interactive terminal timeout.') + while not buf.endswith((":~$ ", "~]$ ", "~]# ")): + try: + chunk = chan.recv(self.__MAX_RECV_BUF) + if not chunk: + break + buf += chunk + if chan.exit_status_ready(): + logger.error('Channel exit status ready') + break + except socket.timeout: + logger.error('Socket timeout: {0}'.format(buf)) + raise Exception('Socket timeout: {0}'.format(buf)) return chan - @staticmethod - def interactive_terminal_exec_command(chan, cmd, prompt, - time_out=10): + def interactive_terminal_exec_command(self, chan, cmd, prompt): """Execute command on interactive terminal. interactive_terminal_open() method has to be called first! @@ -220,8 +287,7 @@ class SSH(object): :param cmd: Command to be executed. :param prompt: Command prompt, sequence of characters used to indicate readiness to accept commands. - :param time_out: Timeout in seconds. - :return: Command output. + :returns: Command output. .. warning:: Interruptingcow is used here, and it uses signal(SIGALRM) to let the operating system interrupt program @@ -232,15 +298,24 @@ class SSH(object): """ chan.sendall('{c}\n'.format(c=cmd)) buf = '' - try: - with timeout(time_out, exception=RuntimeError): - while not buf.endswith(prompt): - if chan.recv_ready(): - buf += chan.recv(4096) - except RuntimeError: - raise Exception("Exec '{c}' timeout.".format(c=cmd)) + while not buf.endswith(prompt): + try: + chunk = chan.recv(self.__MAX_RECV_BUF) + if not chunk: + break + buf += chunk + if chan.exit_status_ready(): + logger.error('Channel exit status ready') + break + except socket.timeout: + logger.error('Socket timeout during execution of command: ' + '{0}\nBuffer content:\n{1}'.format(cmd, buf)) + raise Exception('Socket timeout during execution of command: ' + '{0}\nBuffer content:\n{1}'.format(cmd, buf)) tmp = buf.replace(cmd.replace('\n', ''), '') - return tmp.replace(prompt, '') + for item in prompt: + tmp.replace(item, '') + return tmp @staticmethod def interactive_terminal_close(chan): @@ -250,17 +325,37 @@ class SSH(object): """ chan.close() - def scp(self, local_path, remote_path): - """Copy files from local_path to remote_path. + def scp(self, local_path, remote_path, get=False, timeout=30): + """Copy files from local_path to remote_path or vice versa. connect() method has to be called first! + + :param local_path: Path to local file that should be uploaded; or + path where to save remote file. + :param remote_path: Remote path where to place uploaded file; or + path to remote file which should be downloaded. + :param get: scp operation to perform. Default is put. + :param timeout: Timeout value in seconds. + :type local_path: str + :type remote_path: str + :type get: bool + :type timeout: int """ - logger.trace('SCP {0} to {1}:{2}'.format( - local_path, self._ssh.get_transport().getpeername(), remote_path)) + if not get: + logger.trace('SCP {0} to {1}:{2}'.format( + local_path, self._ssh.get_transport().getpeername(), + remote_path)) + else: + logger.trace('SCP {0}:{1} to {2}'.format( + self._ssh.get_transport().getpeername(), remote_path, + local_path)) # SCPCLient takes a paramiko transport as its only argument - scp = SCPClient(self._ssh.get_transport()) + scp = SCPClient(self._ssh.get_transport(), socket_timeout=timeout) start = time() - scp.put(local_path, remote_path) + if not get: + scp.put(local_path, remote_path) + else: + scp.get(remote_path, local_path) scp.close() end = time() logger.trace('SCP took {0} seconds'.format(end-start)) @@ -281,8 +376,8 @@ def exec_cmd(node, cmd, timeout=600, sudo=False): ssh = SSH() try: ssh.connect(node) - except Exception, e: - logger.error("Failed to connect to node" + str(e)) + except SSHException as err: + logger.error("Failed to connect to node" + str(err)) return None, None, None try: @@ -291,8 +386,8 @@ def exec_cmd(node, cmd, timeout=600, sudo=False): else: (ret_code, stdout, stderr) = ssh.exec_command_sudo(cmd, timeout=timeout) - except Exception, e: - logger.error(e) + except SSHException as err: + logger.error(err) return None, None, None return ret_code, stdout, stderr @@ -305,7 +400,7 @@ def exec_cmd_no_error(node, cmd, timeout=600, sudo=False): Returns (stdout, stderr). """ - (rc, stdout, stderr) = exec_cmd(node, cmd, timeout=timeout, sudo=sudo) - assert_equal(rc, 0, 'Command execution failed: "{}"\n{}'. + (ret_code, stdout, stderr) = exec_cmd(node, cmd, timeout=timeout, sudo=sudo) + assert_equal(ret_code, 0, 'Command execution failed: "{}"\n{}'. format(cmd, stderr)) return stdout, stderr