X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Flibraries%2Fpython%2Fssh.py;h=f35b925d106ee2b0bd38b60497a4f22c338a00b2;hp=ad5fb27b8533970ce13b1da96332a6a2650ca903;hb=bdc400b3feacd10d8130dedaa7df8df2d2d57feb;hpb=2d54ff480b3f135ef4be951c065b4a6d19a138f8 diff --git a/resources/libraries/python/ssh.py b/resources/libraries/python/ssh.py index ad5fb27b85..f35b925d10 100644 --- a/resources/libraries/python/ssh.py +++ b/resources/libraries/python/ssh.py @@ -10,14 +10,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import StringIO +from time import time, sleep + +import socket import paramiko from paramiko import RSAKey -import StringIO +from paramiko.ssh_exception import SSHException from scp import SCPClient -from time import time -from robot.api import logger from interruptingcow import timeout -from robot.utils.asserts import assert_equal, assert_not_equal +from robot.api import logger +from robot.utils.asserts import assert_equal __all__ = ["exec_cmd", "exec_cmd_no_error"] @@ -30,9 +34,11 @@ class SSH(object): __existing_connections = {} def __init__(self): - pass + self._ssh = None + self._node = None - def _node_hash(self, node): + @staticmethod + def _node_hash(node): return hash(frozenset([node['host'], node['port']])) def connect(self, node): @@ -40,6 +46,7 @@ class SSH(object): If there already is a connection to the node, this method reuses it. """ + self._node = node node_hash = self._node_hash(node) if node_hash in SSH.__existing_connections: self._ssh = SSH.__existing_connections[node_hash] @@ -64,64 +71,109 @@ class SSH(object): logger.debug('new ssh: {0}'.format(self._ssh)) logger.debug('Connect peer: {0}'. - format(self._ssh.get_transport().getpeername())) + format(self._ssh.get_transport().getpeername())) logger.debug('Connections: {0}'.format(str(SSH.__existing_connections))) + def disconnect(self, node): + """Close SSH connection to the node. + + :param node: The node to disconnect from. + :type node: dict + """ + node_hash = self._node_hash(node) + if node_hash in SSH.__existing_connections: + logger.debug('Disconnecting peer: {}, {}'. + format(node['host'], node['port'])) + ssh = SSH.__existing_connections.pop(node_hash) + ssh.close() + + def _reconnect(self): + node = self._node + self.disconnect(node) + self.connect(node) + logger.debug('Reconnecting peer done: {}'. + format(self._ssh.get_transport().getpeername())) + def exec_command(self, cmd, timeout=10): """Execute SSH command on a new channel on the connected Node. - Returns (return_code, stdout, stderr). + :param cmd: Command to run on the Node. + :param timeout: Maximal time in seconds to wait while the command is + done. If is None then wait forever. + :type cmd: str + :type timeout: int + :return return_code, stdout, stderr + :rtype: tuple(int, str, str) + :raise socket.timeout: If command is not finished in timeout time. """ + start = time() + stdout = StringIO.StringIO() + stderr = StringIO.StringIO() + try: + chan = self._ssh.get_transport().open_session(timeout=5) + except AttributeError: + self._reconnect() + chan = self._ssh.get_transport().open_session(timeout=5) + except SSHException: + self._reconnect() + chan = self._ssh.get_transport().open_session(timeout=5) + chan.settimeout(timeout) logger.trace('exec_command on {0}: {1}' .format(self._ssh.get_transport().getpeername(), cmd)) - start = time() - chan = self._ssh.get_transport().open_session() - if timeout is not None: - chan.settimeout(int(timeout)) + chan.exec_command(cmd) + while not chan.exit_status_ready() and timeout is not None: + if chan.recv_ready(): + stdout.write(chan.recv(self.__MAX_RECV_BUF)) + + if chan.recv_stderr_ready(): + stderr.write(chan.recv_stderr(self.__MAX_RECV_BUF)) + + if time() - start > timeout: + raise socket.timeout( + 'Timeout exception.\n' + 'Current contents of stdout buffer: {0}\n' + 'Current contents of stderr buffer: {1}\n' + .format(stdout.getvalue(), stderr.getvalue()) + ) + + sleep(0.1) + return_code = chan.recv_exit_status() + + while chan.recv_ready(): + stdout.write(chan.recv(self.__MAX_RECV_BUF)) + + while chan.recv_stderr_ready(): + stderr.write(chan.recv_stderr(self.__MAX_RECV_BUF)) + end = time() logger.trace('exec_command on {0} took {1} seconds'.format( self._ssh.get_transport().getpeername(), end-start)) - stdout = "" - while True: - buf = chan.recv(self.__MAX_RECV_BUF) - stdout += buf - if not buf: - break - - stderr = "" - while True: - buf = chan.recv_stderr(self.__MAX_RECV_BUF) - stderr += buf - if not buf: - break - - return_code = chan.recv_exit_status() logger.trace('chan_recv/_stderr took {} seconds'.format(time()-end)) logger.trace('return RC {}'.format(return_code)) - logger.trace('return STDOUT {}'.format(stdout)) - logger.trace('return STDERR {}'.format(stderr)) - return return_code, stdout, stderr + logger.trace('return STDOUT {}'.format(stdout.getvalue())) + logger.trace('return STDERR {}'.format(stderr.getvalue())) + return return_code, stdout.getvalue(), stderr.getvalue() - def exec_command_sudo(self, cmd, cmd_input=None, timeout=10): + def exec_command_sudo(self, cmd, cmd_input=None, timeout=30): """Execute SSH command with sudo on a new channel on the connected Node. - :param cmd: Command to be executed. - :param cmd_input: Input redirected to the command. - :param timeout: Timeout. - :return: return_code, stdout, stderr + :param cmd: Command to be executed. + :param cmd_input: Input redirected to the command. + :param timeout: Timeout. + :return: return_code, stdout, stderr - :Example: + :Example: - >>> from ssh import SSH - >>> ssh = SSH() - >>> ssh.connect(node) - >>> #Execute command without input (sudo -S cmd) - >>> ssh.exec_command_sudo("ifconfig eth0 down") - >>> #Execute command with input (sudo -S cmd <<< "input") - >>> ssh.exec_command_sudo("vpp_api_test", "dump_interface_table") + >>> from ssh import SSH + >>> ssh = SSH() + >>> ssh.connect(node) + >>> # Execute command without input (sudo -S cmd) + >>> ssh.exec_command_sudo("ifconfig eth0 down") + >>> # Execute command with input (sudo -S cmd <<< "input") + >>> ssh.exec_command_sudo("vpp_api_test", "dump_interface_table") """ if cmd_input is None: command = 'sudo -S {c}'.format(c=cmd) @@ -132,15 +184,15 @@ class SSH(object): def interactive_terminal_open(self, time_out=10): """Open interactive terminal on a new channel on the connected Node. - :param time_out: Timeout in seconds. - :return: SSH channel with opened terminal. + :param time_out: Timeout in seconds. + :return: SSH channel with opened terminal. - .. warning:: Interruptingcow is used here, and it uses - signal(SIGALRM) to let the operating system interrupt program - execution. This has the following limitations: Python signal - handlers only apply to the main thread, so you cannot use this - from other threads. You must not use this in a program that - uses SIGALRM itself (this includes certain profilers) + .. warning:: Interruptingcow is used here, and it uses + signal(SIGALRM) to let the operating system interrupt program + execution. This has the following limitations: Python signal + handlers only apply to the main thread, so you cannot use this + from other threads. You must not use this in a program that + uses SIGALRM itself (this includes certain profilers) """ chan = self._ssh.get_transport().open_session() chan.get_pty() @@ -157,25 +209,26 @@ class SSH(object): raise Exception('Open interactive terminal timeout.') return chan - def interactive_terminal_exec_command(self, chan, cmd, prompt, + @staticmethod + def interactive_terminal_exec_command(chan, cmd, prompt, time_out=10): """Execute command on interactive terminal. - interactive_terminal_open() method has to be called first! - - :param chan: SSH channel with opened terminal. - :param cmd: Command to be executed. - :param prompt: Command prompt, sequence of characters used to - indicate readiness to accept commands. - :param time_out: Timeout in seconds. - :return: Command output. - - .. warning:: Interruptingcow is used here, and it uses - signal(SIGALRM) to let the operating system interrupt program - execution. This has the following limitations: Python signal - handlers only apply to the main thread, so you cannot use this - from other threads. You must not use this in a program that - uses SIGALRM itself (this includes certain profilers) + interactive_terminal_open() method has to be called first! + + :param chan: SSH channel with opened terminal. + :param cmd: Command to be executed. + :param prompt: Command prompt, sequence of characters used to + indicate readiness to accept commands. + :param time_out: Timeout in seconds. + :return: Command output. + + .. warning:: Interruptingcow is used here, and it uses + signal(SIGALRM) to let the operating system interrupt program + execution. This has the following limitations: Python signal + handlers only apply to the main thread, so you cannot use this + from other threads. You must not use this in a program that + uses SIGALRM itself (this includes certain profilers) """ chan.sendall('{c}\n'.format(c=cmd)) buf = '' @@ -189,10 +242,11 @@ class SSH(object): tmp = buf.replace(cmd.replace('\n', ''), '') return tmp.replace(prompt, '') - def interactive_terminal_close(self, chan): + @staticmethod + def interactive_terminal_close(chan): """Close interactive terminal SSH channel. - :param: chan: SSH channel to be closed. + :param: chan: SSH channel to be closed. """ chan.close() @@ -212,7 +266,7 @@ class SSH(object): logger.trace('SCP took {0} seconds'.format(end-start)) -def exec_cmd(node, cmd, timeout=None, sudo=False): +def exec_cmd(node, cmd, timeout=600, sudo=False): """Convenience function to ssh/exec/return rc, out & err. Returns (rc, stdout, stderr). @@ -228,8 +282,8 @@ def exec_cmd(node, cmd, timeout=None, sudo=False): try: ssh.connect(node) except Exception, e: - logger.error("Failed to connect to node" + e) - return None + logger.error("Failed to connect to node" + str(e)) + return None, None, None try: if not sudo: @@ -239,12 +293,14 @@ def exec_cmd(node, cmd, timeout=None, sudo=False): timeout=timeout) except Exception, e: logger.error(e) - return None + return None, None, None + + return ret_code, stdout, stderr - return (ret_code, stdout, stderr) -def exec_cmd_no_error(node, cmd, timeout=None, sudo=False): +def exec_cmd_no_error(node, cmd, timeout=600, sudo=False): """Convenience function to ssh/exec/return out & err. + Verifies that return code is zero. Returns (stdout, stderr). @@ -252,4 +308,4 @@ def exec_cmd_no_error(node, cmd, timeout=None, sudo=False): (rc, stdout, stderr) = exec_cmd(node, cmd, timeout=timeout, sudo=sudo) assert_equal(rc, 0, 'Command execution failed: "{}"\n{}'. format(cmd, stderr)) - return (stdout, stderr) + return stdout, stderr