1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Library for SSH connection management."""
19 from time import time, sleep
21 from paramiko import RSAKey, SSHClient, AutoAddPolicy
22 from paramiko.ssh_exception import SSHException, NoValidConnectionsError
23 from robot.api import logger
24 from scp import SCPClient, SCPException
26 from resources.libraries.python.OptionString import OptionString
28 __all__ = ["exec_cmd", "exec_cmd_no_error"]
33 def raise_from(raising, excepted):
34 """Function to be replaced by "raise from" in Python 3.
36 Neither "six" nor "future" offer good enough implementation right now.
37 chezsoi.org/lucas/blog/displaying-chained-exceptions-stacktraces-in-python-2
39 Current implementation just logs excepted error, and raises the new one.
41 :param raising: The exception to raise.
42 :param excepted: The exception we excepted and want to log.
43 :type raising: BaseException
44 :type excepted: BaseException
47 logger.error("Excepted: {exc!r}\nRaising: {rai!r}".format(
48 exc=excepted, rai=raising))
52 class SSHTimeout(Exception):
53 """This exception is raised when a timeout occurs."""
58 """Contains methods for managing and using SSH connections."""
60 __MAX_RECV_BUF = 10*1024*1024
61 __existing_connections = {}
69 """Get IP address and port hash from node dictionary.
71 :param node: Node in topology.
73 :returns: IP address and port for the specified node.
77 return hash(frozenset([node['host'], node['port']]))
79 def connect(self, node, attempts=5):
80 """Connect to node prior to running exec_command or scp.
82 If there already is a connection to the node, this method reuses it.
84 :param node: Node in topology.
85 :param attempts: Number of reconnect attempts.
88 :raises IOError: If cannot connect to host.
91 node_hash = self._node_hash(node)
92 if node_hash in SSH.__existing_connections:
93 self._ssh = SSH.__existing_connections[node_hash]
94 if self._ssh.get_transport().is_active():
95 logger.debug('Reusing SSH: {ssh}'.format(ssh=self._ssh))
98 self._reconnect(attempts-1)
100 raise IOError('Cannot connect to {host}'.
101 format(host=node['host']))
106 if 'priv_key' in node:
107 pkey = RSAKey.from_private_key(
108 StringIO.StringIO(node['priv_key']))
110 self._ssh = SSHClient()
111 self._ssh.set_missing_host_key_policy(AutoAddPolicy())
113 self._ssh.connect(node['host'], username=node['username'],
114 password=node.get('password'), pkey=pkey,
117 self._ssh.get_transport().set_keepalive(10)
119 SSH.__existing_connections[node_hash] = self._ssh
120 logger.debug('New SSH to {peer} took {total} seconds: {ssh}'.
122 peer=self._ssh.get_transport().getpeername(),
123 total=(time() - start),
125 except SSHException as exc:
126 raise_from(IOError('Cannot connect to {host}'.format(
127 host=node['host'])), exc)
128 except NoValidConnectionsError as err:
130 'Unable to connect to port {port} on {host}'.format(
131 port=node['port'], host=node['host'])), err)
133 def disconnect(self, node=None):
134 """Close SSH connection to the node.
136 :param node: The node to disconnect from. None means last connected.
137 :type node: dict or None
143 node_hash = self._node_hash(node)
144 if node_hash in SSH.__existing_connections:
145 logger.debug('Disconnecting peer: {host}, {port}'.
146 format(host=node['host'], port=node['port']))
147 ssh = SSH.__existing_connections.pop(node_hash)
150 def _reconnect(self, attempts=0):
151 """Close the SSH connection and open it again.
153 :param attempts: Number of reconnect attempts.
157 self.disconnect(node)
158 self.connect(node, attempts)
159 logger.debug('Reconnecting peer done: {host}, {port}'.
160 format(host=node['host'], port=node['port']))
162 def exec_command(self, cmd, timeout=10, log_stdout_err=True):
163 """Execute SSH command on a new channel on the connected Node.
165 :param cmd: Command to run on the Node.
166 :param timeout: Maximal time in seconds to wait until the command is
167 done. If set to None then wait forever.
168 :param log_stdout_err: If True, stdout and stderr are logged. stdout
169 and stderr are logged also if the return code is not zero
170 independently of the value of log_stdout_err.
171 :type cmd: str or OptionString
173 :type log_stdout_err: bool
174 :returns: return_code, stdout, stderr
175 :rtype: tuple(int, str, str)
176 :raises SSHTimeout: If command is not finished in timeout time.
178 if isinstance(cmd, (list, tuple)):
179 cmd = OptionString(cmd)
181 stdout = StringIO.StringIO()
182 stderr = StringIO.StringIO()
184 chan = self._ssh.get_transport().open_session(timeout=5)
185 peer = self._ssh.get_transport().getpeername()
186 except (AttributeError, SSHException):
188 chan = self._ssh.get_transport().open_session(timeout=5)
189 peer = self._ssh.get_transport().getpeername()
190 chan.settimeout(timeout)
192 logger.trace('exec_command on {peer} with timeout {timeout}: {cmd}'
193 .format(peer=peer, timeout=timeout, cmd=cmd))
196 chan.exec_command(cmd)
197 while not chan.exit_status_ready() and timeout is not None:
198 if chan.recv_ready():
199 stdout.write(chan.recv(self.__MAX_RECV_BUF))
201 if chan.recv_stderr_ready():
202 stderr.write(chan.recv_stderr(self.__MAX_RECV_BUF))
204 if time() - start > timeout:
206 'Timeout exception during execution of command: {cmd}\n'
207 'Current contents of stdout buffer: {stdout}\n'
208 'Current contents of stderr buffer: {stderr}\n'
209 .format(cmd=cmd, stdout=stdout.getvalue(),
210 stderr=stderr.getvalue())
214 return_code = chan.recv_exit_status()
216 while chan.recv_ready():
217 stdout.write(chan.recv(self.__MAX_RECV_BUF))
219 while chan.recv_stderr_ready():
220 stderr.write(chan.recv_stderr(self.__MAX_RECV_BUF))
223 logger.trace('exec_command on {peer} took {total} seconds'.
224 format(peer=peer, total=end-start))
226 logger.trace('return RC {rc}'.format(rc=return_code))
227 if log_stdout_err or int(return_code):
228 logger.trace('return STDOUT {stdout}'.
229 format(stdout=stdout.getvalue()))
230 logger.trace('return STDERR {stderr}'.
231 format(stderr=stderr.getvalue()))
232 return return_code, stdout.getvalue(), stderr.getvalue()
234 def exec_command_sudo(self, cmd, cmd_input=None, timeout=30,
235 log_stdout_err=True):
236 """Execute SSH command with sudo on a new channel on the connected Node.
238 :param cmd: Command to be executed.
239 :param cmd_input: Input redirected to the command.
240 :param timeout: Timeout.
241 :param log_stdout_err: If True, stdout and stderr are logged.
245 :type log_stdout_err: bool
246 :returns: return_code, stdout, stderr
247 :rtype: tuple(int, str, str)
251 >>> from ssh import SSH
253 >>> ssh.connect(node)
254 >>> # Execute command without input (sudo -S cmd)
255 >>> ssh.exec_command_sudo("ifconfig eth0 down")
256 >>> # Execute command with input (sudo -S cmd <<< "input")
257 >>> ssh.exec_command_sudo("vpp_api_test", "dump_interface_table")
259 if isinstance(cmd, (list, tuple)):
260 cmd = OptionString(cmd)
261 if cmd_input is None:
262 command = 'sudo -S {c}'.format(c=cmd)
264 command = 'sudo -S {c} <<< "{i}"'.format(c=cmd, i=cmd_input)
265 return self.exec_command(command, timeout,
266 log_stdout_err=log_stdout_err)
268 def exec_command_lxc(self, lxc_cmd, lxc_name, lxc_params='', sudo=True,
270 """Execute command in LXC on a new SSH channel on the connected Node.
272 :param lxc_cmd: Command to be executed.
273 :param lxc_name: LXC name.
274 :param lxc_params: Additional parameters for LXC attach.
275 :param sudo: Run in privileged LXC mode. Default: privileged
276 :param timeout: Timeout.
279 :type lxc_params: str
282 :returns: return_code, stdout, stderr
284 command = "lxc-attach {p} --name {n} -- /bin/sh -c '{c}'"\
285 .format(p=lxc_params, n=lxc_name, c=lxc_cmd)
288 command = 'sudo -S {c}'.format(c=command)
289 return self.exec_command(command, timeout)
291 def interactive_terminal_open(self, time_out=45):
292 """Open interactive terminal on a new channel on the connected Node.
294 :param time_out: Timeout in seconds.
295 :returns: SSH channel with opened terminal.
297 .. warning:: Interruptingcow is used here, and it uses
298 signal(SIGALRM) to let the operating system interrupt program
299 execution. This has the following limitations: Python signal
300 handlers only apply to the main thread, so you cannot use this
301 from other threads. You must not use this in a program that
302 uses SIGALRM itself (this includes certain profilers)
304 chan = self._ssh.get_transport().open_session()
307 chan.settimeout(int(time_out))
308 chan.set_combine_stderr(True)
311 while not buf.endswith((":~# ", ":~$ ", "~]$ ", "~]# ")):
313 chunk = chan.recv(self.__MAX_RECV_BUF)
317 if chan.exit_status_ready():
318 logger.error('Channel exit status ready')
320 except socket.timeout as exc:
321 raise_from(Exception('Socket timeout: {0}'.format(buf)), exc)
324 def interactive_terminal_exec_command(self, chan, cmd, prompt):
325 """Execute command on interactive terminal.
327 interactive_terminal_open() method has to be called first!
329 :param chan: SSH channel with opened terminal.
330 :param cmd: Command to be executed.
331 :param prompt: Command prompt, sequence of characters used to
332 indicate readiness to accept commands.
333 :returns: Command output.
335 .. warning:: Interruptingcow is used here, and it uses
336 signal(SIGALRM) to let the operating system interrupt program
337 execution. This has the following limitations: Python signal
338 handlers only apply to the main thread, so you cannot use this
339 from other threads. You must not use this in a program that
340 uses SIGALRM itself (this includes certain profilers)
342 chan.sendall('{c}\n'.format(c=cmd))
344 while not buf.endswith(prompt):
346 chunk = chan.recv(self.__MAX_RECV_BUF)
350 if chan.exit_status_ready():
351 logger.error('Channel exit status ready')
353 except socket.timeout as exc:
354 raise_from(Exception(
355 'Socket timeout during execution of command: '
356 '{0}\nBuffer content:\n{1}'.format(cmd, buf)), exc)
357 tmp = buf.replace(cmd.replace('\n', ''), '')
359 tmp.replace(item, '')
363 def interactive_terminal_close(chan):
364 """Close interactive terminal SSH channel.
366 :param chan: SSH channel to be closed.
370 def scp(self, local_path, remote_path, get=False, timeout=30,
372 """Copy files from local_path to remote_path or vice versa.
374 connect() method has to be called first!
376 :param local_path: Path to local file that should be uploaded; or
377 path where to save remote file.
378 :param remote_path: Remote path where to place uploaded file; or
379 path to remote file which should be downloaded.
380 :param get: scp operation to perform. Default is put.
381 :param timeout: Timeout value in seconds.
382 :param wildcard: If path has wildcard characters. Default is false.
383 :type local_path: str
384 :type remote_path: str
390 logger.trace('SCP {0} to {1}:{2}'.format(
391 local_path, self._ssh.get_transport().getpeername(),
394 logger.trace('SCP {0}:{1} to {2}'.format(
395 self._ssh.get_transport().getpeername(), remote_path,
397 # SCPCLient takes a paramiko transport as its only argument
399 scp = SCPClient(self._ssh.get_transport(), socket_timeout=timeout)
401 scp = SCPClient(self._ssh.get_transport(), sanitize=lambda x: x,
402 socket_timeout=timeout)
405 scp.put(local_path, remote_path)
407 scp.get(remote_path, local_path)
410 logger.trace('SCP took {0} seconds'.format(end-start))
413 def exec_cmd(node, cmd, timeout=600, sudo=False, disconnect=False):
414 """Convenience function to ssh/exec/return rc, out & err.
416 Returns (rc, stdout, stderr).
418 :param node: The node to execute command on.
419 :param cmd: Command to execute.
420 :param timeout: Timeout value in seconds. Default: 600.
421 :param sudo: Sudo privilege execution flag. Default: False.
422 :param disconnect: Close the opened SSH connection if True.
424 :type cmd: str or OptionString
427 :type disconnect: bool
428 :returns: RC, Stdout, Stderr.
429 :rtype: tuple(int, str, str)
432 raise TypeError('Node parameter is None')
434 raise TypeError('Command parameter is None')
436 raise ValueError('Empty command parameter')
440 if node.get('host_port') is not None:
442 ssh_node['host'] = '127.0.0.1'
443 ssh_node['port'] = node['port']
444 ssh_node['username'] = node['username']
445 ssh_node['password'] = node['password']
447 options = '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
448 tnl = '-L {port}:127.0.0.1:{port}'.format(port=node['port'])
449 ssh_cmd = 'ssh {tnl} {op} {user}@{host} -p {host_port}'.\
450 format(tnl=tnl, op=options, user=node['host_username'],
451 host=node['host'], host_port=node['host_port'])
452 logger.trace('Initializing local port forwarding:\n{ssh_cmd}'.
453 format(ssh_cmd=ssh_cmd))
454 child = pexpect.spawn(ssh_cmd)
455 child.expect('.* password: ')
456 logger.trace(child.after)
457 child.sendline(node['host_password'])
458 child.expect('Welcome .*')
459 logger.trace(child.after)
460 logger.trace('Local port forwarding finished.')
465 ssh.connect(ssh_node)
466 except SSHException as err:
467 logger.error("Failed to connect to node" + repr(err))
468 return None, None, None
472 (ret_code, stdout, stderr) = ssh.exec_command(cmd, timeout=timeout)
474 (ret_code, stdout, stderr) = ssh.exec_command_sudo(cmd,
476 except SSHException as err:
477 logger.error(repr(err))
478 return None, None, None
483 return ret_code, stdout, stderr
486 def exec_cmd_no_error(
487 node, cmd, timeout=600, sudo=False, message=None, disconnect=False,
488 retries=0, include_reason=False):
489 """Convenience function to ssh/exec/return out & err.
491 Verifies that return code is zero.
492 Supports retries, timeout is related to each try separately then. There is
493 sleep(1) before each retry.
494 Disconnect (if enabled) is applied after each try.
496 :param node: DUT node.
497 :param cmd: Command to be executed.
498 :param timeout: Timeout value in seconds. Default: 600.
499 :param sudo: Sudo privilege execution flag. Default: False.
500 :param message: Error message in case of failure. Default: None.
501 :param disconnect: Close the opened SSH connection if True.
502 :param retries: How many times to retry on failure.
503 :param include_reason: Whether default info should be appended to message.
505 :type cmd: str or OptionString
509 :type disconnect: bool
511 :type include_reason: bool
512 :returns: Stdout, Stderr.
513 :rtype: tuple(str, str)
514 :raises RuntimeError: If bash return code is not 0.
516 for _ in range(retries + 1):
517 ret_code, stdout, stderr = exec_cmd(
518 node, cmd, timeout=timeout, sudo=sudo, disconnect=disconnect)
523 msg = 'Command execution failed: "{cmd}"\nRC: {rc}\n{stderr}'.format(
524 cmd=cmd, rc=ret_code, stderr=stderr)
528 msg = message + '\n' + msg
531 raise RuntimeError(msg)
533 return stdout, stderr
537 node, local_path, remote_path, get=False, timeout=30, disconnect=False):
538 """Copy files from local_path to remote_path or vice versa.
540 :param node: SUT node.
541 :param local_path: Path to local file that should be uploaded; or
542 path where to save remote file.
543 :param remote_path: Remote path where to place uploaded file; or
544 path to remote file which should be downloaded.
545 :param get: scp operation to perform. Default is put.
546 :param timeout: Timeout value in seconds.
547 :param disconnect: Close the opened SSH connection if True.
549 :type local_path: str
550 :type remote_path: str
553 :type disconnect: bool
554 :raises RuntimeError: If SSH connection failed or SCP transfer failed.
560 except SSHException as exc:
561 raise_from(RuntimeError(
562 'Failed to connect to {host}!'.format(host=node['host'])), exc)
564 ssh.scp(local_path, remote_path, get, timeout)
565 except SCPException as exc:
566 raise_from(RuntimeError(
567 'SCP execution failed on {host}!'.format(host=node['host'])), exc)