Fix: Timeout during VPP installation

[csit.git] / resources / tools / topo_installation.py

diff --git a/resources/tools/topo_installation.py b/resources/tools/topo_installation.py
index b052a85..5dbaaaa 100755 (executable)
--- a/resources/tools/topo_installation.py
+++ b/resources/tools/topo_installation.py
@@ -35,9 +35,9 @@ def ssh_no_error(ssh, cmd, sudo=False):
     """
 
     if sudo:
-        ret, stdo, stde = ssh.exec_command_sudo(cmd)
+        ret, stdo, stde = ssh.exec_command_sudo(cmd, timeout=60)
     else:
-        ret, stdo, stde = ssh.exec_command(cmd)
+        ret, stdo, stde = ssh.exec_command(cmd, timeout=60)
 
     if ret != 0:
         print 'Command execution failed: "{}"'.format(cmd)
@@ -47,6 +47,28 @@ def ssh_no_error(ssh, cmd, sudo=False):
 
     return stdo
 
+def ssh_ignore_error(ssh, cmd, sudo=False):
+    """Execute a command over ssh channel, ignore errors.
+
+    :param ssh: SSH() object connected to a node.
+    :param cmd: Command line to execute on remote node.
+    :type ssh: SSH() object
+    :type cmd: str
+    :return: stdout from the SSH command.
+    :rtype: str
+    """
+
+    if sudo:
+        ret, stdo, stde = ssh.exec_command_sudo(cmd)
+    else:
+        ret, stdo, stde = ssh.exec_command(cmd)
+
+    if ret != 0:
+        print 'Command execution failed: "{}"'.format(cmd)
+        print 'stdout: {0}'.format(stdo)
+        print 'stderr: {0}'.format(stde)
+
+    return stdo
 
 def main():
     """Copy and installation of VPP packages."""
@@ -78,12 +100,16 @@ def main():
             if cancel_installation:
                 # Remove installation directory on DUT
                 cmd = "rm -r {}".format(install_dir)
-                stdout = ssh_no_error(ssh, cmd)
+                stdout = ssh_ignore_error(ssh, cmd)
                 print "###TI {}".format(stdout)
 
                 cmd = "dpkg -l | grep vpp"
                 ret, _, _ = ssh.exec_command(cmd)
                 if ret == 0:
+                    # Try to fix interrupted installations
+                    cmd = 'dpkg --configure -a'
+                    stdout = ssh_no_error(ssh, cmd, sudo=True)
+                    print "###TI {}".format(stdout)
                     # Try to remove installed vpp.* packages
                     cmd = 'apt-get purge -y "vpp.*"'
                     stdout = ssh_no_error(ssh, cmd, sudo=True)
@@ -102,6 +128,10 @@ def main():
                 cmd = "dpkg -l | grep vpp"
                 ret, _, _ = ssh.exec_command(cmd)
                 if ret == 0:
+                    # Try to fix interrupted installations
+                    cmd = 'dpkg --configure -a'
+                    stdout = ssh_no_error(ssh, cmd, sudo=True)
+                    print "###TI {}".format(stdout)
                     # Try to remove installed vpp.* packages
                     cmd = 'apt-get purge -y "vpp.*"'
                     stdout = ssh_no_error(ssh, cmd, sudo=True)