CSIT-563: HC Test: improved Lisp test coverage

[csit.git] / resources / tools / topo_installation.py

diff --git a/resources/tools/topo_installation.py b/resources/tools/topo_installation.py
index dc45d23..0488bda 100755 (executable)
--- a/resources/tools/topo_installation.py
+++ b/resources/tools/topo_installation.py
@@ -22,6 +22,7 @@ from yaml import load
 
 from resources.libraries.python.ssh import SSH
 
+
 def ssh_no_error(ssh, cmd, sudo=False):
     """Execute a command over ssh channel, and log and exit if the command
     fails.
@@ -34,16 +35,40 @@ def ssh_no_error(ssh, cmd, sudo=False):
     :rtype: str
     """
 
-    if sudo == True:
+    if sudo:
+        ret, stdo, stde = ssh.exec_command_sudo(cmd, timeout=60)
+    else:
+        ret, stdo, stde = ssh.exec_command(cmd, timeout=60)
+
+    if ret != 0:
+        print 'Command execution failed: "{}"'.format(cmd)
+        print 'stdout: {0}'.format(stdo)
+        print 'stderr: {0}'.format(stde)
+        raise RuntimeError('Unexpected ssh command failure')
+
+    return stdo
+
+
+def ssh_ignore_error(ssh, cmd, sudo=False):
+    """Execute a command over ssh channel, ignore errors.
+
+    :param ssh: SSH() object connected to a node.
+    :param cmd: Command line to execute on remote node.
+    :type ssh: SSH() object
+    :type cmd: str
+    :return: stdout from the SSH command.
+    :rtype: str
+    """
+
+    if sudo:
         ret, stdo, stde = ssh.exec_command_sudo(cmd)
     else:
         ret, stdo, stde = ssh.exec_command(cmd)
 
-    if 0 != ret:
+    if ret != 0:
         print 'Command execution failed: "{}"'.format(cmd)
         print 'stdout: {0}'.format(stdo)
         print 'stderr: {0}'.format(stde)
-        raise RuntimeError('Unexpected ssh command failure')
 
     return stdo
 
@@ -78,11 +103,23 @@ def main():
             if cancel_installation:
                 # Remove installation directory on DUT
                 cmd = "rm -r {}".format(install_dir)
-                stdout = ssh_no_error(ssh, cmd)
+                stdout = ssh_ignore_error(ssh, cmd)
                 print "###TI {}".format(stdout)
+
+                cmd = "dpkg -l | grep vpp"
+                ret, _, _ = ssh.exec_command(cmd)
+                if ret == 0:
+                    # Try to fix interrupted installations
+                    cmd = 'dpkg --configure -a'
+                    stdout = ssh_no_error(ssh, cmd, sudo=True)
+                    print "###TI {}".format(stdout)
+                    # Try to remove installed vpp.* packages
+                    cmd = 'apt-get purge -y "vpp.*"'
+                    stdout = ssh_no_error(ssh, cmd, sudo=True)
+                    print "###TI {}".format(stdout)
             else:
                 # Create installation directory on DUT
-                cmd = "mkdir {}".format(install_dir)
+                cmd = "rm -r {0}; mkdir {0}".format(install_dir)
                 stdout = ssh_no_error(ssh, cmd)
                 print "###TI {}".format(stdout)
 
@@ -91,6 +128,18 @@ def main():
                     print "###TI scp: {}".format(deb)
                     ssh.scp(local_path=deb, remote_path=install_dir)
 
+                cmd = "dpkg -l | grep vpp"
+                ret, _, _ = ssh.exec_command(cmd)
+                if ret == 0:
+                    # Try to fix interrupted installations
+                    cmd = 'dpkg --configure -a'
+                    stdout = ssh_no_error(ssh, cmd, sudo=True)
+                    print "###TI {}".format(stdout)
+                    # Try to remove installed vpp.* packages
+                    cmd = 'apt-get purge -y "vpp.*"'
+                    stdout = ssh_no_error(ssh, cmd, sudo=True)
+                    print "###TI {}".format(stdout)
+
                 # Installation of VPP deb packages
                 cmd = "dpkg -i --force-all {}/*.deb".format(install_dir)
                 stdout = ssh_no_error(ssh, cmd, sudo=True)