FrameworkSetup: catch all exceptions in threads

[csit.git] / resources / libraries / python / SetupFramework.py

diff --git a/resources/libraries/python/SetupFramework.py b/resources/libraries/python/SetupFramework.py
index 45447e9..c27dbe0 100644 (file)
--- a/resources/libraries/python/SetupFramework.py
+++ b/resources/libraries/python/SetupFramework.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -19,6 +19,7 @@ supposed to end up here.
 from os import environ, remove
 from tempfile import NamedTemporaryFile
 import threading
+import traceback
 
 from robot.api import logger
 
@@ -126,7 +127,8 @@ def create_env_directory_at_node(node):
     )
     cmd = f"cd {con.REMOTE_FW_DIR} && rm -rf env && virtualenv " \
         f"-p $(which python3) --system-site-packages --never-download env " \
-        f"&& source env/bin/activate && pip3 install -r requirements.txt"
+        f"&& source env/bin/activate && ANSIBLE_SKIP_CONFLICT_CHECK=1 " \
+        f"pip3 install -r requirements.txt"
     exec_cmd_no_error(
         node, cmd, timeout=100, include_reason=True,
         message=f"Failed install at node {node[u'type']} host {node[u'host']}, "
@@ -138,17 +140,19 @@ def create_env_directory_at_node(node):
     )
 
 
-def setup_node(node, tarball, remote_tarball, results=None):
+def setup_node(node, tarball, remote_tarball, results=None, logs=None):
     """Copy a tarball to a node and extract it.
 
     :param node: A node where the tarball will be copied and extracted.
     :param tarball: Local path of tarball to be copied.
     :param remote_tarball: Remote path of the tarball.
     :param results: A list where to store the result of node setup, optional.
+    :param logs: A list where to store anything that should be logged.
     :type node: dict
     :type tarball: str
     :type remote_tarball: str
     :type results: list
+    :type logs: list
     :returns: True - success, False - error
     :rtype: bool
     """
@@ -157,11 +161,14 @@ def setup_node(node, tarball, remote_tarball, results=None):
         extract_tarball_at_node(remote_tarball, node)
         if node[u"type"] == NodeType.TG:
             create_env_directory_at_node(node)
-    except RuntimeError as exc:
-        logger.console(
-            f"Node {node[u'type']} host {node[u'host']}, port {node[u'port']} "
-            f"setup failed, error: {exc!r}"
-        )
+    except Exception:
+        # any exception must result in result = False
+        # since this runs in a thread and can't be caught anywhere else
+        err_msg = f"Node {node[u'type']} host {node[u'host']}, " \
+                  f"port {node[u'port']} setup failed."
+        logger.console(err_msg)
+        if isinstance(logs, list):
+            logs.append(f"{err_msg} Exception: {traceback.format_exc()}")
         result = False
     else:
         logger.console(
@@ -207,23 +214,26 @@ def delete_framework_dir(node):
     )
 
 
-def cleanup_node(node, results=None):
+def cleanup_node(node, results=None, logs=None):
     """Delete a tarball from a node.
 
     :param node: A node where the tarball will be delete.
     :param results: A list where to store the result of node cleanup, optional.
+    :param logs: A list where to store anything that should be logged.
     :type node: dict
     :type results: list
+    :type logs: list
     :returns: True - success, False - error
     :rtype: bool
     """
     try:
         delete_framework_dir(node)
-    except RuntimeError:
-        logger.error(
-            f"Cleanup of node {node[u'type']} host {node[u'host']}, "
-            f"port {node[u'port']} failed."
-        )
+    except Exception:
+        err_msg = f"Cleanup of node {node[u'type']} host {node[u'host']}, " \
+                  f"port {node[u'port']} failed."
+        logger.console(err_msg)
+        if isinstance(logs, list):
+            logs.append(f"{err_msg} Exception: {traceback.format_exc()}")
         result = False
     else:
         logger.console(
@@ -261,16 +271,17 @@ class SetupFramework:
         remote_tarball = f"{tarball}"
 
         results = list()
+        logs = list()
         threads = list()
 
         for node in nodes.values():
-            args = node, tarball, remote_tarball, results
+            args = node, tarball, remote_tarball, results, logs
             thread = threading.Thread(target=setup_node, args=args)
             thread.start()
             threads.append(thread)
 
         logger.info(
-            f"Executing node setups in parallel, waiting for threads to end"
+            u"Executing node setups in parallel, waiting for threads to end."
         )
 
         for thread in threads:
@@ -278,6 +289,9 @@ class SetupFramework:
 
         logger.info(f"Results: {results}")
 
+        for log in logs:
+            logger.trace(log)
+
         delete_local_tarball(tarball)
         if all(results):
             logger.console(u"All nodes are ready.")
@@ -303,10 +317,12 @@ class CleanupFramework:
         """
 
         results = list()
+        logs = list()
         threads = list()
 
         for node in nodes.values():
-            thread = threading.Thread(target=cleanup_node, args=(node, results))
+            thread = threading.Thread(target=cleanup_node,
+                                      args=(node, results, logs))
             thread.start()
             threads.append(thread)
 
@@ -319,6 +335,9 @@ class CleanupFramework:
 
         logger.info(f"Results: {results}")
 
+        for log in logs:
+            logger.trace(log)
+
         if all(results):
             logger.console(u"All nodes cleaned up.")
         else: