make test: code cleanup

[vpp.git] / test / run_tests.py

diff --git a/test/run_tests.py b/test/run_tests.py
index fd8ca1f..24c51a1 100644 (file)
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -13,6 +13,12 @@ from debug import spawn_gdb
 from log import global_logger
 from discover_tests import discover_tests
 
+# timeout which controls how long the child has to finish after seeing
+# a core dump in test temporary directory. If this is exceeded, parent assumes
+# that child process is stuck (e.g. waiting for shm mutex, which will never
+# get unlocked) and kill the child
+core_timeout = 3
+
 
 def test_runner_wrapper(suite, keep_alive_pipe, result_pipe, failed_pipe):
     result = not VppTestRunner(
@@ -62,6 +68,8 @@ def run_forked(suite):
     result = None
     failed = set()
     last_heard = time.time()
+    core_detected_at = None
+    debug_core = os.getenv("DEBUG", "").lower() == "core"
     while True:
         readable = select.select([keep_alive_parent_end.fileno(),
                                   result_parent_end.fileno(),
@@ -82,7 +90,8 @@ def run_forked(suite):
                 failed.add(failed_test.__name__)
             last_heard = time.time()
         fail = False
-        if last_heard + test_timeout < time.time():
+        if last_heard + test_timeout < time.time() and \
+                not os.path.isfile("%s/_core_handled" % last_test_temp_dir):
             fail = True
             global_logger.critical("Timeout while waiting for child test "
                                    "runner process (last test running was "
@@ -93,13 +102,29 @@ def run_forked(suite):
             global_logger.critical("Child process unexpectedly died (last "
                                    "test running was `%s' in `%s')!" %
                                    (last_test, last_test_temp_dir))
+        elif last_test_temp_dir and last_test_vpp_binary:
+            core_path = "%s/core" % last_test_temp_dir
+            if os.path.isfile(core_path):
+                if core_detected_at is None:
+                    core_detected_at = time.time()
+                elif core_detected_at + core_timeout < time.time():
+                    if not os.path.isfile(
+                            "%s/_core_handled" % last_test_temp_dir):
+                        global_logger.critical(
+                            "Child unresponsive and core-file exists in test "
+                            "temporary directory!")
+                        fail = True
+
         if fail:
             failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
             lttd = last_test_temp_dir.split("/")[-1]
             link_path = '%s%s-FAILED' % (failed_dir, lttd)
             global_logger.error("Creating a link to the failed " +
                                 "test: %s -> %s" % (link_path, lttd))
-            os.symlink(last_test_temp_dir, link_path)
+            try:
+                os.symlink(last_test_temp_dir, link_path)
+            except:
+                pass
             api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
             if os.path.isfile(api_post_mortem_path):
                 global_logger.error("Copying api_post_mortem.%d to %s" %
@@ -110,7 +135,7 @@ def run_forked(suite):
                 if os.path.isfile(core_path):
                     global_logger.error("Core-file exists in test temporary "
                                         "directory: %s!" % core_path)
-                    if d and d.lower() == "core":
+                    if debug_core:
                         spawn_gdb(last_test_vpp_binary, core_path,
                                   global_logger)
             child.terminate()
@@ -140,6 +165,9 @@ if __name__ == '__main__':
     except:
         debug = None
 
+    s = os.getenv("STEP", "n")
+    step = True if s.lower() in ("y", "yes", "1") else False
+
     parser = argparse.ArgumentParser(description="VPP unit tests")
     parser.add_argument("-f", "--failfast", action='count',
                         help="fast failure flag")
@@ -164,7 +192,11 @@ if __name__ == '__main__':
     attempts = retries + 1
     if attempts > 1:
         print("Perform %s attempts to pass the suite..." % attempts)
-    if debug is None or debug.lower() not in ["gdb", "gdbserver"]:
+    if (debug is not None and debug.lower() in ["gdb", "gdbserver"]) or step:
+        # don't fork if requiring interactive terminal..
+        sys.exit(not VppTestRunner(
+            verbosity=verbose, failfast=failfast).run(suite).wasSuccessful())
+    else:
         while True:
             result, failed = run_forked(suite)
             attempts = attempts - 1
@@ -174,7 +206,3 @@ if __name__ == '__main__':
                 suite = suite_from_failed(suite, failed)
                 continue
             sys.exit(result)
-
-    # don't fork if debugging..
-    sys.exit(not VppTestRunner(verbosity=verbose,
-                               failfast=failfast).run(suite).wasSuccessful())