test/run_tests.py

   1 #!/usr/bin/env python
   2
   3 import sys
   4 import shutil
   5 import os
   6 import select
   7 import unittest
   8 import argparse
   9 import time
  10 from multiprocessing import Process, Pipe
  11 from framework import VppTestRunner
  12 from debug import spawn_gdb
  13 from log import global_logger
  14 from discover_tests import discover_tests
  15 from subprocess import check_output, CalledProcessError
  16 from util import check_core_path
  17
  18 # timeout which controls how long the child has to finish after seeing
  19 # a core dump in test temporary directory. If this is exceeded, parent assumes
  20 # that child process is stuck (e.g. waiting for shm mutex, which will never
  21 # get unlocked) and kill the child
  22 core_timeout = 3
  23
  24
  25 def test_runner_wrapper(suite, keep_alive_pipe, result_pipe, failed_pipe):
  26     result = not VppTestRunner(
  27         keep_alive_pipe=keep_alive_pipe,
  28         failed_pipe=failed_pipe,
  29         verbosity=verbose,
  30         failfast=failfast).run(suite).wasSuccessful()
  31     result_pipe.send(result)
  32     result_pipe.close()
  33     keep_alive_pipe.close()
  34     failed_pipe.close()
  35
  36
  37 class add_to_suite_callback:
  38     def __init__(self, suite):
  39         self.suite = suite
  40
  41     def __call__(self, file_name, cls, method):
  42         suite.addTest(cls(method))
  43
  44
  45 class Filter_by_class_list:
  46     def __init__(self, class_list):
  47         self.class_list = class_list
  48
  49     def __call__(self, file_name, class_name, func_name):
  50         return class_name in self.class_list
  51
  52
  53 def suite_from_failed(suite, failed):
  54     filter_cb = Filter_by_class_list(failed)
  55     suite = VppTestRunner.filter_tests(suite, filter_cb)
  56     if 0 == suite.countTestCases():
  57         raise Exception("Suite is empty after filtering out the failed tests!")
  58     return suite
  59
  60
  61 def run_forked(suite):
  62     keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
  63     result_parent_end, result_child_end = Pipe(duplex=False)
  64     failed_parent_end, failed_child_end = Pipe(duplex=False)
  65
  66     child = Process(target=test_runner_wrapper,
  67                     args=(suite, keep_alive_child_end, result_child_end,
  68                           failed_child_end))
  69     child.start()
  70     last_test_temp_dir = None
  71     last_test_vpp_binary = None
  72     last_test = None
  73     result = None
  74     failed = set()
  75     last_heard = time.time()
  76     core_detected_at = None
  77     debug_core = os.getenv("DEBUG", "").lower() == "core"
  78     while True:
  79         readable = select.select([keep_alive_parent_end.fileno(),
  80                                   result_parent_end.fileno(),
  81                                   failed_parent_end.fileno(),
  82                                   ],
  83                                  [], [], 1)[0]
  84         if result_parent_end.fileno() in readable:
  85             result = result_parent_end.recv()
  86             break
  87         if keep_alive_parent_end.fileno() in readable:
  88             while keep_alive_parent_end.poll():
  89                 last_test, last_test_vpp_binary,\
  90                     last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
  91             last_heard = time.time()
  92         if failed_parent_end.fileno() in readable:
  93             while failed_parent_end.poll():
  94                 failed_test = failed_parent_end.recv()
  95                 failed.add(failed_test.__name__)
  96             last_heard = time.time()
  97         fail = False
  98         if last_heard + test_timeout < time.time() and \
  99                 not os.path.isfile("%s/_core_handled" % last_test_temp_dir):
 100             fail = True
 101             global_logger.critical("Timeout while waiting for child test "
 102                                    "runner process (last test running was "
 103                                    "`%s' in `%s')!" %
 104                                    (last_test, last_test_temp_dir))
 105         elif not child.is_alive():
 106             fail = True
 107             global_logger.critical("Child python process unexpectedly died "
 108                                    "(last test running was `%s' in `%s')!" %
 109                                    (last_test, last_test_temp_dir))
 110         elif last_test_temp_dir and last_test_vpp_binary:
 111             core_path = "%s/core" % last_test_temp_dir
 112             if os.path.isfile(core_path):
 113                 if core_detected_at is None:
 114                     core_detected_at = time.time()
 115                 elif core_detected_at + core_timeout < time.time():
 116                     if not os.path.isfile(
 117                             "%s/_core_handled" % last_test_temp_dir):
 118                         global_logger.critical(
 119                             "Child python process unresponsive and core-file "
 120                             "exists in test temporary directory!")
 121                         fail = True
 122
 123         if fail:
 124             failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
 125             lttd = last_test_temp_dir.split("/")[-1]
 126             link_path = '%s%s-FAILED' % (failed_dir, lttd)
 127             global_logger.error("Creating a link to the failed " +
 128                                 "test: %s -> %s" % (link_path, lttd))
 129             try:
 130                 os.symlink(last_test_temp_dir, link_path)
 131             except Exception:
 132                 pass
 133             api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
 134             if os.path.isfile(api_post_mortem_path):
 135                 global_logger.error("Copying api_post_mortem.%d to %s" %
 136                                     (vpp_pid, last_test_temp_dir))
 137                 shutil.copy2(api_post_mortem_path, last_test_temp_dir)
 138             if last_test_temp_dir and last_test_vpp_binary:
 139                 core_path = "%s/core" % last_test_temp_dir
 140                 if os.path.isfile(core_path):
 141                     global_logger.error("Core-file exists in test temporary "
 142                                         "directory: %s!" % core_path)
 143                     check_core_path(global_logger, core_path)
 144                     global_logger.debug("Running `file %s':" % core_path)
 145                     try:
 146                         info = check_output(["file", core_path])
 147                         global_logger.debug(info)
 148                     except CalledProcessError as e:
 149                         global_logger.error(
 150                             "Could not run `file' utility on core-file, "
 151                             "rc=%s" % e.returncode)
 152                         pass
 153                     if debug_core:
 154                         spawn_gdb(last_test_vpp_binary, core_path,
 155                                   global_logger)
 156             child.terminate()
 157             result = -1
 158             break
 159     keep_alive_parent_end.close()
 160     result_parent_end.close()
 161     failed_parent_end.close()
 162     return result, failed
 163
 164
 165 if __name__ == '__main__':
 166
 167     try:
 168         verbose = int(os.getenv("V", 0))
 169     except ValueError:
 170         verbose = 0
 171
 172     default_test_timeout = 600  # 10 minutes
 173     try:
 174         test_timeout = int(os.getenv("TIMEOUT", default_test_timeout))
 175     except ValueError:
 176         test_timeout = default_test_timeout
 177
 178     debug = os.getenv("DEBUG")
 179
 180     s = os.getenv("STEP", "n")
 181     step = True if s.lower() in ("y", "yes", "1") else False
 182
 183     parser = argparse.ArgumentParser(description="VPP unit tests")
 184     parser.add_argument("-f", "--failfast", action='count',
 185                         help="fast failure flag")
 186     parser.add_argument("-d", "--dir", action='append', type=str,
 187                         help="directory containing test files "
 188                              "(may be specified multiple times)")
 189     args = parser.parse_args()
 190     failfast = True if args.failfast == 1 else False
 191
 192     suite = unittest.TestSuite()
 193     cb = add_to_suite_callback(suite)
 194     for d in args.dir:
 195         print("Adding tests from directory tree %s" % d)
 196         discover_tests(d, cb)
 197
 198     try:
 199         retries = int(os.getenv("RETRIES", 0))
 200     except ValueError:
 201         retries = 0
 202
 203     try:
 204         force_foreground = int(os.getenv("FORCE_FOREGROUND", 0))
 205     except ValueError:
 206         force_foreground = 0
 207     attempts = retries + 1
 208     if attempts > 1:
 209         print("Perform %s attempts to pass the suite..." % attempts)
 210     if (debug is not None and debug.lower() in ["gdb", "gdbserver"]) or step\
 211             or force_foreground:
 212         # don't fork if requiring interactive terminal..
 213         sys.exit(not VppTestRunner(
 214             verbosity=verbose, failfast=failfast).run(suite).wasSuccessful())
 215     else:
 216         while True:
 217             result, failed = run_forked(suite)
 218             attempts = attempts - 1
 219             print("%s test(s) failed, %s attempt(s) left" %
 220                   (len(failed), attempts))
 221             if len(failed) > 0 and attempts > 0:
 222                 suite = suite_from_failed(suite, failed)
 223                 continue
 224             sys.exit(result)