test/run_tests.py

   1 #!/usr/bin/env python
   2
   3 import sys
   4 import shutil
   5 import os
   6 import select
   7 import unittest
   8 import argparse
   9 import time
  10 from multiprocessing import Process, Pipe
  11 from framework import VppTestRunner
  12 from debug import spawn_gdb
  13 from log import global_logger
  14 from discover_tests import discover_tests
  15 from subprocess import check_output, CalledProcessError
  16
  17 # timeout which controls how long the child has to finish after seeing
  18 # a core dump in test temporary directory. If this is exceeded, parent assumes
  19 # that child process is stuck (e.g. waiting for shm mutex, which will never
  20 # get unlocked) and kill the child
  21 core_timeout = 3
  22
  23
  24 def test_runner_wrapper(suite, keep_alive_pipe, result_pipe, failed_pipe):
  25     result = not VppTestRunner(
  26         keep_alive_pipe=keep_alive_pipe,
  27         failed_pipe=failed_pipe,
  28         verbosity=verbose,
  29         failfast=failfast).run(suite).wasSuccessful()
  30     result_pipe.send(result)
  31     result_pipe.close()
  32     keep_alive_pipe.close()
  33     failed_pipe.close()
  34
  35
  36 class add_to_suite_callback:
  37     def __init__(self, suite):
  38         self.suite = suite
  39
  40     def __call__(self, file_name, cls, method):
  41         suite.addTest(cls(method))
  42
  43
  44 class Filter_by_class_list:
  45     def __init__(self, class_list):
  46         self.class_list = class_list
  47
  48     def __call__(self, file_name, class_name, func_name):
  49         return class_name in self.class_list
  50
  51
  52 def suite_from_failed(suite, failed):
  53     filter_cb = Filter_by_class_list(failed)
  54     return VppTestRunner.filter_tests(suite, filter_cb)
  55
  56
  57 def run_forked(suite):
  58     keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
  59     result_parent_end, result_child_end = Pipe(duplex=False)
  60     failed_parent_end, failed_child_end = Pipe(duplex=False)
  61
  62     child = Process(target=test_runner_wrapper,
  63                     args=(suite, keep_alive_child_end, result_child_end,
  64                           failed_child_end))
  65     child.start()
  66     last_test_temp_dir = None
  67     last_test_vpp_binary = None
  68     last_test = None
  69     result = None
  70     failed = set()
  71     last_heard = time.time()
  72     core_detected_at = None
  73     debug_core = os.getenv("DEBUG", "").lower() == "core"
  74     while True:
  75         readable = select.select([keep_alive_parent_end.fileno(),
  76                                   result_parent_end.fileno(),
  77                                   failed_parent_end.fileno(),
  78                                   ],
  79                                  [], [], 1)[0]
  80         if result_parent_end.fileno() in readable:
  81             result = result_parent_end.recv()
  82             break
  83         if keep_alive_parent_end.fileno() in readable:
  84             while keep_alive_parent_end.poll():
  85                 last_test, last_test_vpp_binary,\
  86                     last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
  87             last_heard = time.time()
  88         if failed_parent_end.fileno() in readable:
  89             while failed_parent_end.poll():
  90                 failed_test = failed_parent_end.recv()
  91                 failed.add(failed_test.__name__)
  92             last_heard = time.time()
  93         fail = False
  94         if last_heard + test_timeout < time.time() and \
  95                 not os.path.isfile("%s/_core_handled" % last_test_temp_dir):
  96             fail = True
  97             global_logger.critical("Timeout while waiting for child test "
  98                                    "runner process (last test running was "
  99                                    "`%s' in `%s')!" %
 100                                    (last_test, last_test_temp_dir))
 101         elif not child.is_alive():
 102             fail = True
 103             global_logger.critical("Child python process unexpectedly died "
 104                                    "(last test running was `%s' in `%s')!" %
 105                                    (last_test, last_test_temp_dir))
 106         elif last_test_temp_dir and last_test_vpp_binary:
 107             core_path = "%s/core" % last_test_temp_dir
 108             if os.path.isfile(core_path):
 109                 if core_detected_at is None:
 110                     core_detected_at = time.time()
 111                 elif core_detected_at + core_timeout < time.time():
 112                     if not os.path.isfile(
 113                             "%s/_core_handled" % last_test_temp_dir):
 114                         global_logger.critical(
 115                             "Child python process unresponsive and core-file "
 116                             "exists in test temporary directory!")
 117                         fail = True
 118
 119         if fail:
 120             failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
 121             lttd = last_test_temp_dir.split("/")[-1]
 122             link_path = '%s%s-FAILED' % (failed_dir, lttd)
 123             global_logger.error("Creating a link to the failed " +
 124                                 "test: %s -> %s" % (link_path, lttd))
 125             try:
 126                 os.symlink(last_test_temp_dir, link_path)
 127             except Exception:
 128                 pass
 129             api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
 130             if os.path.isfile(api_post_mortem_path):
 131                 global_logger.error("Copying api_post_mortem.%d to %s" %
 132                                     (vpp_pid, last_test_temp_dir))
 133                 shutil.copy2(api_post_mortem_path, last_test_temp_dir)
 134             if last_test_temp_dir and last_test_vpp_binary:
 135                 core_path = "%s/core" % last_test_temp_dir
 136                 if os.path.isfile(core_path):
 137                     global_logger.error("Core-file exists in test temporary "
 138                                         "directory: %s!" % core_path)
 139                     global_logger.debug("Running `file %s':" % core_path)
 140                     try:
 141                         info = check_output(["file", core_path])
 142                         global_logger.debug(info)
 143                     except CalledProcessError as e:
 144                         global_logger.error(
 145                             "Could not run `file' utility on core-file, "
 146                             "rc=%s" % e.returncode)
 147                         pass
 148                     if debug_core:
 149                         spawn_gdb(last_test_vpp_binary, core_path,
 150                                   global_logger)
 151             child.terminate()
 152             result = -1
 153             break
 154     keep_alive_parent_end.close()
 155     result_parent_end.close()
 156     failed_parent_end.close()
 157     return result, failed
 158
 159
 160 if __name__ == '__main__':
 161
 162     try:
 163         verbose = int(os.getenv("V", 0))
 164     except ValueError:
 165         verbose = 0
 166
 167     default_test_timeout = 600  # 10 minutes
 168     try:
 169         test_timeout = int(os.getenv("TIMEOUT", default_test_timeout))
 170     except ValueError:
 171         test_timeout = default_test_timeout
 172
 173     debug = os.getenv("DEBUG")
 174
 175     s = os.getenv("STEP", "n")
 176     step = True if s.lower() in ("y", "yes", "1") else False
 177
 178     parser = argparse.ArgumentParser(description="VPP unit tests")
 179     parser.add_argument("-f", "--failfast", action='count',
 180                         help="fast failure flag")
 181     parser.add_argument("-d", "--dir", action='append', type=str,
 182                         help="directory containing test files "
 183                              "(may be specified multiple times)")
 184     args = parser.parse_args()
 185     failfast = True if args.failfast == 1 else False
 186
 187     suite = unittest.TestSuite()
 188     cb = add_to_suite_callback(suite)
 189     for d in args.dir:
 190         print("Adding tests from directory tree %s" % d)
 191         discover_tests(d, cb)
 192
 193     try:
 194         retries = int(os.getenv("RETRIES", 0))
 195     except ValueError:
 196         retries = 0
 197     attempts = retries + 1
 198     if attempts > 1:
 199         print("Perform %s attempts to pass the suite..." % attempts)
 200     if (debug is not None and debug.lower() in ["gdb", "gdbserver"]) or step:
 201         # don't fork if requiring interactive terminal..
 202         sys.exit(not VppTestRunner(
 203             verbosity=verbose, failfast=failfast).run(suite).wasSuccessful())
 204     else:
 205         while True:
 206             result, failed = run_forked(suite)
 207             attempts = attempts - 1
 208             print("%s test(s) failed, %s attempt(s) left" %
 209                   (len(failed), attempts))
 210             if len(failed) > 0 and attempts > 0:
 211                 suite = suite_from_failed(suite, failed)
 212                 continue
 213             sys.exit(result)