test/run_tests.py

   1 #!/usr/bin/env python
   2
   3 import sys
   4 import shutil
   5 import os
   6 import select
   7 import unittest
   8 import argparse
   9 import time
  10 from multiprocessing import Process, Pipe
  11 from framework import VppTestRunner
  12 from debug import spawn_gdb
  13 from log import global_logger
  14 from discover_tests import discover_tests
  15 from subprocess import check_output, CalledProcessError
  16 from util import check_core_path
  17
  18 # timeout which controls how long the child has to finish after seeing
  19 # a core dump in test temporary directory. If this is exceeded, parent assumes
  20 # that child process is stuck (e.g. waiting for shm mutex, which will never
  21 # get unlocked) and kill the child
  22 core_timeout = 3
  23
  24
  25 def test_runner_wrapper(suite, keep_alive_pipe, result_pipe, failed_pipe):
  26     result = not VppTestRunner(
  27         keep_alive_pipe=keep_alive_pipe,
  28         failed_pipe=failed_pipe,
  29         verbosity=verbose,
  30         failfast=failfast).run(suite).wasSuccessful()
  31     result_pipe.send(result)
  32     result_pipe.close()
  33     keep_alive_pipe.close()
  34     failed_pipe.close()
  35
  36
  37 class add_to_suite_callback:
  38     def __init__(self, suite):
  39         self.suite = suite
  40
  41     def __call__(self, file_name, cls, method):
  42         suite.addTest(cls(method))
  43
  44
  45 class Filter_by_class_list:
  46     def __init__(self, class_list):
  47         self.class_list = class_list
  48
  49     def __call__(self, file_name, class_name, func_name):
  50         return class_name in self.class_list
  51
  52
  53 def suite_from_failed(suite, failed):
  54     filter_cb = Filter_by_class_list(failed)
  55     return VppTestRunner.filter_tests(suite, filter_cb)
  56
  57
  58 def run_forked(suite):
  59     keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
  60     result_parent_end, result_child_end = Pipe(duplex=False)
  61     failed_parent_end, failed_child_end = Pipe(duplex=False)
  62
  63     child = Process(target=test_runner_wrapper,
  64                     args=(suite, keep_alive_child_end, result_child_end,
  65                           failed_child_end))
  66     child.start()
  67     last_test_temp_dir = None
  68     last_test_vpp_binary = None
  69     last_test = None
  70     result = None
  71     failed = set()
  72     last_heard = time.time()
  73     core_detected_at = None
  74     debug_core = os.getenv("DEBUG", "").lower() == "core"
  75     while True:
  76         readable = select.select([keep_alive_parent_end.fileno(),
  77                                   result_parent_end.fileno(),
  78                                   failed_parent_end.fileno(),
  79                                   ],
  80                                  [], [], 1)[0]
  81         if result_parent_end.fileno() in readable:
  82             result = result_parent_end.recv()
  83             break
  84         if keep_alive_parent_end.fileno() in readable:
  85             while keep_alive_parent_end.poll():
  86                 last_test, last_test_vpp_binary,\
  87                     last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
  88             last_heard = time.time()
  89         if failed_parent_end.fileno() in readable:
  90             while failed_parent_end.poll():
  91                 failed_test = failed_parent_end.recv()
  92                 failed.add(failed_test.__name__)
  93             last_heard = time.time()
  94         fail = False
  95         if last_heard + test_timeout < time.time() and \
  96                 not os.path.isfile("%s/_core_handled" % last_test_temp_dir):
  97             fail = True
  98             global_logger.critical("Timeout while waiting for child test "
  99                                    "runner process (last test running was "
 100                                    "`%s' in `%s')!" %
 101                                    (last_test, last_test_temp_dir))
 102         elif not child.is_alive():
 103             fail = True
 104             global_logger.critical("Child python process unexpectedly died "
 105                                    "(last test running was `%s' in `%s')!" %
 106                                    (last_test, last_test_temp_dir))
 107         elif last_test_temp_dir and last_test_vpp_binary:
 108             core_path = "%s/core" % last_test_temp_dir
 109             if os.path.isfile(core_path):
 110                 if core_detected_at is None:
 111                     core_detected_at = time.time()
 112                 elif core_detected_at + core_timeout < time.time():
 113                     if not os.path.isfile(
 114                             "%s/_core_handled" % last_test_temp_dir):
 115                         global_logger.critical(
 116                             "Child python process unresponsive and core-file "
 117                             "exists in test temporary directory!")
 118                         fail = True
 119
 120         if fail:
 121             failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
 122             lttd = last_test_temp_dir.split("/")[-1]
 123             link_path = '%s%s-FAILED' % (failed_dir, lttd)
 124             global_logger.error("Creating a link to the failed " +
 125                                 "test: %s -> %s" % (link_path, lttd))
 126             try:
 127                 os.symlink(last_test_temp_dir, link_path)
 128             except Exception:
 129                 pass
 130             api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
 131             if os.path.isfile(api_post_mortem_path):
 132                 global_logger.error("Copying api_post_mortem.%d to %s" %
 133                                     (vpp_pid, last_test_temp_dir))
 134                 shutil.copy2(api_post_mortem_path, last_test_temp_dir)
 135             if last_test_temp_dir and last_test_vpp_binary:
 136                 core_path = "%s/core" % last_test_temp_dir
 137                 if os.path.isfile(core_path):
 138                     global_logger.error("Core-file exists in test temporary "
 139                                         "directory: %s!" % core_path)
 140                     check_core_path(global_logger, core_path)
 141                     global_logger.debug("Running `file %s':" % core_path)
 142                     try:
 143                         info = check_output(["file", core_path])
 144                         global_logger.debug(info)
 145                     except CalledProcessError as e:
 146                         global_logger.error(
 147                             "Could not run `file' utility on core-file, "
 148                             "rc=%s" % e.returncode)
 149                         pass
 150                     if debug_core:
 151                         spawn_gdb(last_test_vpp_binary, core_path,
 152                                   global_logger)
 153             child.terminate()
 154             result = -1
 155             break
 156     keep_alive_parent_end.close()
 157     result_parent_end.close()
 158     failed_parent_end.close()
 159     return result, failed
 160
 161
 162 if __name__ == '__main__':
 163
 164     try:
 165         verbose = int(os.getenv("V", 0))
 166     except ValueError:
 167         verbose = 0
 168
 169     default_test_timeout = 600  # 10 minutes
 170     try:
 171         test_timeout = int(os.getenv("TIMEOUT", default_test_timeout))
 172     except ValueError:
 173         test_timeout = default_test_timeout
 174
 175     debug = os.getenv("DEBUG")
 176
 177     s = os.getenv("STEP", "n")
 178     step = True if s.lower() in ("y", "yes", "1") else False
 179
 180     parser = argparse.ArgumentParser(description="VPP unit tests")
 181     parser.add_argument("-f", "--failfast", action='count',
 182                         help="fast failure flag")
 183     parser.add_argument("-d", "--dir", action='append', type=str,
 184                         help="directory containing test files "
 185                              "(may be specified multiple times)")
 186     args = parser.parse_args()
 187     failfast = True if args.failfast == 1 else False
 188
 189     suite = unittest.TestSuite()
 190     cb = add_to_suite_callback(suite)
 191     for d in args.dir:
 192         print("Adding tests from directory tree %s" % d)
 193         discover_tests(d, cb)
 194
 195     try:
 196         retries = int(os.getenv("RETRIES", 0))
 197     except ValueError:
 198         retries = 0
 199     attempts = retries + 1
 200     if attempts > 1:
 201         print("Perform %s attempts to pass the suite..." % attempts)
 202     if (debug is not None and debug.lower() in ["gdb", "gdbserver"]) or step:
 203         # don't fork if requiring interactive terminal..
 204         sys.exit(not VppTestRunner(
 205             verbosity=verbose, failfast=failfast).run(suite).wasSuccessful())
 206     else:
 207         while True:
 208             result, failed = run_forked(suite)
 209             attempts = attempts - 1
 210             print("%s test(s) failed, %s attempt(s) left" %
 211                   (len(failed), attempts))
 212             if len(failed) > 0 and attempts > 0:
 213                 suite = suite_from_failed(suite, failed)
 214                 continue
 215             sys.exit(result)