test/run_tests.py

   1 #!/usr/bin/env python
   2
   3 import sys
   4 import shutil
   5 import os
   6 import select
   7 import unittest
   8 import argparse
   9 import time
  10 from multiprocessing import Process, Pipe
  11 from framework import VppTestRunner
  12 from debug import spawn_gdb
  13 from log import global_logger
  14 from discover_tests import discover_tests
  15
  16 # timeout which controls how long the child has to finish after seeing
  17 # a core dump in test temporary directory. If this is exceeded, parent assumes
  18 # that child process is stuck (e.g. waiting for shm mutex, which will never
  19 # get unlocked) and kill the child
  20 core_timeout = 3
  21
  22
  23 def test_runner_wrapper(suite, keep_alive_pipe, result_pipe, failed_pipe):
  24     result = not VppTestRunner(
  25         keep_alive_pipe=keep_alive_pipe,
  26         failed_pipe=failed_pipe,
  27         verbosity=verbose,
  28         failfast=failfast).run(suite).wasSuccessful()
  29     result_pipe.send(result)
  30     result_pipe.close()
  31     keep_alive_pipe.close()
  32     failed_pipe.close()
  33
  34
  35 class add_to_suite_callback:
  36     def __init__(self, suite):
  37         self.suite = suite
  38
  39     def __call__(self, file_name, cls, method):
  40         suite.addTest(cls(method))
  41
  42
  43 class Filter_by_class_list:
  44     def __init__(self, class_list):
  45         self.class_list = class_list
  46
  47     def __call__(self, file_name, class_name, func_name):
  48         return class_name in self.class_list
  49
  50
  51 def suite_from_failed(suite, failed):
  52     filter_cb = Filter_by_class_list(failed)
  53     return VppTestRunner.filter_tests(suite, filter_cb)
  54
  55
  56 def run_forked(suite):
  57     keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
  58     result_parent_end, result_child_end = Pipe(duplex=False)
  59     failed_parent_end, failed_child_end = Pipe(duplex=False)
  60
  61     child = Process(target=test_runner_wrapper,
  62                     args=(suite, keep_alive_child_end, result_child_end,
  63                           failed_child_end))
  64     child.start()
  65     last_test_temp_dir = None
  66     last_test_vpp_binary = None
  67     last_test = None
  68     result = None
  69     failed = set()
  70     last_heard = time.time()
  71     core_detected_at = None
  72     debug_core = os.getenv("DEBUG", "").lower() == "core"
  73     while True:
  74         readable = select.select([keep_alive_parent_end.fileno(),
  75                                   result_parent_end.fileno(),
  76                                   failed_parent_end.fileno(),
  77                                   ],
  78                                  [], [], 1)[0]
  79         if result_parent_end.fileno() in readable:
  80             result = result_parent_end.recv()
  81             break
  82         if keep_alive_parent_end.fileno() in readable:
  83             while keep_alive_parent_end.poll():
  84                 last_test, last_test_vpp_binary,\
  85                     last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
  86             last_heard = time.time()
  87         if failed_parent_end.fileno() in readable:
  88             while failed_parent_end.poll():
  89                 failed_test = failed_parent_end.recv()
  90                 failed.add(failed_test.__name__)
  91             last_heard = time.time()
  92         fail = False
  93         if last_heard + test_timeout < time.time() and \
  94                 not os.path.isfile("%s/_core_handled" % last_test_temp_dir):
  95             fail = True
  96             global_logger.critical("Timeout while waiting for child test "
  97                                    "runner process (last test running was "
  98                                    "`%s' in `%s')!" %
  99                                    (last_test, last_test_temp_dir))
 100         elif not child.is_alive():
 101             fail = True
 102             global_logger.critical("Child process unexpectedly died (last "
 103                                    "test running was `%s' in `%s')!" %
 104                                    (last_test, last_test_temp_dir))
 105         elif last_test_temp_dir and last_test_vpp_binary:
 106             core_path = "%s/core" % last_test_temp_dir
 107             if os.path.isfile(core_path):
 108                 if core_detected_at is None:
 109                     core_detected_at = time.time()
 110                 elif core_detected_at + core_timeout < time.time():
 111                     if not os.path.isfile(
 112                             "%s/_core_handled" % last_test_temp_dir):
 113                         global_logger.critical(
 114                             "Child unresponsive and core-file exists in test "
 115                             "temporary directory!")
 116                         fail = True
 117
 118         if fail:
 119             failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
 120             lttd = last_test_temp_dir.split("/")[-1]
 121             link_path = '%s%s-FAILED' % (failed_dir, lttd)
 122             global_logger.error("Creating a link to the failed " +
 123                                 "test: %s -> %s" % (link_path, lttd))
 124             try:
 125                 os.symlink(last_test_temp_dir, link_path)
 126             except:
 127                 pass
 128             api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
 129             if os.path.isfile(api_post_mortem_path):
 130                 global_logger.error("Copying api_post_mortem.%d to %s" %
 131                                     (vpp_pid, last_test_temp_dir))
 132                 shutil.copy2(api_post_mortem_path, last_test_temp_dir)
 133             if last_test_temp_dir and last_test_vpp_binary:
 134                 core_path = "%s/core" % last_test_temp_dir
 135                 if os.path.isfile(core_path):
 136                     global_logger.error("Core-file exists in test temporary "
 137                                         "directory: %s!" % core_path)
 138                     if debug_core:
 139                         spawn_gdb(last_test_vpp_binary, core_path,
 140                                   global_logger)
 141             child.terminate()
 142             result = -1
 143             break
 144     keep_alive_parent_end.close()
 145     result_parent_end.close()
 146     failed_parent_end.close()
 147     return result, failed
 148
 149
 150 if __name__ == '__main__':
 151
 152     try:
 153         verbose = int(os.getenv("V", 0))
 154     except:
 155         verbose = 0
 156
 157     default_test_timeout = 600  # 10 minutes
 158     try:
 159         test_timeout = int(os.getenv("TIMEOUT", default_test_timeout))
 160     except:
 161         test_timeout = default_test_timeout
 162
 163     try:
 164         debug = os.getenv("DEBUG")
 165     except:
 166         debug = None
 167
 168     s = os.getenv("STEP", "n")
 169     step = True if s.lower() in ("y", "yes", "1") else False
 170
 171     parser = argparse.ArgumentParser(description="VPP unit tests")
 172     parser.add_argument("-f", "--failfast", action='count',
 173                         help="fast failure flag")
 174     parser.add_argument("-d", "--dir", action='append', type=str,
 175                         help="directory containing test files "
 176                              "(may be specified multiple times)")
 177     args = parser.parse_args()
 178     failfast = True if args.failfast == 1 else False
 179
 180     suite = unittest.TestSuite()
 181     cb = add_to_suite_callback(suite)
 182     for d in args.dir:
 183         print("Adding tests from directory tree %s" % d)
 184         discover_tests(d, cb)
 185
 186     try:
 187         retries = int(os.getenv("RETRIES"))
 188     except:
 189         retries = 0
 190     if retries is None:
 191         retries = 0
 192     attempts = retries + 1
 193     if attempts > 1:
 194         print("Perform %s attempts to pass the suite..." % attempts)
 195     if (debug is not None and debug.lower() in ["gdb", "gdbserver"]) or step:
 196         # don't fork if requiring interactive terminal..
 197         sys.exit(not VppTestRunner(
 198             verbosity=verbose, failfast=failfast).run(suite).wasSuccessful())
 199     else:
 200         while True:
 201             result, failed = run_forked(suite)
 202             attempts = attempts - 1
 203             print("%s test(s) failed, %s attempt(s) left" %
 204                   (len(failed), attempts))
 205             if len(failed) > 0 and attempts > 0:
 206                 suite = suite_from_failed(suite, failed)
 207                 continue
 208             sys.exit(result)