10 from multiprocessing import Process, Pipe
11 from framework import VppTestRunner
12 from debug import spawn_gdb
13 from log import global_logger
14 from discover_tests import discover_tests
15 from subprocess import check_output, CalledProcessError
16 from util import check_core_path
18 # timeout which controls how long the child has to finish after seeing
19 # a core dump in test temporary directory. If this is exceeded, parent assumes
20 # that child process is stuck (e.g. waiting for shm mutex, which will never
21 # get unlocked) and kill the child
25 def test_runner_wrapper(suite, keep_alive_pipe, result_pipe, failed_pipe):
26 result = not VppTestRunner(
27 keep_alive_pipe=keep_alive_pipe,
28 failed_pipe=failed_pipe,
30 failfast=failfast).run(suite).wasSuccessful()
31 result_pipe.send(result)
33 keep_alive_pipe.close()
37 class add_to_suite_callback:
38 def __init__(self, suite):
41 def __call__(self, file_name, cls, method):
42 suite.addTest(cls(method))
45 class Filter_by_class_list:
46 def __init__(self, class_list):
47 self.class_list = class_list
49 def __call__(self, file_name, class_name, func_name):
50 return class_name in self.class_list
53 def suite_from_failed(suite, failed):
54 filter_cb = Filter_by_class_list(failed)
55 return VppTestRunner.filter_tests(suite, filter_cb)
58 def run_forked(suite):
59 keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
60 result_parent_end, result_child_end = Pipe(duplex=False)
61 failed_parent_end, failed_child_end = Pipe(duplex=False)
63 child = Process(target=test_runner_wrapper,
64 args=(suite, keep_alive_child_end, result_child_end,
67 last_test_temp_dir = None
68 last_test_vpp_binary = None
72 last_heard = time.time()
73 core_detected_at = None
74 debug_core = os.getenv("DEBUG", "").lower() == "core"
76 readable = select.select([keep_alive_parent_end.fileno(),
77 result_parent_end.fileno(),
78 failed_parent_end.fileno(),
81 if result_parent_end.fileno() in readable:
82 result = result_parent_end.recv()
84 if keep_alive_parent_end.fileno() in readable:
85 while keep_alive_parent_end.poll():
86 last_test, last_test_vpp_binary,\
87 last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
88 last_heard = time.time()
89 if failed_parent_end.fileno() in readable:
90 while failed_parent_end.poll():
91 failed_test = failed_parent_end.recv()
92 failed.add(failed_test.__name__)
93 last_heard = time.time()
95 if last_heard + test_timeout < time.time() and \
96 not os.path.isfile("%s/_core_handled" % last_test_temp_dir):
98 global_logger.critical("Timeout while waiting for child test "
99 "runner process (last test running was "
101 (last_test, last_test_temp_dir))
102 elif not child.is_alive():
104 global_logger.critical("Child python process unexpectedly died "
105 "(last test running was `%s' in `%s')!" %
106 (last_test, last_test_temp_dir))
107 elif last_test_temp_dir and last_test_vpp_binary:
108 core_path = "%s/core" % last_test_temp_dir
109 if os.path.isfile(core_path):
110 if core_detected_at is None:
111 core_detected_at = time.time()
112 elif core_detected_at + core_timeout < time.time():
113 if not os.path.isfile(
114 "%s/_core_handled" % last_test_temp_dir):
115 global_logger.critical(
116 "Child python process unresponsive and core-file "
117 "exists in test temporary directory!")
121 failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
122 lttd = last_test_temp_dir.split("/")[-1]
123 link_path = '%s%s-FAILED' % (failed_dir, lttd)
124 global_logger.error("Creating a link to the failed " +
125 "test: %s -> %s" % (link_path, lttd))
127 os.symlink(last_test_temp_dir, link_path)
130 api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
131 if os.path.isfile(api_post_mortem_path):
132 global_logger.error("Copying api_post_mortem.%d to %s" %
133 (vpp_pid, last_test_temp_dir))
134 shutil.copy2(api_post_mortem_path, last_test_temp_dir)
135 if last_test_temp_dir and last_test_vpp_binary:
136 core_path = "%s/core" % last_test_temp_dir
137 if os.path.isfile(core_path):
138 global_logger.error("Core-file exists in test temporary "
139 "directory: %s!" % core_path)
140 check_core_path(global_logger, core_path)
141 global_logger.debug("Running `file %s':" % core_path)
143 info = check_output(["file", core_path])
144 global_logger.debug(info)
145 except CalledProcessError as e:
147 "Could not run `file' utility on core-file, "
148 "rc=%s" % e.returncode)
151 spawn_gdb(last_test_vpp_binary, core_path,
156 keep_alive_parent_end.close()
157 result_parent_end.close()
158 failed_parent_end.close()
159 return result, failed
162 if __name__ == '__main__':
165 verbose = int(os.getenv("V", 0))
169 default_test_timeout = 600 # 10 minutes
171 test_timeout = int(os.getenv("TIMEOUT", default_test_timeout))
173 test_timeout = default_test_timeout
175 debug = os.getenv("DEBUG")
177 s = os.getenv("STEP", "n")
178 step = True if s.lower() in ("y", "yes", "1") else False
180 parser = argparse.ArgumentParser(description="VPP unit tests")
181 parser.add_argument("-f", "--failfast", action='count',
182 help="fast failure flag")
183 parser.add_argument("-d", "--dir", action='append', type=str,
184 help="directory containing test files "
185 "(may be specified multiple times)")
186 args = parser.parse_args()
187 failfast = True if args.failfast == 1 else False
189 suite = unittest.TestSuite()
190 cb = add_to_suite_callback(suite)
192 print("Adding tests from directory tree %s" % d)
193 discover_tests(d, cb)
196 retries = int(os.getenv("RETRIES", 0))
199 attempts = retries + 1
201 print("Perform %s attempts to pass the suite..." % attempts)
202 if (debug is not None and debug.lower() in ["gdb", "gdbserver"]) or step:
203 # don't fork if requiring interactive terminal..
204 sys.exit(not VppTestRunner(
205 verbosity=verbose, failfast=failfast).run(suite).wasSuccessful())
208 result, failed = run_forked(suite)
209 attempts = attempts - 1
210 print("%s test(s) failed, %s attempt(s) left" %
211 (len(failed), attempts))
212 if len(failed) > 0 and attempts > 0:
213 suite = suite_from_failed(suite, failed)