10 from multiprocessing import Process, Pipe
11 from framework import VppTestRunner
12 from debug import spawn_gdb
13 from log import global_logger
14 from discover_tests import discover_tests
15 from subprocess import check_output, CalledProcessError
16 from util import check_core_path
18 # timeout which controls how long the child has to finish after seeing
19 # a core dump in test temporary directory. If this is exceeded, parent assumes
20 # that child process is stuck (e.g. waiting for shm mutex, which will never
21 # get unlocked) and kill the child
25 def test_runner_wrapper(suite, keep_alive_pipe, result_pipe, failed_pipe):
26 result = not VppTestRunner(
27 keep_alive_pipe=keep_alive_pipe,
28 failed_pipe=failed_pipe,
30 failfast=failfast).run(suite).wasSuccessful()
31 result_pipe.send(result)
33 keep_alive_pipe.close()
37 class add_to_suite_callback:
38 def __init__(self, suite):
41 def __call__(self, file_name, cls, method):
42 suite.addTest(cls(method))
45 class Filter_by_class_list:
46 def __init__(self, class_list):
47 self.class_list = class_list
49 def __call__(self, file_name, class_name, func_name):
50 return class_name in self.class_list
53 def suite_from_failed(suite, failed):
54 filter_cb = Filter_by_class_list(failed)
55 suite = VppTestRunner.filter_tests(suite, filter_cb)
56 if 0 == suite.countTestCases():
57 raise Exception("Suite is empty after filtering out the failed tests!")
61 def run_forked(suite):
62 keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
63 result_parent_end, result_child_end = Pipe(duplex=False)
64 failed_parent_end, failed_child_end = Pipe(duplex=False)
66 child = Process(target=test_runner_wrapper,
67 args=(suite, keep_alive_child_end, result_child_end,
70 last_test_temp_dir = None
71 last_test_vpp_binary = None
75 last_heard = time.time()
76 core_detected_at = None
77 debug_core = os.getenv("DEBUG", "").lower() == "core"
79 readable = select.select([keep_alive_parent_end.fileno(),
80 result_parent_end.fileno(),
81 failed_parent_end.fileno(),
84 if result_parent_end.fileno() in readable:
85 result = result_parent_end.recv()
87 if keep_alive_parent_end.fileno() in readable:
88 while keep_alive_parent_end.poll():
89 last_test, last_test_vpp_binary,\
90 last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
91 last_heard = time.time()
92 if failed_parent_end.fileno() in readable:
93 while failed_parent_end.poll():
94 failed_test = failed_parent_end.recv()
95 failed.add(failed_test.__name__)
96 last_heard = time.time()
98 if last_heard + test_timeout < time.time() and \
99 not os.path.isfile("%s/_core_handled" % last_test_temp_dir):
101 global_logger.critical("Timeout while waiting for child test "
102 "runner process (last test running was "
104 (last_test, last_test_temp_dir))
105 elif not child.is_alive():
107 global_logger.critical("Child python process unexpectedly died "
108 "(last test running was `%s' in `%s')!" %
109 (last_test, last_test_temp_dir))
110 elif last_test_temp_dir and last_test_vpp_binary:
111 core_path = "%s/core" % last_test_temp_dir
112 if os.path.isfile(core_path):
113 if core_detected_at is None:
114 core_detected_at = time.time()
115 elif core_detected_at + core_timeout < time.time():
116 if not os.path.isfile(
117 "%s/_core_handled" % last_test_temp_dir):
118 global_logger.critical(
119 "Child python process unresponsive and core-file "
120 "exists in test temporary directory!")
124 failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
125 lttd = last_test_temp_dir.split("/")[-1]
126 link_path = '%s%s-FAILED' % (failed_dir, lttd)
127 global_logger.error("Creating a link to the failed " +
128 "test: %s -> %s" % (link_path, lttd))
130 os.symlink(last_test_temp_dir, link_path)
133 api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
134 if os.path.isfile(api_post_mortem_path):
135 global_logger.error("Copying api_post_mortem.%d to %s" %
136 (vpp_pid, last_test_temp_dir))
137 shutil.copy2(api_post_mortem_path, last_test_temp_dir)
138 if last_test_temp_dir and last_test_vpp_binary:
139 core_path = "%s/core" % last_test_temp_dir
140 if os.path.isfile(core_path):
141 global_logger.error("Core-file exists in test temporary "
142 "directory: %s!" % core_path)
143 check_core_path(global_logger, core_path)
144 global_logger.debug("Running `file %s':" % core_path)
146 info = check_output(["file", core_path])
147 global_logger.debug(info)
148 except CalledProcessError as e:
150 "Could not run `file' utility on core-file, "
151 "rc=%s" % e.returncode)
154 spawn_gdb(last_test_vpp_binary, core_path,
159 keep_alive_parent_end.close()
160 result_parent_end.close()
161 failed_parent_end.close()
162 return result, failed
165 if __name__ == '__main__':
168 verbose = int(os.getenv("V", 0))
172 default_test_timeout = 600 # 10 minutes
174 test_timeout = int(os.getenv("TIMEOUT", default_test_timeout))
176 test_timeout = default_test_timeout
178 debug = os.getenv("DEBUG")
180 s = os.getenv("STEP", "n")
181 step = True if s.lower() in ("y", "yes", "1") else False
183 parser = argparse.ArgumentParser(description="VPP unit tests")
184 parser.add_argument("-f", "--failfast", action='count',
185 help="fast failure flag")
186 parser.add_argument("-d", "--dir", action='append', type=str,
187 help="directory containing test files "
188 "(may be specified multiple times)")
189 args = parser.parse_args()
190 failfast = True if args.failfast == 1 else False
192 suite = unittest.TestSuite()
193 cb = add_to_suite_callback(suite)
195 print("Adding tests from directory tree %s" % d)
196 discover_tests(d, cb)
199 retries = int(os.getenv("RETRIES", 0))
204 force_foreground = int(os.getenv("FORCE_FOREGROUND", 0))
207 attempts = retries + 1
209 print("Perform %s attempts to pass the suite..." % attempts)
210 if (debug is not None and debug.lower() in ["gdb", "gdbserver"]) or step\
212 # don't fork if requiring interactive terminal..
213 sys.exit(not VppTestRunner(
214 verbosity=verbose, failfast=failfast).run(suite).wasSuccessful())
217 result, failed = run_forked(suite)
218 attempts = attempts - 1
219 print("%s test(s) failed, %s attempt(s) left" %
220 (len(failed), attempts))
221 if len(failed) > 0 and attempts > 0:
222 suite = suite_from_failed(suite, failed)