10 from multiprocessing import Process, Pipe
11 from framework import VppTestRunner
12 from debug import spawn_gdb
13 from log import global_logger
14 from discover_tests import discover_tests
15 from subprocess import check_output, CalledProcessError
17 # timeout which controls how long the child has to finish after seeing
18 # a core dump in test temporary directory. If this is exceeded, parent assumes
19 # that child process is stuck (e.g. waiting for shm mutex, which will never
20 # get unlocked) and kill the child
24 def test_runner_wrapper(suite, keep_alive_pipe, result_pipe, failed_pipe):
25 result = not VppTestRunner(
26 keep_alive_pipe=keep_alive_pipe,
27 failed_pipe=failed_pipe,
29 failfast=failfast).run(suite).wasSuccessful()
30 result_pipe.send(result)
32 keep_alive_pipe.close()
36 class add_to_suite_callback:
37 def __init__(self, suite):
40 def __call__(self, file_name, cls, method):
41 suite.addTest(cls(method))
44 class Filter_by_class_list:
45 def __init__(self, class_list):
46 self.class_list = class_list
48 def __call__(self, file_name, class_name, func_name):
49 return class_name in self.class_list
52 def suite_from_failed(suite, failed):
53 filter_cb = Filter_by_class_list(failed)
54 return VppTestRunner.filter_tests(suite, filter_cb)
57 def run_forked(suite):
58 keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
59 result_parent_end, result_child_end = Pipe(duplex=False)
60 failed_parent_end, failed_child_end = Pipe(duplex=False)
62 child = Process(target=test_runner_wrapper,
63 args=(suite, keep_alive_child_end, result_child_end,
66 last_test_temp_dir = None
67 last_test_vpp_binary = None
71 last_heard = time.time()
72 core_detected_at = None
73 debug_core = os.getenv("DEBUG", "").lower() == "core"
75 readable = select.select([keep_alive_parent_end.fileno(),
76 result_parent_end.fileno(),
77 failed_parent_end.fileno(),
80 if result_parent_end.fileno() in readable:
81 result = result_parent_end.recv()
83 if keep_alive_parent_end.fileno() in readable:
84 while keep_alive_parent_end.poll():
85 last_test, last_test_vpp_binary,\
86 last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
87 last_heard = time.time()
88 if failed_parent_end.fileno() in readable:
89 while failed_parent_end.poll():
90 failed_test = failed_parent_end.recv()
91 failed.add(failed_test.__name__)
92 last_heard = time.time()
94 if last_heard + test_timeout < time.time() and \
95 not os.path.isfile("%s/_core_handled" % last_test_temp_dir):
97 global_logger.critical("Timeout while waiting for child test "
98 "runner process (last test running was "
100 (last_test, last_test_temp_dir))
101 elif not child.is_alive():
103 global_logger.critical("Child python process unexpectedly died "
104 "(last test running was `%s' in `%s')!" %
105 (last_test, last_test_temp_dir))
106 elif last_test_temp_dir and last_test_vpp_binary:
107 core_path = "%s/core" % last_test_temp_dir
108 if os.path.isfile(core_path):
109 if core_detected_at is None:
110 core_detected_at = time.time()
111 elif core_detected_at + core_timeout < time.time():
112 if not os.path.isfile(
113 "%s/_core_handled" % last_test_temp_dir):
114 global_logger.critical(
115 "Child python process unresponsive and core-file "
116 "exists in test temporary directory!")
120 failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
121 lttd = last_test_temp_dir.split("/")[-1]
122 link_path = '%s%s-FAILED' % (failed_dir, lttd)
123 global_logger.error("Creating a link to the failed " +
124 "test: %s -> %s" % (link_path, lttd))
126 os.symlink(last_test_temp_dir, link_path)
129 api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
130 if os.path.isfile(api_post_mortem_path):
131 global_logger.error("Copying api_post_mortem.%d to %s" %
132 (vpp_pid, last_test_temp_dir))
133 shutil.copy2(api_post_mortem_path, last_test_temp_dir)
134 if last_test_temp_dir and last_test_vpp_binary:
135 core_path = "%s/core" % last_test_temp_dir
136 if os.path.isfile(core_path):
137 global_logger.error("Core-file exists in test temporary "
138 "directory: %s!" % core_path)
139 global_logger.debug("Running `file %s':" % core_path)
141 info = check_output(["file", core_path])
142 global_logger.debug(info)
143 except CalledProcessError as e:
145 "Could not run `file' utility on core-file, "
146 "rc=%s" % e.returncode)
149 spawn_gdb(last_test_vpp_binary, core_path,
154 keep_alive_parent_end.close()
155 result_parent_end.close()
156 failed_parent_end.close()
157 return result, failed
160 if __name__ == '__main__':
163 verbose = int(os.getenv("V", 0))
167 default_test_timeout = 600 # 10 minutes
169 test_timeout = int(os.getenv("TIMEOUT", default_test_timeout))
171 test_timeout = default_test_timeout
173 debug = os.getenv("DEBUG")
175 s = os.getenv("STEP", "n")
176 step = True if s.lower() in ("y", "yes", "1") else False
178 parser = argparse.ArgumentParser(description="VPP unit tests")
179 parser.add_argument("-f", "--failfast", action='count',
180 help="fast failure flag")
181 parser.add_argument("-d", "--dir", action='append', type=str,
182 help="directory containing test files "
183 "(may be specified multiple times)")
184 args = parser.parse_args()
185 failfast = True if args.failfast == 1 else False
187 suite = unittest.TestSuite()
188 cb = add_to_suite_callback(suite)
190 print("Adding tests from directory tree %s" % d)
191 discover_tests(d, cb)
194 retries = int(os.getenv("RETRIES", 0))
197 attempts = retries + 1
199 print("Perform %s attempts to pass the suite..." % attempts)
200 if (debug is not None and debug.lower() in ["gdb", "gdbserver"]) or step:
201 # don't fork if requiring interactive terminal..
202 sys.exit(not VppTestRunner(
203 verbosity=verbose, failfast=failfast).run(suite).wasSuccessful())
206 result, failed = run_forked(suite)
207 attempts = attempts - 1
208 print("%s test(s) failed, %s attempt(s) left" %
209 (len(failed), attempts))
210 if len(failed) > 0 and attempts > 0:
211 suite = suite_from_failed(suite, failed)