14 from multiprocessing import Process, Pipe, cpu_count
15 from multiprocessing.queues import Queue
16 from multiprocessing.managers import BaseManager
18 from framework import VppTestRunner, running_extended_tests, VppTestCase, \
19 get_testcase_doc_name, get_test_description, PASS, FAIL, ERROR, SKIP, \
21 from debug import spawn_gdb
22 from log import get_parallel_logger, double_line_delim, RED, YELLOW, GREEN, \
23 colorize, single_line_delim
24 from discover_tests import discover_tests
25 from subprocess import check_output, CalledProcessError
26 from util import check_core_path, get_core_path, is_core_present
28 # timeout which controls how long the child has to finish after seeing
29 # a core dump in test temporary directory. If this is exceeded, parent assumes
30 # that child process is stuck (e.g. waiting for shm mutex, which will never
31 # get unlocked) and kill the child
33 min_req_shm = 536870912 # min 512MB shm required
34 # 128MB per extra process
35 shm_per_process = 134217728
38 class StreamQueue(Queue):
43 sys.__stdout__.flush()
44 sys.__stderr__.flush()
47 return self._writer.fileno()
50 class StreamQueueManager(BaseManager):
54 StreamQueueManager.register('StreamQueue', StreamQueue)
57 class TestResult(dict):
58 def __init__(self, testcase_suite, testcases_by_id=None):
59 super(TestResult, self).__init__()
66 self.testcase_suite = testcase_suite
67 self.testcases = [testcase for testcase in testcase_suite]
68 self.testcases_by_id = testcases_by_id
70 def was_successful(self):
71 return 0 == len(self[FAIL]) == len(self[ERROR]) \
72 and len(self[PASS] + self[SKIP]) \
73 == self.testcase_suite.countTestCases() == len(self[TEST_RUN])
75 def no_tests_run(self):
76 return 0 == len(self[TEST_RUN])
78 def process_result(self, test_id, result):
79 self[result].append(test_id)
81 def suite_from_failed(self):
83 for testcase in self.testcase_suite:
85 if tc_id not in self[PASS] and tc_id not in self[SKIP]:
88 return suite_from_failed(self.testcase_suite, rerun_ids)
90 def get_testcase_names(self, test_id):
91 # could be tearDownClass (test_ipsec_esp.TestIpsecEsp1)
92 setup_teardown_match = re.match(
93 r'((tearDownClass)|(setUpClass)) \((.+\..+)\)', test_id)
94 if setup_teardown_match:
95 test_name, _, _, testcase_name = setup_teardown_match.groups()
96 if len(testcase_name.split('.')) == 2:
97 for key in self.testcases_by_id.keys():
98 if key.startswith(testcase_name):
101 testcase_name = self._get_testcase_doc_name(testcase_name)
103 test_name = self._get_test_description(test_id)
104 testcase_name = self._get_testcase_doc_name(test_id)
106 return testcase_name, test_name
108 def _get_test_description(self, test_id):
109 if test_id in self.testcases_by_id:
110 desc = get_test_description(descriptions,
111 self.testcases_by_id[test_id])
116 def _get_testcase_doc_name(self, test_id):
117 if test_id in self.testcases_by_id:
118 doc_name = get_testcase_doc_name(self.testcases_by_id[test_id])
124 def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue,
125 finished_pipe, result_pipe, logger):
126 sys.stdout = stdouterr_queue
127 sys.stderr = stdouterr_queue
128 VppTestCase.parallel_handler = logger.handlers[0]
129 result = VppTestRunner(keep_alive_pipe=keep_alive_pipe,
130 descriptions=descriptions,
132 result_pipe=result_pipe,
134 print_summary=False).run(suite)
135 finished_pipe.send(result.wasSuccessful())
136 finished_pipe.close()
137 keep_alive_pipe.close()
140 class TestCaseWrapper(object):
141 def __init__(self, testcase_suite, manager):
142 self.keep_alive_parent_end, self.keep_alive_child_end = Pipe(
144 self.finished_parent_end, self.finished_child_end = Pipe(duplex=False)
145 self.result_parent_end, self.result_child_end = Pipe(duplex=False)
146 self.testcase_suite = testcase_suite
147 if sys.version[0] == '2':
148 self.stdouterr_queue = manager.StreamQueue()
150 from multiprocessing import get_context
151 self.stdouterr_queue = manager.StreamQueue(ctx=get_context())
152 self.logger = get_parallel_logger(self.stdouterr_queue)
153 self.child = Process(target=test_runner_wrapper,
154 args=(testcase_suite,
155 self.keep_alive_child_end,
156 self.stdouterr_queue,
157 self.finished_child_end,
158 self.result_child_end,
162 self.last_test_temp_dir = None
163 self.last_test_vpp_binary = None
164 self._last_test = None
165 self.last_test_id = None
167 self.last_heard = time.time()
168 self.core_detected_at = None
169 self.testcases_by_id = {}
170 self.testclasess_with_core = {}
171 for testcase in self.testcase_suite:
172 self.testcases_by_id[testcase.id()] = testcase
173 self.result = TestResult(testcase_suite, self.testcases_by_id)
177 return self._last_test
180 def last_test(self, test_id):
181 self.last_test_id = test_id
182 if test_id in self.testcases_by_id:
183 testcase = self.testcases_by_id[test_id]
184 self._last_test = testcase.shortDescription()
185 if not self._last_test:
186 self._last_test = str(testcase)
188 self._last_test = test_id
190 def add_testclass_with_core(self):
191 if self.last_test_id in self.testcases_by_id:
192 test = self.testcases_by_id[self.last_test_id]
193 class_name = unittest.util.strclass(test.__class__)
194 test_name = "'{}' ({})".format(get_test_description(descriptions,
198 test_name = self.last_test_id
199 class_name = re.match(r'((tearDownClass)|(setUpClass)) '
200 r'\((.+\..+)\)', test_name).groups()[3]
201 if class_name not in self.testclasess_with_core:
202 self.testclasess_with_core[class_name] = (
204 self.last_test_vpp_binary,
205 self.last_test_temp_dir)
207 def close_pipes(self):
208 self.keep_alive_child_end.close()
209 self.finished_child_end.close()
210 self.result_child_end.close()
211 self.keep_alive_parent_end.close()
212 self.finished_parent_end.close()
213 self.result_parent_end.close()
215 def was_successful(self):
216 return self.result.was_successful()
219 def stdouterr_reader_wrapper(unread_testcases, finished_unread_testcases,
222 while read_testcases.is_set() or unread_testcases:
223 if finished_unread_testcases:
224 read_testcase = finished_unread_testcases.pop()
225 unread_testcases.remove(read_testcase)
226 elif unread_testcases:
227 read_testcase = unread_testcases.pop()
230 while data is not None:
231 sys.stdout.write(data)
232 data = read_testcase.stdouterr_queue.get()
234 read_testcase.stdouterr_queue.close()
235 finished_unread_testcases.discard(read_testcase)
239 def handle_failed_suite(logger, last_test_temp_dir, vpp_pid):
240 if last_test_temp_dir:
241 # Need to create link in case of a timeout or core dump without failure
242 lttd = os.path.basename(last_test_temp_dir)
243 failed_dir = os.getenv('FAILED_DIR')
244 link_path = '%s%s-FAILED' % (failed_dir, lttd)
245 if not os.path.exists(link_path):
246 os.symlink(last_test_temp_dir, link_path)
247 logger.error("Symlink to failed testcase directory: %s -> %s"
250 # Report core existence
251 core_path = get_core_path(last_test_temp_dir)
252 if os.path.exists(core_path):
254 "Core-file exists in test temporary directory: %s!" %
256 check_core_path(logger, core_path)
257 logger.debug("Running 'file %s':" % core_path)
259 info = check_output(["file", core_path])
261 except CalledProcessError as e:
262 logger.error("Subprocess returned with return code "
263 "while running `file' utility on core-file "
265 "rc=%s", e.returncode)
267 logger.error("Subprocess returned with OS error while "
268 "running 'file' utility "
270 "(%s) %s", e.errno, e.strerror)
271 except Exception as e:
272 logger.exception("Unexpected error running `file' utility "
276 # Copy api post mortem
277 api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
278 if os.path.isfile(api_post_mortem_path):
279 logger.error("Copying api_post_mortem.%d to %s" %
280 (vpp_pid, last_test_temp_dir))
281 shutil.copy2(api_post_mortem_path, last_test_temp_dir)
284 def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
285 if is_core_present(tempdir):
287 print('VPP core detected in %s. Last test running was %s' %
288 (tempdir, core_crash_test))
289 print(single_line_delim)
290 spawn_gdb(vpp_binary, get_core_path(tempdir))
291 print(single_line_delim)
293 print("Compressing core-file in test directory `%s'" % tempdir)
294 os.system("gzip %s" % get_core_path(tempdir))
297 def handle_cores(failed_testcases):
298 for failed_testcase in failed_testcases:
299 tcs_with_core = failed_testcase.testclasess_with_core
301 for test, vpp_binary, tempdir in tcs_with_core.values():
302 check_and_handle_core(vpp_binary, tempdir, test)
305 def process_finished_testsuite(wrapped_testcase_suite,
306 finished_testcase_suites,
307 failed_wrapped_testcases,
309 results.append(wrapped_testcase_suite.result)
310 finished_testcase_suites.add(wrapped_testcase_suite)
312 if failfast and not wrapped_testcase_suite.was_successful():
315 if not wrapped_testcase_suite.was_successful():
316 failed_wrapped_testcases.add(wrapped_testcase_suite)
317 handle_failed_suite(wrapped_testcase_suite.logger,
318 wrapped_testcase_suite.last_test_temp_dir,
319 wrapped_testcase_suite.vpp_pid)
324 def run_forked(testcase_suites):
325 wrapped_testcase_suites = set()
327 # suites are unhashable, need to use list
329 unread_testcases = set()
330 finished_unread_testcases = set()
331 manager = StreamQueueManager()
333 for i in range(concurrent_tests):
335 wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0),
337 wrapped_testcase_suites.add(wrapped_testcase_suite)
338 unread_testcases.add(wrapped_testcase_suite)
342 read_from_testcases = threading.Event()
343 read_from_testcases.set()
344 stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper,
345 args=(unread_testcases,
346 finished_unread_testcases,
347 read_from_testcases))
348 stdouterr_thread.start()
350 failed_wrapped_testcases = set()
354 while wrapped_testcase_suites:
355 finished_testcase_suites = set()
356 for wrapped_testcase_suite in wrapped_testcase_suites:
357 while wrapped_testcase_suite.result_parent_end.poll():
358 wrapped_testcase_suite.result.process_result(
359 *wrapped_testcase_suite.result_parent_end.recv())
360 wrapped_testcase_suite.last_heard = time.time()
362 while wrapped_testcase_suite.keep_alive_parent_end.poll():
363 wrapped_testcase_suite.last_test, \
364 wrapped_testcase_suite.last_test_vpp_binary, \
365 wrapped_testcase_suite.last_test_temp_dir, \
366 wrapped_testcase_suite.vpp_pid = \
367 wrapped_testcase_suite.keep_alive_parent_end.recv()
368 wrapped_testcase_suite.last_heard = time.time()
370 if wrapped_testcase_suite.finished_parent_end.poll():
371 wrapped_testcase_suite.finished_parent_end.recv()
372 wrapped_testcase_suite.last_heard = time.time()
373 stop_run = process_finished_testsuite(
374 wrapped_testcase_suite,
375 finished_testcase_suites,
376 failed_wrapped_testcases,
381 if wrapped_testcase_suite.last_heard + test_timeout < \
384 wrapped_testcase_suite.logger.critical(
385 "Child test runner process timed out "
386 "(last test running was `%s' in `%s')!" %
387 (wrapped_testcase_suite.last_test,
388 wrapped_testcase_suite.last_test_temp_dir))
389 elif not wrapped_testcase_suite.child.is_alive():
391 wrapped_testcase_suite.logger.critical(
392 "Child test runner process unexpectedly died "
393 "(last test running was `%s' in `%s')!" %
394 (wrapped_testcase_suite.last_test,
395 wrapped_testcase_suite.last_test_temp_dir))
396 elif wrapped_testcase_suite.last_test_temp_dir and \
397 wrapped_testcase_suite.last_test_vpp_binary:
399 wrapped_testcase_suite.last_test_temp_dir):
400 wrapped_testcase_suite.add_testclass_with_core()
401 if wrapped_testcase_suite.core_detected_at is None:
402 wrapped_testcase_suite.core_detected_at = \
404 elif wrapped_testcase_suite.core_detected_at + \
405 core_timeout < time.time():
406 wrapped_testcase_suite.logger.critical(
407 "Child test runner process unresponsive and "
408 "core-file exists in test temporary directory "
409 "(last test running was `%s' in `%s')!" %
410 (wrapped_testcase_suite.last_test,
411 wrapped_testcase_suite.last_test_temp_dir))
415 wrapped_testcase_suite.child.terminate()
417 # terminating the child process tends to leave orphan
419 if wrapped_testcase_suite.vpp_pid:
420 os.kill(wrapped_testcase_suite.vpp_pid,
425 wrapped_testcase_suite.result.crashed = True
426 wrapped_testcase_suite.result.process_result(
427 wrapped_testcase_suite.last_test_id, ERROR)
428 stop_run = process_finished_testsuite(
429 wrapped_testcase_suite,
430 finished_testcase_suites,
431 failed_wrapped_testcases,
434 for finished_testcase in finished_testcase_suites:
435 finished_testcase.child.join()
436 finished_testcase.close_pipes()
437 wrapped_testcase_suites.remove(finished_testcase)
438 finished_unread_testcases.add(finished_testcase)
439 finished_testcase.stdouterr_queue.put(None)
441 while testcase_suites:
442 results.append(TestResult(testcase_suites.pop(0)))
443 elif testcase_suites:
444 new_testcase = TestCaseWrapper(testcase_suites.pop(0),
446 wrapped_testcase_suites.add(new_testcase)
447 unread_testcases.add(new_testcase)
450 for wrapped_testcase_suite in wrapped_testcase_suites:
451 wrapped_testcase_suite.child.terminate()
452 wrapped_testcase_suite.stdouterr_queue.put(None)
455 read_from_testcases.clear()
456 stdouterr_thread.join(test_timeout)
459 handle_cores(failed_wrapped_testcases)
463 class SplitToSuitesCallback:
464 def __init__(self, filter_callback):
466 self.suite_name = 'default'
467 self.filter_callback = filter_callback
468 self.filtered = unittest.TestSuite()
470 def __call__(self, file_name, cls, method):
471 test_method = cls(method)
472 if self.filter_callback(file_name, cls.__name__, method):
473 self.suite_name = file_name + cls.__name__
474 if self.suite_name not in self.suites:
475 self.suites[self.suite_name] = unittest.TestSuite()
476 self.suites[self.suite_name].addTest(test_method)
479 self.filtered.addTest(test_method)
485 def parse_test_option():
486 f = os.getenv(test_option, None)
487 filter_file_name = None
488 filter_class_name = None
489 filter_func_name = None
494 raise Exception("Unrecognized %s option: %s" %
497 if parts[2] not in ('*', ''):
498 filter_func_name = parts[2]
499 if parts[1] not in ('*', ''):
500 filter_class_name = parts[1]
501 if parts[0] not in ('*', ''):
502 if parts[0].startswith('test_'):
503 filter_file_name = parts[0]
505 filter_file_name = 'test_%s' % parts[0]
507 if f.startswith('test_'):
510 filter_file_name = 'test_%s' % f
512 filter_file_name = '%s.py' % filter_file_name
513 return filter_file_name, filter_class_name, filter_func_name
516 def filter_tests(tests, filter_cb):
517 result = unittest.suite.TestSuite()
519 if isinstance(t, unittest.suite.TestSuite):
520 # this is a bunch of tests, recursively filter...
521 x = filter_tests(t, filter_cb)
522 if x.countTestCases() > 0:
524 elif isinstance(t, unittest.TestCase):
525 # this is a single test
526 parts = t.id().split('.')
527 # t.id() for common cases like this:
528 # test_classifier.TestClassifier.test_acl_ip
529 # apply filtering only if it is so
531 if not filter_cb(parts[0], parts[1], parts[2]):
535 # unexpected object, don't touch it
540 class FilterByTestOption:
541 def __init__(self, filter_file_name, filter_class_name, filter_func_name):
542 self.filter_file_name = filter_file_name
543 self.filter_class_name = filter_class_name
544 self.filter_func_name = filter_func_name
546 def __call__(self, file_name, class_name, func_name):
547 if self.filter_file_name:
548 fn_match = fnmatch.fnmatch(file_name, self.filter_file_name)
551 if self.filter_class_name and class_name != self.filter_class_name:
553 if self.filter_func_name and func_name != self.filter_func_name:
558 class FilterByClassList:
559 def __init__(self, classes_with_filenames):
560 self.classes_with_filenames = classes_with_filenames
562 def __call__(self, file_name, class_name, func_name):
563 return '.'.join([file_name, class_name]) in self.classes_with_filenames
566 def suite_from_failed(suite, failed):
567 failed = {x.rsplit('.', 1)[0] for x in failed}
568 filter_cb = FilterByClassList(failed)
569 suite = filter_tests(suite, filter_cb)
573 class AllResults(dict):
575 super(AllResults, self).__init__()
576 self.all_testcases = 0
577 self.results_per_suite = []
584 self.testsuites_no_tests_run = []
586 def add_results(self, result):
587 self.results_per_suite.append(result)
588 result_types = [PASS, FAIL, ERROR, SKIP, TEST_RUN]
589 for result_type in result_types:
590 self[result_type] += len(result[result_type])
592 def add_result(self, result):
594 self.all_testcases += result.testcase_suite.countTestCases()
595 self.add_results(result)
597 if result.no_tests_run():
598 self.testsuites_no_tests_run.append(result.testcase_suite)
603 elif not result.was_successful():
607 self.rerun.append(result.testcase_suite)
611 def print_results(self):
613 print(double_line_delim)
614 print('TEST RESULTS:')
615 print(' Scheduled tests: {}'.format(self.all_testcases))
616 print(' Executed tests: {}'.format(self[TEST_RUN]))
617 print(' Passed tests: {}'.format(
618 colorize(str(self[PASS]), GREEN)))
620 print(' Skipped tests: {}'.format(
621 colorize(str(self[SKIP]), YELLOW)))
622 if self.not_executed > 0:
623 print(' Not Executed tests: {}'.format(
624 colorize(str(self.not_executed), RED)))
626 print(' Failures: {}'.format(
627 colorize(str(self[FAIL]), RED)))
629 print(' Errors: {}'.format(
630 colorize(str(self[ERROR]), RED)))
632 if self.all_failed > 0:
633 print('FAILURES AND ERRORS IN TESTS:')
634 for result in self.results_per_suite:
635 failed_testcase_ids = result[FAIL]
636 errored_testcase_ids = result[ERROR]
637 old_testcase_name = None
638 if failed_testcase_ids or errored_testcase_ids:
639 for failed_test_id in failed_testcase_ids:
640 new_testcase_name, test_name = \
641 result.get_testcase_names(failed_test_id)
642 if new_testcase_name != old_testcase_name:
643 print(' Testcase name: {}'.format(
644 colorize(new_testcase_name, RED)))
645 old_testcase_name = new_testcase_name
646 print(' FAILURE: {} [{}]'.format(
647 colorize(test_name, RED), failed_test_id))
648 for failed_test_id in errored_testcase_ids:
649 new_testcase_name, test_name = \
650 result.get_testcase_names(failed_test_id)
651 if new_testcase_name != old_testcase_name:
652 print(' Testcase name: {}'.format(
653 colorize(new_testcase_name, RED)))
654 old_testcase_name = new_testcase_name
655 print(' ERROR: {} [{}]'.format(
656 colorize(test_name, RED), failed_test_id))
657 if self.testsuites_no_tests_run:
658 print('TESTCASES WHERE NO TESTS WERE SUCCESSFULLY EXECUTED:')
660 for testsuite in self.testsuites_no_tests_run:
661 for testcase in testsuite:
662 tc_classes.add(get_testcase_doc_name(testcase))
663 for tc_class in tc_classes:
664 print(' {}'.format(colorize(tc_class, RED)))
666 print(double_line_delim)
670 def not_executed(self):
671 return self.all_testcases - self[TEST_RUN]
674 def all_failed(self):
675 return self[FAIL] + self[ERROR]
678 def parse_results(results):
680 Prints the number of scheduled, executed, not executed, passed, failed,
681 errored and skipped tests and details about failed and errored tests.
683 Also returns all suites where any test failed.
689 results_per_suite = AllResults()
692 for result in results:
693 result_code = results_per_suite.add_result(result)
696 elif result_code == -1:
699 results_per_suite.print_results()
707 return return_code, results_per_suite.rerun
710 def parse_digit_env(env_var, default):
711 value = os.getenv(env_var, default)
716 print('WARNING: unsupported value "%s" for env var "%s",'
717 'defaulting to %s' % (value, env_var, default))
722 if __name__ == '__main__':
724 verbose = parse_digit_env("V", 0)
726 test_timeout = parse_digit_env("TIMEOUT", 600) # default = 10 minutes
728 retries = parse_digit_env("RETRIES", 0)
730 debug = os.getenv("DEBUG", "n").lower() in ["gdb", "gdbserver"]
732 debug_core = os.getenv("DEBUG", "").lower() == "core"
733 compress_core = framework.BoolEnvironmentVariable("CORE_COMPRESS")
735 step = framework.BoolEnvironmentVariable("STEP")
736 force_foreground = framework.BoolEnvironmentVariable("FORCE_FOREGROUND")
738 run_interactive = debug or step or force_foreground
740 test_jobs = os.getenv("TEST_JOBS", "1").lower() # default = 1 process
741 if test_jobs == 'auto':
744 print('Interactive mode required, running on one core')
746 shm_free = psutil.disk_usage('/dev/shm').free
747 shm_max_processes = 1
748 if shm_free < min_req_shm:
749 raise Exception('Not enough free space in /dev/shm. Required '
750 'free space is at least %sM.'
751 % (min_req_shm >> 20))
753 extra_shm = shm_free - min_req_shm
754 shm_max_processes += extra_shm / shm_per_process
755 concurrent_tests = min(cpu_count(), shm_max_processes)
756 print('Found enough resources to run tests with %s cores'
758 elif test_jobs.isdigit():
759 concurrent_tests = int(test_jobs)
763 if run_interactive and concurrent_tests > 1:
764 raise NotImplementedError(
765 'Running tests interactively (DEBUG is gdb or gdbserver or STEP '
766 'is set) in parallel (TEST_JOBS is more than 1) is not supported')
768 parser = argparse.ArgumentParser(description="VPP unit tests")
769 parser.add_argument("-f", "--failfast", action='store_true',
770 help="fast failure flag")
771 parser.add_argument("-d", "--dir", action='append', type=str,
772 help="directory containing test files "
773 "(may be specified multiple times)")
774 args = parser.parse_args()
775 failfast = args.failfast
778 print("Running tests using custom test runner") # debug message
779 filter_file, filter_class, filter_func = parse_test_option()
781 print("Active filters: file=%s, class=%s, function=%s" % (
782 filter_file, filter_class, filter_func))
784 filter_cb = FilterByTestOption(filter_file, filter_class, filter_func)
786 ignore_path = os.getenv("VENV_PATH", None)
787 cb = SplitToSuitesCallback(filter_cb)
789 print("Adding tests from directory tree %s" % d)
790 discover_tests(d, cb, ignore_path)
792 # suites are not hashable, need to use list
795 for testcase_suite in cb.suites.values():
796 tests_amount += testcase_suite.countTestCases()
797 suites.append(testcase_suite)
799 print("%s out of %s tests match specified filters" % (
800 tests_amount, tests_amount + cb.filtered.countTestCases()))
802 if not running_extended_tests:
803 print("Not running extended tests (some tests will be skipped)")
805 attempts = retries + 1
807 print("Perform %s attempts to pass the suite..." % attempts)
809 if run_interactive and suites:
810 # don't fork if requiring interactive terminal
811 print('Running tests in foreground in the current process')
812 full_suite = unittest.TestSuite()
813 map(full_suite.addTests, suites)
814 result = VppTestRunner(verbosity=verbose,
816 print_summary=True).run(full_suite)
817 was_successful = result.wasSuccessful()
818 if not was_successful:
819 for test_case_info in result.failed_test_cases_info:
820 handle_failed_suite(test_case_info.logger,
821 test_case_info.tempdir,
822 test_case_info.vpp_pid)
823 if test_case_info in result.core_crash_test_cases_info:
824 check_and_handle_core(test_case_info.vpp_bin_path,
825 test_case_info.tempdir,
826 test_case_info.core_crash_test)
828 sys.exit(not was_successful)
830 print('Running each VPPTestCase in a separate background process'
831 ' with {} parallel process(es)'.format(concurrent_tests))
833 while suites and attempts > 0:
834 results = run_forked(suites)
835 exit_code, suites = parse_results(results)
838 print('Test run was successful')
840 print('%s attempt(s) left.' % attempts)