14 import multiprocessing
15 from multiprocessing import Process, Pipe, cpu_count
16 from multiprocessing.queues import Queue
17 from multiprocessing.managers import BaseManager
19 from framework import VppTestRunner, running_extended_tests, VppTestCase, \
20 get_testcase_doc_name, get_test_description, PASS, FAIL, ERROR, SKIP, \
22 from debug import spawn_gdb
23 from log import get_parallel_logger, double_line_delim, RED, YELLOW, GREEN, \
24 colorize, single_line_delim
25 from discover_tests import discover_tests
26 from subprocess import check_output, CalledProcessError
27 from util import check_core_path, get_core_path, is_core_present
29 # timeout which controls how long the child has to finish after seeing
30 # a core dump in test temporary directory. If this is exceeded, parent assumes
31 # that child process is stuck (e.g. waiting for shm mutex, which will never
32 # get unlocked) and kill the child
34 min_req_shm = 536870912 # min 512MB shm required
35 # 128MB per extra process
36 shm_per_process = 134217728
39 class StreamQueue(Queue):
44 sys.__stdout__.flush()
45 sys.__stderr__.flush()
48 return self._writer.fileno()
51 class StreamQueueManager(BaseManager):
55 StreamQueueManager.register('StreamQueue', StreamQueue)
58 class TestResult(dict):
59 def __init__(self, testcase_suite, testcases_by_id=None):
60 super(TestResult, self).__init__()
67 self.testcase_suite = testcase_suite
68 self.testcases = [testcase for testcase in testcase_suite]
69 self.testcases_by_id = testcases_by_id
71 def was_successful(self):
72 return 0 == len(self[FAIL]) == len(self[ERROR]) \
73 and len(self[PASS] + self[SKIP]) \
74 == self.testcase_suite.countTestCases() == len(self[TEST_RUN])
76 def no_tests_run(self):
77 return 0 == len(self[TEST_RUN])
79 def process_result(self, test_id, result):
80 self[result].append(test_id)
82 def suite_from_failed(self):
84 for testcase in self.testcase_suite:
86 if tc_id not in self[PASS] and tc_id not in self[SKIP]:
89 return suite_from_failed(self.testcase_suite, rerun_ids)
91 def get_testcase_names(self, test_id):
92 # could be tearDownClass (test_ipsec_esp.TestIpsecEsp1)
93 setup_teardown_match = re.match(
94 r'((tearDownClass)|(setUpClass)) \((.+\..+)\)', test_id)
95 if setup_teardown_match:
96 test_name, _, _, testcase_name = setup_teardown_match.groups()
97 if len(testcase_name.split('.')) == 2:
98 for key in self.testcases_by_id.keys():
99 if key.startswith(testcase_name):
102 testcase_name = self._get_testcase_doc_name(testcase_name)
104 test_name = self._get_test_description(test_id)
105 testcase_name = self._get_testcase_doc_name(test_id)
107 return testcase_name, test_name
109 def _get_test_description(self, test_id):
110 if test_id in self.testcases_by_id:
111 desc = get_test_description(descriptions,
112 self.testcases_by_id[test_id])
117 def _get_testcase_doc_name(self, test_id):
118 if test_id in self.testcases_by_id:
119 doc_name = get_testcase_doc_name(self.testcases_by_id[test_id])
125 def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue,
126 finished_pipe, result_pipe, logger):
127 sys.stdout = stdouterr_queue
128 sys.stderr = stdouterr_queue
129 VppTestCase.parallel_handler = logger.handlers[0]
130 result = VppTestRunner(keep_alive_pipe=keep_alive_pipe,
131 descriptions=descriptions,
133 result_pipe=result_pipe,
135 print_summary=False).run(suite)
136 finished_pipe.send(result.wasSuccessful())
137 finished_pipe.close()
138 keep_alive_pipe.close()
141 class TestCaseWrapper(object):
142 def __init__(self, testcase_suite, manager):
143 self.keep_alive_parent_end, self.keep_alive_child_end = Pipe(
145 self.finished_parent_end, self.finished_child_end = Pipe(duplex=False)
146 self.result_parent_end, self.result_child_end = Pipe(duplex=False)
147 self.testcase_suite = testcase_suite
148 if sys.version[0] == '2':
149 self.stdouterr_queue = manager.StreamQueue()
151 from multiprocessing import get_context
152 self.stdouterr_queue = manager.StreamQueue(ctx=get_context())
153 self.logger = get_parallel_logger(self.stdouterr_queue)
154 self.child = Process(target=test_runner_wrapper,
155 args=(testcase_suite,
156 self.keep_alive_child_end,
157 self.stdouterr_queue,
158 self.finished_child_end,
159 self.result_child_end,
163 self.last_test_temp_dir = None
164 self.last_test_vpp_binary = None
165 self._last_test = None
166 self.last_test_id = None
168 self.last_heard = time.time()
169 self.core_detected_at = None
170 self.testcases_by_id = {}
171 self.testclasess_with_core = {}
172 for testcase in self.testcase_suite:
173 self.testcases_by_id[testcase.id()] = testcase
174 self.result = TestResult(testcase_suite, self.testcases_by_id)
178 return self._last_test
181 def last_test(self, test_id):
182 self.last_test_id = test_id
183 if test_id in self.testcases_by_id:
184 testcase = self.testcases_by_id[test_id]
185 self._last_test = testcase.shortDescription()
186 if not self._last_test:
187 self._last_test = str(testcase)
189 self._last_test = test_id
191 def add_testclass_with_core(self):
192 if self.last_test_id in self.testcases_by_id:
193 test = self.testcases_by_id[self.last_test_id]
194 class_name = unittest.util.strclass(test.__class__)
195 test_name = "'{}' ({})".format(get_test_description(descriptions,
199 test_name = self.last_test_id
200 class_name = re.match(r'((tearDownClass)|(setUpClass)) '
201 r'\((.+\..+)\)', test_name).groups()[3]
202 if class_name not in self.testclasess_with_core:
203 self.testclasess_with_core[class_name] = (
205 self.last_test_vpp_binary,
206 self.last_test_temp_dir)
208 def close_pipes(self):
209 self.keep_alive_child_end.close()
210 self.finished_child_end.close()
211 self.result_child_end.close()
212 self.keep_alive_parent_end.close()
213 self.finished_parent_end.close()
214 self.result_parent_end.close()
216 def was_successful(self):
217 return self.result.was_successful()
220 def stdouterr_reader_wrapper(unread_testcases, finished_unread_testcases,
223 while read_testcases.is_set() or unread_testcases:
224 if finished_unread_testcases:
225 read_testcase = finished_unread_testcases.pop()
226 unread_testcases.remove(read_testcase)
227 elif unread_testcases:
228 read_testcase = unread_testcases.pop()
231 while data is not None:
232 sys.stdout.write(data)
233 data = read_testcase.stdouterr_queue.get()
235 read_testcase.stdouterr_queue.close()
236 finished_unread_testcases.discard(read_testcase)
240 def handle_failed_suite(logger, last_test_temp_dir, vpp_pid):
241 if last_test_temp_dir:
242 # Need to create link in case of a timeout or core dump without failure
243 lttd = os.path.basename(last_test_temp_dir)
244 failed_dir = os.getenv('FAILED_DIR')
245 link_path = '%s%s-FAILED' % (failed_dir, lttd)
246 if not os.path.exists(link_path):
247 os.symlink(last_test_temp_dir, link_path)
248 logger.error("Symlink to failed testcase directory: %s -> %s"
251 # Report core existence
252 core_path = get_core_path(last_test_temp_dir)
253 if os.path.exists(core_path):
255 "Core-file exists in test temporary directory: %s!" %
257 check_core_path(logger, core_path)
258 logger.debug("Running 'file %s':" % core_path)
260 info = check_output(["file", core_path])
262 except CalledProcessError as e:
263 logger.error("Subprocess returned with return code "
264 "while running `file' utility on core-file "
266 "rc=%s", e.returncode)
268 logger.error("Subprocess returned with OS error while "
269 "running 'file' utility "
271 "(%s) %s", e.errno, e.strerror)
272 except Exception as e:
273 logger.exception("Unexpected error running `file' utility "
277 # Copy api post mortem
278 api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
279 if os.path.isfile(api_post_mortem_path):
280 logger.error("Copying api_post_mortem.%d to %s" %
281 (vpp_pid, last_test_temp_dir))
282 shutil.copy2(api_post_mortem_path, last_test_temp_dir)
285 def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
286 if is_core_present(tempdir):
288 print('VPP core detected in %s. Last test running was %s' %
289 (tempdir, core_crash_test))
290 print(single_line_delim)
291 spawn_gdb(vpp_binary, get_core_path(tempdir))
292 print(single_line_delim)
294 print("Compressing core-file in test directory `%s'" % tempdir)
295 os.system("gzip %s" % get_core_path(tempdir))
298 def handle_cores(failed_testcases):
299 for failed_testcase in failed_testcases:
300 tcs_with_core = failed_testcase.testclasess_with_core
302 for test, vpp_binary, tempdir in tcs_with_core.values():
303 check_and_handle_core(vpp_binary, tempdir, test)
306 def process_finished_testsuite(wrapped_testcase_suite,
307 finished_testcase_suites,
308 failed_wrapped_testcases,
310 results.append(wrapped_testcase_suite.result)
311 finished_testcase_suites.add(wrapped_testcase_suite)
313 if failfast and not wrapped_testcase_suite.was_successful():
316 if not wrapped_testcase_suite.was_successful():
317 failed_wrapped_testcases.add(wrapped_testcase_suite)
318 handle_failed_suite(wrapped_testcase_suite.logger,
319 wrapped_testcase_suite.last_test_temp_dir,
320 wrapped_testcase_suite.vpp_pid)
325 def run_forked(testcase_suites):
326 wrapped_testcase_suites = set()
328 # suites are unhashable, need to use list
330 unread_testcases = set()
331 finished_unread_testcases = set()
332 manager = StreamQueueManager()
334 for i in range(concurrent_tests):
336 wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0),
338 wrapped_testcase_suites.add(wrapped_testcase_suite)
339 unread_testcases.add(wrapped_testcase_suite)
343 read_from_testcases = threading.Event()
344 read_from_testcases.set()
345 stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper,
346 args=(unread_testcases,
347 finished_unread_testcases,
348 read_from_testcases))
349 stdouterr_thread.start()
351 failed_wrapped_testcases = set()
355 while wrapped_testcase_suites:
356 finished_testcase_suites = set()
357 for wrapped_testcase_suite in wrapped_testcase_suites:
358 while wrapped_testcase_suite.result_parent_end.poll():
359 wrapped_testcase_suite.result.process_result(
360 *wrapped_testcase_suite.result_parent_end.recv())
361 wrapped_testcase_suite.last_heard = time.time()
363 while wrapped_testcase_suite.keep_alive_parent_end.poll():
364 wrapped_testcase_suite.last_test, \
365 wrapped_testcase_suite.last_test_vpp_binary, \
366 wrapped_testcase_suite.last_test_temp_dir, \
367 wrapped_testcase_suite.vpp_pid = \
368 wrapped_testcase_suite.keep_alive_parent_end.recv()
369 wrapped_testcase_suite.last_heard = time.time()
371 if wrapped_testcase_suite.finished_parent_end.poll():
372 wrapped_testcase_suite.finished_parent_end.recv()
373 wrapped_testcase_suite.last_heard = time.time()
374 stop_run = process_finished_testsuite(
375 wrapped_testcase_suite,
376 finished_testcase_suites,
377 failed_wrapped_testcases,
382 if wrapped_testcase_suite.last_heard + test_timeout < \
385 wrapped_testcase_suite.logger.critical(
386 "Child test runner process timed out "
387 "(last test running was `%s' in `%s')!" %
388 (wrapped_testcase_suite.last_test,
389 wrapped_testcase_suite.last_test_temp_dir))
390 elif not wrapped_testcase_suite.child.is_alive():
392 wrapped_testcase_suite.logger.critical(
393 "Child test runner process unexpectedly died "
394 "(last test running was `%s' in `%s')!" %
395 (wrapped_testcase_suite.last_test,
396 wrapped_testcase_suite.last_test_temp_dir))
397 elif wrapped_testcase_suite.last_test_temp_dir and \
398 wrapped_testcase_suite.last_test_vpp_binary:
400 wrapped_testcase_suite.last_test_temp_dir):
401 wrapped_testcase_suite.add_testclass_with_core()
402 if wrapped_testcase_suite.core_detected_at is None:
403 wrapped_testcase_suite.core_detected_at = \
405 elif wrapped_testcase_suite.core_detected_at + \
406 core_timeout < time.time():
407 wrapped_testcase_suite.logger.critical(
408 "Child test runner process unresponsive and "
409 "core-file exists in test temporary directory "
410 "(last test running was `%s' in `%s')!" %
411 (wrapped_testcase_suite.last_test,
412 wrapped_testcase_suite.last_test_temp_dir))
416 wrapped_testcase_suite.child.terminate()
418 # terminating the child process tends to leave orphan
420 if wrapped_testcase_suite.vpp_pid:
421 os.kill(wrapped_testcase_suite.vpp_pid,
426 wrapped_testcase_suite.result.crashed = True
427 wrapped_testcase_suite.result.process_result(
428 wrapped_testcase_suite.last_test_id, ERROR)
429 stop_run = process_finished_testsuite(
430 wrapped_testcase_suite,
431 finished_testcase_suites,
432 failed_wrapped_testcases,
435 for finished_testcase in finished_testcase_suites:
436 # Somewhat surprisingly, the join below may
437 # timeout, even if client signaled that
438 # it finished - so we note it just in case.
439 join_start = time.time()
440 finished_testcase.child.join(test_finished_join_timeout)
441 join_end = time.time()
442 if join_end - join_start >= test_finished_join_timeout:
443 finished_testcase.logger.error(
444 "Timeout joining finished test: %s (pid %d)" %
445 (finished_testcase.last_test,
446 finished_testcase.child.pid))
447 finished_testcase.close_pipes()
448 wrapped_testcase_suites.remove(finished_testcase)
449 finished_unread_testcases.add(finished_testcase)
450 finished_testcase.stdouterr_queue.put(None)
452 while testcase_suites:
453 results.append(TestResult(testcase_suites.pop(0)))
454 elif testcase_suites:
455 new_testcase = TestCaseWrapper(testcase_suites.pop(0),
457 wrapped_testcase_suites.add(new_testcase)
458 unread_testcases.add(new_testcase)
461 for wrapped_testcase_suite in wrapped_testcase_suites:
462 wrapped_testcase_suite.child.terminate()
463 wrapped_testcase_suite.stdouterr_queue.put(None)
466 read_from_testcases.clear()
467 stdouterr_thread.join(test_timeout)
470 handle_cores(failed_wrapped_testcases)
474 class SplitToSuitesCallback:
475 def __init__(self, filter_callback):
477 self.suite_name = 'default'
478 self.filter_callback = filter_callback
479 self.filtered = unittest.TestSuite()
481 def __call__(self, file_name, cls, method):
482 test_method = cls(method)
483 if self.filter_callback(file_name, cls.__name__, method):
484 self.suite_name = file_name + cls.__name__
485 if self.suite_name not in self.suites:
486 self.suites[self.suite_name] = unittest.TestSuite()
487 self.suites[self.suite_name].addTest(test_method)
490 self.filtered.addTest(test_method)
496 def parse_test_option():
497 f = os.getenv(test_option, None)
498 filter_file_name = None
499 filter_class_name = None
500 filter_func_name = None
505 raise Exception("Unrecognized %s option: %s" %
508 if parts[2] not in ('*', ''):
509 filter_func_name = parts[2]
510 if parts[1] not in ('*', ''):
511 filter_class_name = parts[1]
512 if parts[0] not in ('*', ''):
513 if parts[0].startswith('test_'):
514 filter_file_name = parts[0]
516 filter_file_name = 'test_%s' % parts[0]
518 if f.startswith('test_'):
521 filter_file_name = 'test_%s' % f
523 filter_file_name = '%s.py' % filter_file_name
524 return filter_file_name, filter_class_name, filter_func_name
527 def filter_tests(tests, filter_cb):
528 result = unittest.suite.TestSuite()
530 if isinstance(t, unittest.suite.TestSuite):
531 # this is a bunch of tests, recursively filter...
532 x = filter_tests(t, filter_cb)
533 if x.countTestCases() > 0:
535 elif isinstance(t, unittest.TestCase):
536 # this is a single test
537 parts = t.id().split('.')
538 # t.id() for common cases like this:
539 # test_classifier.TestClassifier.test_acl_ip
540 # apply filtering only if it is so
542 if not filter_cb(parts[0], parts[1], parts[2]):
546 # unexpected object, don't touch it
551 class FilterByTestOption:
552 def __init__(self, filter_file_name, filter_class_name, filter_func_name):
553 self.filter_file_name = filter_file_name
554 self.filter_class_name = filter_class_name
555 self.filter_func_name = filter_func_name
557 def __call__(self, file_name, class_name, func_name):
558 if self.filter_file_name:
559 fn_match = fnmatch.fnmatch(file_name, self.filter_file_name)
562 if self.filter_class_name and class_name != self.filter_class_name:
564 if self.filter_func_name and func_name != self.filter_func_name:
569 class FilterByClassList:
570 def __init__(self, classes_with_filenames):
571 self.classes_with_filenames = classes_with_filenames
573 def __call__(self, file_name, class_name, func_name):
574 return '.'.join([file_name, class_name]) in self.classes_with_filenames
577 def suite_from_failed(suite, failed):
578 failed = {x.rsplit('.', 1)[0] for x in failed}
579 filter_cb = FilterByClassList(failed)
580 suite = filter_tests(suite, filter_cb)
584 class AllResults(dict):
586 super(AllResults, self).__init__()
587 self.all_testcases = 0
588 self.results_per_suite = []
595 self.testsuites_no_tests_run = []
597 def add_results(self, result):
598 self.results_per_suite.append(result)
599 result_types = [PASS, FAIL, ERROR, SKIP, TEST_RUN]
600 for result_type in result_types:
601 self[result_type] += len(result[result_type])
603 def add_result(self, result):
605 self.all_testcases += result.testcase_suite.countTestCases()
606 self.add_results(result)
608 if result.no_tests_run():
609 self.testsuites_no_tests_run.append(result.testcase_suite)
614 elif not result.was_successful():
618 self.rerun.append(result.testcase_suite)
622 def print_results(self):
624 print(double_line_delim)
625 print('TEST RESULTS:')
626 print(' Scheduled tests: {}'.format(self.all_testcases))
627 print(' Executed tests: {}'.format(self[TEST_RUN]))
628 print(' Passed tests: {}'.format(
629 colorize(str(self[PASS]), GREEN)))
631 print(' Skipped tests: {}'.format(
632 colorize(str(self[SKIP]), YELLOW)))
633 if self.not_executed > 0:
634 print(' Not Executed tests: {}'.format(
635 colorize(str(self.not_executed), RED)))
637 print(' Failures: {}'.format(
638 colorize(str(self[FAIL]), RED)))
640 print(' Errors: {}'.format(
641 colorize(str(self[ERROR]), RED)))
643 if self.all_failed > 0:
644 print('FAILURES AND ERRORS IN TESTS:')
645 for result in self.results_per_suite:
646 failed_testcase_ids = result[FAIL]
647 errored_testcase_ids = result[ERROR]
648 old_testcase_name = None
649 if failed_testcase_ids:
650 for failed_test_id in failed_testcase_ids:
651 new_testcase_name, test_name = \
652 result.get_testcase_names(failed_test_id)
653 if new_testcase_name != old_testcase_name:
654 print(' Testcase name: {}'.format(
655 colorize(new_testcase_name, RED)))
656 old_testcase_name = new_testcase_name
657 print(' FAILURE: {} [{}]'.format(
658 colorize(test_name, RED), failed_test_id))
659 if errored_testcase_ids:
660 for errored_test_id in errored_testcase_ids:
661 new_testcase_name, test_name = \
662 result.get_testcase_names(errored_test_id)
663 if new_testcase_name != old_testcase_name:
664 print(' Testcase name: {}'.format(
665 colorize(new_testcase_name, RED)))
666 old_testcase_name = new_testcase_name
667 print(' ERROR: {} [{}]'.format(
668 colorize(test_name, RED), errored_test_id))
669 if self.testsuites_no_tests_run:
670 print('TESTCASES WHERE NO TESTS WERE SUCCESSFULLY EXECUTED:')
672 for testsuite in self.testsuites_no_tests_run:
673 for testcase in testsuite:
674 tc_classes.add(get_testcase_doc_name(testcase))
675 for tc_class in tc_classes:
676 print(' {}'.format(colorize(tc_class, RED)))
678 print(double_line_delim)
682 def not_executed(self):
683 return self.all_testcases - self[TEST_RUN]
686 def all_failed(self):
687 return self[FAIL] + self[ERROR]
690 def parse_results(results):
692 Prints the number of scheduled, executed, not executed, passed, failed,
693 errored and skipped tests and details about failed and errored tests.
695 Also returns all suites where any test failed.
701 results_per_suite = AllResults()
704 for result in results:
705 result_code = results_per_suite.add_result(result)
708 elif result_code == -1:
711 results_per_suite.print_results()
719 return return_code, results_per_suite.rerun
722 def parse_digit_env(env_var, default):
723 value = os.getenv(env_var, default)
728 print('WARNING: unsupported value "%s" for env var "%s",'
729 'defaulting to %s' % (value, env_var, default))
734 if __name__ == '__main__':
736 verbose = parse_digit_env("V", 0)
738 test_timeout = parse_digit_env("TIMEOUT", 600) # default = 10 minutes
740 test_finished_join_timeout = 15
742 retries = parse_digit_env("RETRIES", 0)
744 debug = os.getenv("DEBUG", "n").lower() in ["gdb", "gdbserver"]
746 debug_core = os.getenv("DEBUG", "").lower() == "core"
747 compress_core = framework.BoolEnvironmentVariable("CORE_COMPRESS")
749 step = framework.BoolEnvironmentVariable("STEP")
750 force_foreground = framework.BoolEnvironmentVariable("FORCE_FOREGROUND")
752 run_interactive = debug or step or force_foreground
755 num_cpus = len(os.sched_getaffinity(0))
756 except AttributeError:
757 num_cpus = multiprocessing.cpu_count()
758 shm_free = psutil.disk_usage('/dev/shm').free
760 print('OS reports %s available cpu(s). Free shm: %s' % (
761 num_cpus, "{:,}MB".format(shm_free / (1024 * 1024))))
763 test_jobs = os.getenv("TEST_JOBS", "1").lower() # default = 1 process
764 if test_jobs == 'auto':
767 print('Interactive mode required, running on one core')
769 shm_max_processes = 1
770 if shm_free < min_req_shm:
771 raise Exception('Not enough free space in /dev/shm. Required '
772 'free space is at least %sM.'
773 % (min_req_shm >> 20))
775 extra_shm = shm_free - min_req_shm
776 shm_max_processes += extra_shm // shm_per_process
777 concurrent_tests = min(cpu_count(), shm_max_processes)
778 print('Found enough resources to run tests with %s cores'
780 elif test_jobs.isdigit():
781 concurrent_tests = int(test_jobs)
782 print("Running on %s core(s) as set by 'TEST_JOBS'." %
786 print('Running on one core.')
788 if run_interactive and concurrent_tests > 1:
789 raise NotImplementedError(
790 'Running tests interactively (DEBUG is gdb or gdbserver or STEP '
791 'is set) in parallel (TEST_JOBS is more than 1) is not supported')
793 parser = argparse.ArgumentParser(description="VPP unit tests")
794 parser.add_argument("-f", "--failfast", action='store_true',
795 help="fast failure flag")
796 parser.add_argument("-d", "--dir", action='append', type=str,
797 help="directory containing test files "
798 "(may be specified multiple times)")
799 args = parser.parse_args()
800 failfast = args.failfast
803 print("Running tests using custom test runner") # debug message
804 filter_file, filter_class, filter_func = parse_test_option()
806 print("Active filters: file=%s, class=%s, function=%s" % (
807 filter_file, filter_class, filter_func))
809 filter_cb = FilterByTestOption(filter_file, filter_class, filter_func)
811 ignore_path = os.getenv("VENV_PATH", None)
812 cb = SplitToSuitesCallback(filter_cb)
814 print("Adding tests from directory tree %s" % d)
815 discover_tests(d, cb, ignore_path)
817 # suites are not hashable, need to use list
820 for testcase_suite in cb.suites.values():
821 tests_amount += testcase_suite.countTestCases()
822 suites.append(testcase_suite)
824 print("%s out of %s tests match specified filters" % (
825 tests_amount, tests_amount + cb.filtered.countTestCases()))
827 if not running_extended_tests:
828 print("Not running extended tests (some tests will be skipped)")
830 attempts = retries + 1
832 print("Perform %s attempts to pass the suite..." % attempts)
834 if run_interactive and suites:
835 # don't fork if requiring interactive terminal
836 print('Running tests in foreground in the current process')
837 full_suite = unittest.TestSuite()
838 full_suite.addTests(suites)
839 result = VppTestRunner(verbosity=verbose,
841 print_summary=True).run(full_suite)
842 was_successful = result.wasSuccessful()
843 if not was_successful:
844 for test_case_info in result.failed_test_cases_info:
845 handle_failed_suite(test_case_info.logger,
846 test_case_info.tempdir,
847 test_case_info.vpp_pid)
848 if test_case_info in result.core_crash_test_cases_info:
849 check_and_handle_core(test_case_info.vpp_bin_path,
850 test_case_info.tempdir,
851 test_case_info.core_crash_test)
853 sys.exit(not was_successful)
855 print('Running each VPPTestCase in a separate background process'
856 ' with {} parallel process(es)'.format(concurrent_tests))
858 while suites and attempts > 0:
859 results = run_forked(suites)
860 exit_code, suites = parse_results(results)
863 print('Test run was successful')
865 print('%s attempt(s) left.' % attempts)