14 import multiprocessing
15 from multiprocessing import Process, Pipe, cpu_count
16 from multiprocessing.queues import Queue
17 from multiprocessing.managers import BaseManager
19 from framework import VppTestRunner, running_extended_tests, VppTestCase, \
20 get_testcase_doc_name, get_test_description, PASS, FAIL, ERROR, SKIP, \
22 from debug import spawn_gdb, start_vpp_in_gdb
23 from log import get_parallel_logger, double_line_delim, RED, YELLOW, GREEN, \
24 colorize, single_line_delim
25 from discover_tests import discover_tests
26 from subprocess import check_output, CalledProcessError
27 from util import check_core_path, get_core_path, is_core_present
29 # timeout which controls how long the child has to finish after seeing
30 # a core dump in test temporary directory. If this is exceeded, parent assumes
31 # that child process is stuck (e.g. waiting for event from vpp) and kill
36 class StreamQueue(Queue):
41 sys.__stdout__.flush()
42 sys.__stderr__.flush()
45 return self._writer.fileno()
48 class StreamQueueManager(BaseManager):
52 StreamQueueManager.register('StreamQueue', StreamQueue)
55 class TestResult(dict):
56 def __init__(self, testcase_suite, testcases_by_id=None):
57 super(TestResult, self).__init__()
64 self.testcase_suite = testcase_suite
65 self.testcases = [testcase for testcase in testcase_suite]
66 self.testcases_by_id = testcases_by_id
68 def was_successful(self):
69 return 0 == len(self[FAIL]) == len(self[ERROR]) \
70 and len(self[PASS] + self[SKIP]) \
71 == self.testcase_suite.countTestCases() == len(self[TEST_RUN])
73 def no_tests_run(self):
74 return 0 == len(self[TEST_RUN])
76 def process_result(self, test_id, result):
77 self[result].append(test_id)
79 def suite_from_failed(self):
81 for testcase in self.testcase_suite:
83 if tc_id not in self[PASS] and tc_id not in self[SKIP]:
86 return suite_from_failed(self.testcase_suite, rerun_ids)
88 def get_testcase_names(self, test_id):
89 # could be tearDownClass (test_ipsec_esp.TestIpsecEsp1)
90 setup_teardown_match = re.match(
91 r'((tearDownClass)|(setUpClass)) \((.+\..+)\)', test_id)
92 if setup_teardown_match:
93 test_name, _, _, testcase_name = setup_teardown_match.groups()
94 if len(testcase_name.split('.')) == 2:
95 for key in self.testcases_by_id.keys():
96 if key.startswith(testcase_name):
99 testcase_name = self._get_testcase_doc_name(testcase_name)
101 test_name = self._get_test_description(test_id)
102 testcase_name = self._get_testcase_doc_name(test_id)
104 return testcase_name, test_name
106 def _get_test_description(self, test_id):
107 if test_id in self.testcases_by_id:
108 desc = get_test_description(descriptions,
109 self.testcases_by_id[test_id])
114 def _get_testcase_doc_name(self, test_id):
115 if test_id in self.testcases_by_id:
116 doc_name = get_testcase_doc_name(self.testcases_by_id[test_id])
122 def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue,
123 finished_pipe, result_pipe, logger):
124 sys.stdout = stdouterr_queue
125 sys.stderr = stdouterr_queue
126 VppTestCase.parallel_handler = logger.handlers[0]
127 result = VppTestRunner(keep_alive_pipe=keep_alive_pipe,
128 descriptions=descriptions,
130 result_pipe=result_pipe,
132 print_summary=False).run(suite)
133 finished_pipe.send(result.wasSuccessful())
134 finished_pipe.close()
135 keep_alive_pipe.close()
138 class TestCaseWrapper(object):
139 def __init__(self, testcase_suite, manager):
140 self.keep_alive_parent_end, self.keep_alive_child_end = Pipe(
142 self.finished_parent_end, self.finished_child_end = Pipe(duplex=False)
143 self.result_parent_end, self.result_child_end = Pipe(duplex=False)
144 self.testcase_suite = testcase_suite
145 if sys.version[0] == '2':
146 self.stdouterr_queue = manager.StreamQueue()
148 from multiprocessing import get_context
149 self.stdouterr_queue = manager.StreamQueue(ctx=get_context())
150 self.logger = get_parallel_logger(self.stdouterr_queue)
151 self.child = Process(target=test_runner_wrapper,
152 args=(testcase_suite,
153 self.keep_alive_child_end,
154 self.stdouterr_queue,
155 self.finished_child_end,
156 self.result_child_end,
160 self.last_test_temp_dir = None
161 self.last_test_vpp_binary = None
162 self._last_test = None
163 self.last_test_id = None
165 self.last_heard = time.time()
166 self.core_detected_at = None
167 self.testcases_by_id = {}
168 self.testclasess_with_core = {}
169 for testcase in self.testcase_suite:
170 self.testcases_by_id[testcase.id()] = testcase
171 self.result = TestResult(testcase_suite, self.testcases_by_id)
175 return self._last_test
178 def last_test(self, test_id):
179 self.last_test_id = test_id
180 if test_id in self.testcases_by_id:
181 testcase = self.testcases_by_id[test_id]
182 self._last_test = testcase.shortDescription()
183 if not self._last_test:
184 self._last_test = str(testcase)
186 self._last_test = test_id
188 def add_testclass_with_core(self):
189 if self.last_test_id in self.testcases_by_id:
190 test = self.testcases_by_id[self.last_test_id]
191 class_name = unittest.util.strclass(test.__class__)
192 test_name = "'{}' ({})".format(get_test_description(descriptions,
196 test_name = self.last_test_id
197 class_name = re.match(r'((tearDownClass)|(setUpClass)) '
198 r'\((.+\..+)\)', test_name).groups()[3]
199 if class_name not in self.testclasess_with_core:
200 self.testclasess_with_core[class_name] = (
202 self.last_test_vpp_binary,
203 self.last_test_temp_dir)
205 def close_pipes(self):
206 self.keep_alive_child_end.close()
207 self.finished_child_end.close()
208 self.result_child_end.close()
209 self.keep_alive_parent_end.close()
210 self.finished_parent_end.close()
211 self.result_parent_end.close()
213 def was_successful(self):
214 return self.result.was_successful()
217 def stdouterr_reader_wrapper(unread_testcases, finished_unread_testcases,
220 while read_testcases.is_set() or unread_testcases:
221 if finished_unread_testcases:
222 read_testcase = finished_unread_testcases.pop()
223 unread_testcases.remove(read_testcase)
224 elif unread_testcases:
225 read_testcase = unread_testcases.pop()
228 while data is not None:
229 sys.stdout.write(data)
230 data = read_testcase.stdouterr_queue.get()
232 read_testcase.stdouterr_queue.close()
233 finished_unread_testcases.discard(read_testcase)
237 def handle_failed_suite(logger, last_test_temp_dir, vpp_pid):
238 if last_test_temp_dir:
239 # Need to create link in case of a timeout or core dump without failure
240 lttd = os.path.basename(last_test_temp_dir)
241 failed_dir = os.getenv('FAILED_DIR')
242 link_path = '%s%s-FAILED' % (failed_dir, lttd)
243 if not os.path.exists(link_path):
244 os.symlink(last_test_temp_dir, link_path)
245 logger.error("Symlink to failed testcase directory: %s -> %s"
248 # Report core existence
249 core_path = get_core_path(last_test_temp_dir)
250 if os.path.exists(core_path):
252 "Core-file exists in test temporary directory: %s!" %
254 check_core_path(logger, core_path)
255 logger.debug("Running 'file %s':" % core_path)
257 info = check_output(["file", core_path])
259 except CalledProcessError as e:
260 logger.error("Subprocess returned with return code "
261 "while running `file' utility on core-file "
263 "rc=%s", e.returncode)
265 logger.error("Subprocess returned with OS error while "
266 "running 'file' utility "
268 "(%s) %s", e.errno, e.strerror)
269 except Exception as e:
270 logger.exception("Unexpected error running `file' utility "
272 logger.error("gdb %s %s" %
273 (os.getenv('VPP_BIN', 'vpp'), core_path))
276 # Copy api post mortem
277 api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
278 if os.path.isfile(api_post_mortem_path):
279 logger.error("Copying api_post_mortem.%d to %s" %
280 (vpp_pid, last_test_temp_dir))
281 shutil.copy2(api_post_mortem_path, last_test_temp_dir)
284 def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
285 if is_core_present(tempdir):
287 print('VPP core detected in %s. Last test running was %s' %
288 (tempdir, core_crash_test))
289 print(single_line_delim)
290 spawn_gdb(vpp_binary, get_core_path(tempdir))
291 print(single_line_delim)
293 print("Compressing core-file in test directory `%s'" % tempdir)
294 os.system("gzip %s" % get_core_path(tempdir))
297 def handle_cores(failed_testcases):
298 for failed_testcase in failed_testcases:
299 tcs_with_core = failed_testcase.testclasess_with_core
301 for test, vpp_binary, tempdir in tcs_with_core.values():
302 check_and_handle_core(vpp_binary, tempdir, test)
305 def process_finished_testsuite(wrapped_testcase_suite,
306 finished_testcase_suites,
307 failed_wrapped_testcases,
309 results.append(wrapped_testcase_suite.result)
310 finished_testcase_suites.add(wrapped_testcase_suite)
312 if failfast and not wrapped_testcase_suite.was_successful():
315 if not wrapped_testcase_suite.was_successful():
316 failed_wrapped_testcases.add(wrapped_testcase_suite)
317 handle_failed_suite(wrapped_testcase_suite.logger,
318 wrapped_testcase_suite.last_test_temp_dir,
319 wrapped_testcase_suite.vpp_pid)
324 def run_forked(testcase_suites):
325 wrapped_testcase_suites = set()
326 solo_testcase_suites = []
327 total_test_runners = 0
329 # suites are unhashable, need to use list
331 unread_testcases = set()
332 finished_unread_testcases = set()
333 manager = StreamQueueManager()
335 total_test_runners = 0
336 while total_test_runners < concurrent_tests:
338 a_suite = testcase_suites.pop(0)
339 if a_suite.is_tagged_run_solo:
340 solo_testcase_suites.append(a_suite)
342 wrapped_testcase_suite = TestCaseWrapper(a_suite,
344 wrapped_testcase_suites.add(wrapped_testcase_suite)
345 unread_testcases.add(wrapped_testcase_suite)
346 total_test_runners = total_test_runners + 1
350 while total_test_runners < 1 and solo_testcase_suites:
351 if solo_testcase_suites:
352 a_suite = solo_testcase_suites.pop(0)
353 wrapped_testcase_suite = TestCaseWrapper(a_suite,
355 wrapped_testcase_suites.add(wrapped_testcase_suite)
356 unread_testcases.add(wrapped_testcase_suite)
357 total_test_runners = total_test_runners + 1
361 read_from_testcases = threading.Event()
362 read_from_testcases.set()
363 stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper,
364 args=(unread_testcases,
365 finished_unread_testcases,
366 read_from_testcases))
367 stdouterr_thread.start()
369 failed_wrapped_testcases = set()
373 while wrapped_testcase_suites:
374 finished_testcase_suites = set()
375 for wrapped_testcase_suite in wrapped_testcase_suites:
376 while wrapped_testcase_suite.result_parent_end.poll():
377 wrapped_testcase_suite.result.process_result(
378 *wrapped_testcase_suite.result_parent_end.recv())
379 wrapped_testcase_suite.last_heard = time.time()
381 while wrapped_testcase_suite.keep_alive_parent_end.poll():
382 wrapped_testcase_suite.last_test, \
383 wrapped_testcase_suite.last_test_vpp_binary, \
384 wrapped_testcase_suite.last_test_temp_dir, \
385 wrapped_testcase_suite.vpp_pid = \
386 wrapped_testcase_suite.keep_alive_parent_end.recv()
387 wrapped_testcase_suite.last_heard = time.time()
389 if wrapped_testcase_suite.finished_parent_end.poll():
390 wrapped_testcase_suite.finished_parent_end.recv()
391 wrapped_testcase_suite.last_heard = time.time()
392 stop_run = process_finished_testsuite(
393 wrapped_testcase_suite,
394 finished_testcase_suites,
395 failed_wrapped_testcases,
400 if wrapped_testcase_suite.last_heard + test_timeout < \
403 wrapped_testcase_suite.logger.critical(
404 "Child test runner process timed out "
405 "(last test running was `%s' in `%s')!" %
406 (wrapped_testcase_suite.last_test,
407 wrapped_testcase_suite.last_test_temp_dir))
408 elif not wrapped_testcase_suite.child.is_alive():
410 wrapped_testcase_suite.logger.critical(
411 "Child test runner process unexpectedly died "
412 "(last test running was `%s' in `%s')!" %
413 (wrapped_testcase_suite.last_test,
414 wrapped_testcase_suite.last_test_temp_dir))
415 elif wrapped_testcase_suite.last_test_temp_dir and \
416 wrapped_testcase_suite.last_test_vpp_binary:
418 wrapped_testcase_suite.last_test_temp_dir):
419 wrapped_testcase_suite.add_testclass_with_core()
420 if wrapped_testcase_suite.core_detected_at is None:
421 wrapped_testcase_suite.core_detected_at = \
423 elif wrapped_testcase_suite.core_detected_at + \
424 core_timeout < time.time():
425 wrapped_testcase_suite.logger.critical(
426 "Child test runner process unresponsive and "
427 "core-file exists in test temporary directory "
428 "(last test running was `%s' in `%s')!" %
429 (wrapped_testcase_suite.last_test,
430 wrapped_testcase_suite.last_test_temp_dir))
434 wrapped_testcase_suite.child.terminate()
436 # terminating the child process tends to leave orphan
438 if wrapped_testcase_suite.vpp_pid:
439 os.kill(wrapped_testcase_suite.vpp_pid,
444 wrapped_testcase_suite.result.crashed = True
445 wrapped_testcase_suite.result.process_result(
446 wrapped_testcase_suite.last_test_id, ERROR)
447 stop_run = process_finished_testsuite(
448 wrapped_testcase_suite,
449 finished_testcase_suites,
450 failed_wrapped_testcases,
453 for finished_testcase in finished_testcase_suites:
454 # Somewhat surprisingly, the join below may
455 # timeout, even if client signaled that
456 # it finished - so we note it just in case.
457 join_start = time.time()
458 finished_testcase.child.join(test_finished_join_timeout)
459 join_end = time.time()
460 if join_end - join_start >= test_finished_join_timeout:
461 finished_testcase.logger.error(
462 "Timeout joining finished test: %s (pid %d)" %
463 (finished_testcase.last_test,
464 finished_testcase.child.pid))
465 finished_testcase.close_pipes()
466 wrapped_testcase_suites.remove(finished_testcase)
467 finished_unread_testcases.add(finished_testcase)
468 finished_testcase.stdouterr_queue.put(None)
469 total_test_runners = total_test_runners - 1
471 while testcase_suites:
472 results.append(TestResult(testcase_suites.pop(0)))
473 elif testcase_suites:
474 a_testcase = testcase_suites.pop(0)
475 while a_testcase and a_testcase.is_tagged_run_solo:
476 solo_testcase_suites.append(a_testcase)
478 a_testcase = testcase_suites.pop(0)
482 new_testcase = TestCaseWrapper(a_testcase,
484 wrapped_testcase_suites.add(new_testcase)
485 total_test_runners = total_test_runners + 1
486 unread_testcases.add(new_testcase)
487 if solo_testcase_suites and total_test_runners == 0:
488 a_testcase = solo_testcase_suites.pop(0)
489 new_testcase = TestCaseWrapper(a_testcase,
491 wrapped_testcase_suites.add(new_testcase)
492 total_test_runners = total_test_runners + 1
493 unread_testcases.add(new_testcase)
496 for wrapped_testcase_suite in wrapped_testcase_suites:
497 wrapped_testcase_suite.child.terminate()
498 wrapped_testcase_suite.stdouterr_queue.put(None)
501 read_from_testcases.clear()
502 stdouterr_thread.join(test_timeout)
505 handle_cores(failed_wrapped_testcases)
509 class SplitToSuitesCallback:
510 def __init__(self, filter_callback):
512 self.suite_name = 'default'
513 self.filter_callback = filter_callback
514 self.filtered = unittest.TestSuite()
516 def __call__(self, file_name, cls, method):
517 test_method = cls(method)
518 if self.filter_callback(file_name, cls.__name__, method):
519 self.suite_name = file_name + cls.__name__
520 if self.suite_name not in self.suites:
521 self.suites[self.suite_name] = unittest.TestSuite()
522 self.suites[self.suite_name].is_tagged_run_solo = False
523 self.suites[self.suite_name].addTest(test_method)
524 if test_method.is_tagged_run_solo():
525 self.suites[self.suite_name].is_tagged_run_solo = True
528 self.filtered.addTest(test_method)
534 def parse_test_option():
535 f = os.getenv(test_option, None)
536 filter_file_name = None
537 filter_class_name = None
538 filter_func_name = None
543 raise Exception("Unrecognized %s option: %s" %
546 if parts[2] not in ('*', ''):
547 filter_func_name = parts[2]
548 if parts[1] not in ('*', ''):
549 filter_class_name = parts[1]
550 if parts[0] not in ('*', ''):
551 if parts[0].startswith('test_'):
552 filter_file_name = parts[0]
554 filter_file_name = 'test_%s' % parts[0]
556 if f.startswith('test_'):
559 filter_file_name = 'test_%s' % f
561 filter_file_name = '%s.py' % filter_file_name
562 return filter_file_name, filter_class_name, filter_func_name
565 def filter_tests(tests, filter_cb):
566 result = unittest.suite.TestSuite()
568 if isinstance(t, unittest.suite.TestSuite):
569 # this is a bunch of tests, recursively filter...
570 x = filter_tests(t, filter_cb)
571 if x.countTestCases() > 0:
573 elif isinstance(t, unittest.TestCase):
574 # this is a single test
575 parts = t.id().split('.')
576 # t.id() for common cases like this:
577 # test_classifier.TestClassifier.test_acl_ip
578 # apply filtering only if it is so
580 if not filter_cb(parts[0], parts[1], parts[2]):
584 # unexpected object, don't touch it
589 class FilterByTestOption:
590 def __init__(self, filter_file_name, filter_class_name, filter_func_name):
591 self.filter_file_name = filter_file_name
592 self.filter_class_name = filter_class_name
593 self.filter_func_name = filter_func_name
595 def __call__(self, file_name, class_name, func_name):
596 if self.filter_file_name:
597 fn_match = fnmatch.fnmatch(file_name, self.filter_file_name)
600 if self.filter_class_name and class_name != self.filter_class_name:
602 if self.filter_func_name and func_name != self.filter_func_name:
607 class FilterByClassList:
608 def __init__(self, classes_with_filenames):
609 self.classes_with_filenames = classes_with_filenames
611 def __call__(self, file_name, class_name, func_name):
612 return '.'.join([file_name, class_name]) in self.classes_with_filenames
615 def suite_from_failed(suite, failed):
616 failed = {x.rsplit('.', 1)[0] for x in failed}
617 filter_cb = FilterByClassList(failed)
618 suite = filter_tests(suite, filter_cb)
622 class AllResults(dict):
624 super(AllResults, self).__init__()
625 self.all_testcases = 0
626 self.results_per_suite = []
633 self.testsuites_no_tests_run = []
635 def add_results(self, result):
636 self.results_per_suite.append(result)
637 result_types = [PASS, FAIL, ERROR, SKIP, TEST_RUN]
638 for result_type in result_types:
639 self[result_type] += len(result[result_type])
641 def add_result(self, result):
643 self.all_testcases += result.testcase_suite.countTestCases()
644 self.add_results(result)
646 if result.no_tests_run():
647 self.testsuites_no_tests_run.append(result.testcase_suite)
652 elif not result.was_successful():
656 self.rerun.append(result.testcase_suite)
660 def print_results(self):
662 print(double_line_delim)
663 print('TEST RESULTS:')
664 print(' Scheduled tests: {}'.format(self.all_testcases))
665 print(' Executed tests: {}'.format(self[TEST_RUN]))
666 print(' Passed tests: {}'.format(
667 colorize(str(self[PASS]), GREEN)))
669 print(' Skipped tests: {}'.format(
670 colorize(str(self[SKIP]), YELLOW)))
671 if self.not_executed > 0:
672 print(' Not Executed tests: {}'.format(
673 colorize(str(self.not_executed), RED)))
675 print(' Failures: {}'.format(
676 colorize(str(self[FAIL]), RED)))
678 print(' Errors: {}'.format(
679 colorize(str(self[ERROR]), RED)))
681 if self.all_failed > 0:
682 print('FAILURES AND ERRORS IN TESTS:')
683 for result in self.results_per_suite:
684 failed_testcase_ids = result[FAIL]
685 errored_testcase_ids = result[ERROR]
686 old_testcase_name = None
687 if failed_testcase_ids:
688 for failed_test_id in failed_testcase_ids:
689 new_testcase_name, test_name = \
690 result.get_testcase_names(failed_test_id)
691 if new_testcase_name != old_testcase_name:
692 print(' Testcase name: {}'.format(
693 colorize(new_testcase_name, RED)))
694 old_testcase_name = new_testcase_name
695 print(' FAILURE: {} [{}]'.format(
696 colorize(test_name, RED), failed_test_id))
697 if errored_testcase_ids:
698 for errored_test_id in errored_testcase_ids:
699 new_testcase_name, test_name = \
700 result.get_testcase_names(errored_test_id)
701 if new_testcase_name != old_testcase_name:
702 print(' Testcase name: {}'.format(
703 colorize(new_testcase_name, RED)))
704 old_testcase_name = new_testcase_name
705 print(' ERROR: {} [{}]'.format(
706 colorize(test_name, RED), errored_test_id))
707 if self.testsuites_no_tests_run:
708 print('TESTCASES WHERE NO TESTS WERE SUCCESSFULLY EXECUTED:')
710 for testsuite in self.testsuites_no_tests_run:
711 for testcase in testsuite:
712 tc_classes.add(get_testcase_doc_name(testcase))
713 for tc_class in tc_classes:
714 print(' {}'.format(colorize(tc_class, RED)))
716 print(double_line_delim)
720 def not_executed(self):
721 return self.all_testcases - self[TEST_RUN]
724 def all_failed(self):
725 return self[FAIL] + self[ERROR]
728 def parse_results(results):
730 Prints the number of scheduled, executed, not executed, passed, failed,
731 errored and skipped tests and details about failed and errored tests.
733 Also returns all suites where any test failed.
739 results_per_suite = AllResults()
742 for result in results:
743 result_code = results_per_suite.add_result(result)
746 elif result_code == -1:
749 results_per_suite.print_results()
757 return return_code, results_per_suite.rerun
760 def parse_digit_env(env_var, default):
761 value = os.getenv(env_var, default)
766 print('WARNING: unsupported value "%s" for env var "%s",'
767 'defaulting to %s' % (value, env_var, default))
772 if __name__ == '__main__':
774 verbose = parse_digit_env("V", 0)
776 test_timeout = parse_digit_env("TIMEOUT", 600) # default = 10 minutes
778 test_finished_join_timeout = 15
780 retries = parse_digit_env("RETRIES", 0)
782 debug = os.getenv("DEBUG", "n").lower() in ["gdb", "gdbserver", "attach"]
784 debug_core = os.getenv("DEBUG", "").lower() == "core"
785 compress_core = framework.BoolEnvironmentVariable("CORE_COMPRESS")
787 if os.getenv("VPP_IN_GDB", "n").lower() in ["1", "y", "yes"]:
791 step = framework.BoolEnvironmentVariable("STEP")
792 force_foreground = framework.BoolEnvironmentVariable("FORCE_FOREGROUND")
794 run_interactive = debug or step or force_foreground
797 num_cpus = len(os.sched_getaffinity(0))
798 except AttributeError:
799 num_cpus = multiprocessing.cpu_count()
801 print("OS reports %s available cpu(s)." % num_cpus)
803 test_jobs = os.getenv("TEST_JOBS", "1").lower() # default = 1 process
804 if test_jobs == 'auto':
807 print('Interactive mode required, running on one core')
809 concurrent_tests = num_cpus
810 print('Found enough resources to run tests with %s cores'
812 elif test_jobs.isdigit():
813 concurrent_tests = int(test_jobs)
814 print("Running on %s core(s) as set by 'TEST_JOBS'." %
818 print('Running on one core.')
820 if run_interactive and concurrent_tests > 1:
821 raise NotImplementedError(
822 'Running tests interactively (DEBUG is gdb[server] or ATTACH or '
823 'STEP is set) in parallel (TEST_JOBS is more than 1) is not '
826 parser = argparse.ArgumentParser(description="VPP unit tests")
827 parser.add_argument("-f", "--failfast", action='store_true',
828 help="fast failure flag")
829 parser.add_argument("-d", "--dir", action='append', type=str,
830 help="directory containing test files "
831 "(may be specified multiple times)")
832 args = parser.parse_args()
833 failfast = args.failfast
836 print("Running tests using custom test runner") # debug message
837 filter_file, filter_class, filter_func = parse_test_option()
839 print("Active filters: file=%s, class=%s, function=%s" % (
840 filter_file, filter_class, filter_func))
842 filter_cb = FilterByTestOption(filter_file, filter_class, filter_func)
844 ignore_path = os.getenv("VENV_PATH", None)
845 cb = SplitToSuitesCallback(filter_cb)
847 print("Adding tests from directory tree %s" % d)
848 discover_tests(d, cb, ignore_path)
850 # suites are not hashable, need to use list
853 for testcase_suite in cb.suites.values():
854 tests_amount += testcase_suite.countTestCases()
855 suites.append(testcase_suite)
857 print("%s out of %s tests match specified filters" % (
858 tests_amount, tests_amount + cb.filtered.countTestCases()))
860 if not running_extended_tests:
861 print("Not running extended tests (some tests will be skipped)")
863 attempts = retries + 1
865 print("Perform %s attempts to pass the suite..." % attempts)
867 if run_interactive and suites:
868 # don't fork if requiring interactive terminal
869 print('Running tests in foreground in the current process')
870 full_suite = unittest.TestSuite()
871 full_suite.addTests(suites)
872 result = VppTestRunner(verbosity=verbose,
874 print_summary=True).run(full_suite)
875 was_successful = result.wasSuccessful()
876 if not was_successful:
877 for test_case_info in result.failed_test_cases_info:
878 handle_failed_suite(test_case_info.logger,
879 test_case_info.tempdir,
880 test_case_info.vpp_pid)
881 if test_case_info in result.core_crash_test_cases_info:
882 check_and_handle_core(test_case_info.vpp_bin_path,
883 test_case_info.tempdir,
884 test_case_info.core_crash_test)
886 sys.exit(not was_successful)
888 print('Running each VPPTestCase in a separate background process'
889 ' with {} parallel process(es)'.format(concurrent_tests))
891 while suites and attempts > 0:
892 results = run_forked(suites)
893 exit_code, suites = parse_results(results)
896 print('Test run was successful')
898 print('%s attempt(s) left.' % attempts)