14 from multiprocessing import Process, Pipe, cpu_count
15 from multiprocessing.queues import Queue
16 from multiprocessing.managers import BaseManager
17 from framework import VppTestRunner, running_extended_tests, VppTestCase, \
18 get_testcase_doc_name, get_test_description, PASS, FAIL, ERROR, SKIP, \
20 from debug import spawn_gdb
21 from log import get_parallel_logger, double_line_delim, RED, YELLOW, GREEN, \
22 colorize, single_line_delim
23 from discover_tests import discover_tests
24 from subprocess import check_output, CalledProcessError
25 from util import check_core_path, get_core_path, is_core_present
27 # timeout which controls how long the child has to finish after seeing
28 # a core dump in test temporary directory. If this is exceeded, parent assumes
29 # that child process is stuck (e.g. waiting for shm mutex, which will never
30 # get unlocked) and kill the child
32 min_req_shm = 536870912 # min 512MB shm required
33 # 128MB per extra process
34 shm_per_process = 134217728
37 class StreamQueue(Queue):
42 sys.__stdout__.flush()
43 sys.__stderr__.flush()
46 return self._writer.fileno()
49 class StreamQueueManager(BaseManager):
53 StreamQueueManager.register('StreamQueue', StreamQueue)
56 class TestResult(dict):
57 def __init__(self, testcase_suite, testcases_by_id=None):
58 super(TestResult, self).__init__()
65 self.testcase_suite = testcase_suite
66 self.testcases = [testcase for testcase in testcase_suite]
67 self.testcases_by_id = testcases_by_id
69 def was_successful(self):
70 return 0 == len(self[FAIL]) == len(self[ERROR]) \
71 and len(self[PASS] + self[SKIP]) \
72 == self.testcase_suite.countTestCases() == len(self[TEST_RUN])
74 def no_tests_run(self):
75 return 0 == len(self[TEST_RUN])
77 def process_result(self, test_id, result):
78 self[result].append(test_id)
80 def suite_from_failed(self):
82 for testcase in self.testcase_suite:
84 if tc_id not in self[PASS] and tc_id not in self[SKIP]:
87 return suite_from_failed(self.testcase_suite, rerun_ids)
89 def get_testcase_names(self, test_id):
90 # could be tearDownClass (test_ipsec_esp.TestIpsecEsp1)
91 setup_teardown_match = re.match(
92 r'((tearDownClass)|(setUpClass)) \((.+\..+)\)', test_id)
93 if setup_teardown_match:
94 test_name, _, _, testcase_name = setup_teardown_match.groups()
95 if len(testcase_name.split('.')) == 2:
96 for key in self.testcases_by_id.keys():
97 if key.startswith(testcase_name):
100 testcase_name = self._get_testcase_doc_name(testcase_name)
102 test_name = self._get_test_description(test_id)
103 testcase_name = self._get_testcase_doc_name(test_id)
105 return testcase_name, test_name
107 def _get_test_description(self, test_id):
108 if test_id in self.testcases_by_id:
109 desc = get_test_description(descriptions,
110 self.testcases_by_id[test_id])
115 def _get_testcase_doc_name(self, test_id):
116 if test_id in self.testcases_by_id:
117 doc_name = get_testcase_doc_name(self.testcases_by_id[test_id])
123 def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue,
124 finished_pipe, result_pipe, logger):
125 sys.stdout = stdouterr_queue
126 sys.stderr = stdouterr_queue
127 VppTestCase.parallel_handler = logger.handlers[0]
128 result = VppTestRunner(keep_alive_pipe=keep_alive_pipe,
129 descriptions=descriptions,
131 result_pipe=result_pipe,
133 print_summary=False).run(suite)
134 finished_pipe.send(result.wasSuccessful())
135 finished_pipe.close()
136 keep_alive_pipe.close()
139 class TestCaseWrapper(object):
140 def __init__(self, testcase_suite, manager):
141 self.keep_alive_parent_end, self.keep_alive_child_end = Pipe(
143 self.finished_parent_end, self.finished_child_end = Pipe(duplex=False)
144 self.result_parent_end, self.result_child_end = Pipe(duplex=False)
145 self.testcase_suite = testcase_suite
146 if sys.version[0] == '2':
147 self.stdouterr_queue = manager.StreamQueue()
149 from multiprocessing import get_context
150 self.stdouterr_queue = manager.StreamQueue(ctx=get_context())
151 self.logger = get_parallel_logger(self.stdouterr_queue)
152 self.child = Process(target=test_runner_wrapper,
153 args=(testcase_suite,
154 self.keep_alive_child_end,
155 self.stdouterr_queue,
156 self.finished_child_end,
157 self.result_child_end,
161 self.last_test_temp_dir = None
162 self.last_test_vpp_binary = None
163 self._last_test = None
164 self.last_test_id = None
166 self.last_heard = time.time()
167 self.core_detected_at = None
168 self.testcases_by_id = {}
169 self.testclasess_with_core = {}
170 for testcase in self.testcase_suite:
171 self.testcases_by_id[testcase.id()] = testcase
172 self.result = TestResult(testcase_suite, self.testcases_by_id)
176 return self._last_test
179 def last_test(self, test_id):
180 self.last_test_id = test_id
181 if test_id in self.testcases_by_id:
182 testcase = self.testcases_by_id[test_id]
183 self._last_test = testcase.shortDescription()
184 if not self._last_test:
185 self._last_test = str(testcase)
187 self._last_test = test_id
189 def add_testclass_with_core(self):
190 if self.last_test_id in self.testcases_by_id:
191 test = self.testcases_by_id[self.last_test_id]
192 class_name = unittest.util.strclass(test.__class__)
193 test_name = "'{}' ({})".format(get_test_description(descriptions,
197 test_name = self.last_test_id
198 class_name = re.match(r'((tearDownClass)|(setUpClass)) '
199 r'\((.+\..+)\)', test_name).groups()[3]
200 if class_name not in self.testclasess_with_core:
201 self.testclasess_with_core[class_name] = (
203 self.last_test_vpp_binary,
204 self.last_test_temp_dir)
206 def close_pipes(self):
207 self.keep_alive_child_end.close()
208 self.finished_child_end.close()
209 self.result_child_end.close()
210 self.keep_alive_parent_end.close()
211 self.finished_parent_end.close()
212 self.result_parent_end.close()
214 def was_successful(self):
215 return self.result.was_successful()
218 def stdouterr_reader_wrapper(unread_testcases, finished_unread_testcases,
221 while read_testcases.is_set() or unread_testcases:
222 if finished_unread_testcases:
223 read_testcase = finished_unread_testcases.pop()
224 unread_testcases.remove(read_testcase)
225 elif unread_testcases:
226 read_testcase = unread_testcases.pop()
229 while data is not None:
230 sys.stdout.write(data)
231 data = read_testcase.stdouterr_queue.get()
233 read_testcase.stdouterr_queue.close()
234 finished_unread_testcases.discard(read_testcase)
238 def handle_failed_suite(logger, last_test_temp_dir, vpp_pid):
239 if last_test_temp_dir:
240 # Need to create link in case of a timeout or core dump without failure
241 lttd = os.path.basename(last_test_temp_dir)
242 failed_dir = os.getenv('FAILED_DIR')
243 link_path = '%s%s-FAILED' % (failed_dir, lttd)
244 if not os.path.exists(link_path):
245 os.symlink(last_test_temp_dir, link_path)
246 logger.error("Symlink to failed testcase directory: %s -> %s"
249 # Report core existence
250 core_path = get_core_path(last_test_temp_dir)
251 if os.path.exists(core_path):
253 "Core-file exists in test temporary directory: %s!" %
255 check_core_path(logger, core_path)
256 logger.debug("Running 'file %s':" % core_path)
258 info = check_output(["file", core_path])
260 except CalledProcessError as e:
261 logger.error("Subprocess returned with return code "
262 "while running `file' utility on core-file "
264 "rc=%s", e.returncode)
266 logger.error("Subprocess returned with OS error while "
267 "running 'file' utility "
269 "(%s) %s", e.errno, e.strerror)
270 except Exception as e:
271 logger.exception("Unexpected error running `file' utility "
275 # Copy api post mortem
276 api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
277 if os.path.isfile(api_post_mortem_path):
278 logger.error("Copying api_post_mortem.%d to %s" %
279 (vpp_pid, last_test_temp_dir))
280 shutil.copy2(api_post_mortem_path, last_test_temp_dir)
283 def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
284 if is_core_present(tempdir):
286 print('VPP core detected in %s. Last test running was %s' %
287 (tempdir, core_crash_test))
288 print(single_line_delim)
289 spawn_gdb(vpp_binary, get_core_path(tempdir))
290 print(single_line_delim)
292 print("Compressing core-file in test directory `%s'" % tempdir)
293 os.system("gzip %s" % get_core_path(tempdir))
296 def handle_cores(failed_testcases):
297 for failed_testcase in failed_testcases:
298 tcs_with_core = failed_testcase.testclasess_with_core
300 for test, vpp_binary, tempdir in tcs_with_core.values():
301 check_and_handle_core(vpp_binary, tempdir, test)
304 def process_finished_testsuite(wrapped_testcase_suite,
305 finished_testcase_suites,
306 failed_wrapped_testcases,
308 results.append(wrapped_testcase_suite.result)
309 finished_testcase_suites.add(wrapped_testcase_suite)
311 if failfast and not wrapped_testcase_suite.was_successful():
314 if not wrapped_testcase_suite.was_successful():
315 failed_wrapped_testcases.add(wrapped_testcase_suite)
316 handle_failed_suite(wrapped_testcase_suite.logger,
317 wrapped_testcase_suite.last_test_temp_dir,
318 wrapped_testcase_suite.vpp_pid)
323 def run_forked(testcase_suites):
324 wrapped_testcase_suites = set()
326 # suites are unhashable, need to use list
328 unread_testcases = set()
329 finished_unread_testcases = set()
330 manager = StreamQueueManager()
332 for i in range(concurrent_tests):
334 wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0),
336 wrapped_testcase_suites.add(wrapped_testcase_suite)
337 unread_testcases.add(wrapped_testcase_suite)
341 read_from_testcases = threading.Event()
342 read_from_testcases.set()
343 stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper,
344 args=(unread_testcases,
345 finished_unread_testcases,
346 read_from_testcases))
347 stdouterr_thread.start()
349 failed_wrapped_testcases = set()
353 while wrapped_testcase_suites:
354 finished_testcase_suites = set()
355 for wrapped_testcase_suite in wrapped_testcase_suites:
356 while wrapped_testcase_suite.result_parent_end.poll():
357 wrapped_testcase_suite.result.process_result(
358 *wrapped_testcase_suite.result_parent_end.recv())
359 wrapped_testcase_suite.last_heard = time.time()
361 while wrapped_testcase_suite.keep_alive_parent_end.poll():
362 wrapped_testcase_suite.last_test, \
363 wrapped_testcase_suite.last_test_vpp_binary, \
364 wrapped_testcase_suite.last_test_temp_dir, \
365 wrapped_testcase_suite.vpp_pid = \
366 wrapped_testcase_suite.keep_alive_parent_end.recv()
367 wrapped_testcase_suite.last_heard = time.time()
369 if wrapped_testcase_suite.finished_parent_end.poll():
370 wrapped_testcase_suite.finished_parent_end.recv()
371 wrapped_testcase_suite.last_heard = time.time()
372 stop_run = process_finished_testsuite(
373 wrapped_testcase_suite,
374 finished_testcase_suites,
375 failed_wrapped_testcases,
380 if wrapped_testcase_suite.last_heard + test_timeout < \
383 wrapped_testcase_suite.logger.critical(
384 "Child test runner process timed out "
385 "(last test running was `%s' in `%s')!" %
386 (wrapped_testcase_suite.last_test,
387 wrapped_testcase_suite.last_test_temp_dir))
388 elif not wrapped_testcase_suite.child.is_alive():
390 wrapped_testcase_suite.logger.critical(
391 "Child test runner process unexpectedly died "
392 "(last test running was `%s' in `%s')!" %
393 (wrapped_testcase_suite.last_test,
394 wrapped_testcase_suite.last_test_temp_dir))
395 elif wrapped_testcase_suite.last_test_temp_dir and \
396 wrapped_testcase_suite.last_test_vpp_binary:
398 wrapped_testcase_suite.last_test_temp_dir):
399 wrapped_testcase_suite.add_testclass_with_core()
400 if wrapped_testcase_suite.core_detected_at is None:
401 wrapped_testcase_suite.core_detected_at = \
403 elif wrapped_testcase_suite.core_detected_at + \
404 core_timeout < time.time():
405 wrapped_testcase_suite.logger.critical(
406 "Child test runner process unresponsive and "
407 "core-file exists in test temporary directory "
408 "(last test running was `%s' in `%s')!" %
409 (wrapped_testcase_suite.last_test,
410 wrapped_testcase_suite.last_test_temp_dir))
414 wrapped_testcase_suite.child.terminate()
416 # terminating the child process tends to leave orphan
418 if wrapped_testcase_suite.vpp_pid:
419 os.kill(wrapped_testcase_suite.vpp_pid,
424 wrapped_testcase_suite.result.crashed = True
425 wrapped_testcase_suite.result.process_result(
426 wrapped_testcase_suite.last_test_id, ERROR)
427 stop_run = process_finished_testsuite(
428 wrapped_testcase_suite,
429 finished_testcase_suites,
430 failed_wrapped_testcases,
433 for finished_testcase in finished_testcase_suites:
434 finished_testcase.child.join()
435 finished_testcase.close_pipes()
436 wrapped_testcase_suites.remove(finished_testcase)
437 finished_unread_testcases.add(finished_testcase)
438 finished_testcase.stdouterr_queue.put(None)
440 while testcase_suites:
441 results.append(TestResult(testcase_suites.pop(0)))
442 elif testcase_suites:
443 new_testcase = TestCaseWrapper(testcase_suites.pop(0),
445 wrapped_testcase_suites.add(new_testcase)
446 unread_testcases.add(new_testcase)
449 for wrapped_testcase_suite in wrapped_testcase_suites:
450 wrapped_testcase_suite.child.terminate()
451 wrapped_testcase_suite.stdouterr_queue.put(None)
454 read_from_testcases.clear()
455 stdouterr_thread.join(test_timeout)
458 handle_cores(failed_wrapped_testcases)
462 class SplitToSuitesCallback:
463 def __init__(self, filter_callback):
465 self.suite_name = 'default'
466 self.filter_callback = filter_callback
467 self.filtered = unittest.TestSuite()
469 def __call__(self, file_name, cls, method):
470 test_method = cls(method)
471 if self.filter_callback(file_name, cls.__name__, method):
472 self.suite_name = file_name + cls.__name__
473 if self.suite_name not in self.suites:
474 self.suites[self.suite_name] = unittest.TestSuite()
475 self.suites[self.suite_name].addTest(test_method)
478 self.filtered.addTest(test_method)
484 def parse_test_option():
485 f = os.getenv(test_option, None)
486 filter_file_name = None
487 filter_class_name = None
488 filter_func_name = None
493 raise Exception("Unrecognized %s option: %s" %
496 if parts[2] not in ('*', ''):
497 filter_func_name = parts[2]
498 if parts[1] not in ('*', ''):
499 filter_class_name = parts[1]
500 if parts[0] not in ('*', ''):
501 if parts[0].startswith('test_'):
502 filter_file_name = parts[0]
504 filter_file_name = 'test_%s' % parts[0]
506 if f.startswith('test_'):
509 filter_file_name = 'test_%s' % f
511 filter_file_name = '%s.py' % filter_file_name
512 return filter_file_name, filter_class_name, filter_func_name
515 def filter_tests(tests, filter_cb):
516 result = unittest.suite.TestSuite()
518 if isinstance(t, unittest.suite.TestSuite):
519 # this is a bunch of tests, recursively filter...
520 x = filter_tests(t, filter_cb)
521 if x.countTestCases() > 0:
523 elif isinstance(t, unittest.TestCase):
524 # this is a single test
525 parts = t.id().split('.')
526 # t.id() for common cases like this:
527 # test_classifier.TestClassifier.test_acl_ip
528 # apply filtering only if it is so
530 if not filter_cb(parts[0], parts[1], parts[2]):
534 # unexpected object, don't touch it
539 class FilterByTestOption:
540 def __init__(self, filter_file_name, filter_class_name, filter_func_name):
541 self.filter_file_name = filter_file_name
542 self.filter_class_name = filter_class_name
543 self.filter_func_name = filter_func_name
545 def __call__(self, file_name, class_name, func_name):
546 if self.filter_file_name:
547 fn_match = fnmatch.fnmatch(file_name, self.filter_file_name)
550 if self.filter_class_name and class_name != self.filter_class_name:
552 if self.filter_func_name and func_name != self.filter_func_name:
557 class FilterByClassList:
558 def __init__(self, classes_with_filenames):
559 self.classes_with_filenames = classes_with_filenames
561 def __call__(self, file_name, class_name, func_name):
562 return '.'.join([file_name, class_name]) in self.classes_with_filenames
565 def suite_from_failed(suite, failed):
566 failed = {x.rsplit('.', 1)[0] for x in failed}
567 filter_cb = FilterByClassList(failed)
568 suite = filter_tests(suite, filter_cb)
572 class AllResults(dict):
574 super(AllResults, self).__init__()
575 self.all_testcases = 0
576 self.results_per_suite = []
583 self.testsuites_no_tests_run = []
585 def add_results(self, result):
586 self.results_per_suite.append(result)
587 result_types = [PASS, FAIL, ERROR, SKIP, TEST_RUN]
588 for result_type in result_types:
589 self[result_type] += len(result[result_type])
591 def add_result(self, result):
593 self.all_testcases += result.testcase_suite.countTestCases()
594 self.add_results(result)
596 if result.no_tests_run():
597 self.testsuites_no_tests_run.append(result.testcase_suite)
602 elif not result.was_successful():
606 self.rerun.append(result.testcase_suite)
610 def print_results(self):
612 print(double_line_delim)
613 print('TEST RESULTS:')
614 print(' Scheduled tests: {}'.format(self.all_testcases))
615 print(' Executed tests: {}'.format(self[TEST_RUN]))
616 print(' Passed tests: {}'.format(
617 colorize(str(self[PASS]), GREEN)))
619 print(' Skipped tests: {}'.format(
620 colorize(str(self[SKIP]), YELLOW)))
621 if self.not_executed > 0:
622 print(' Not Executed tests: {}'.format(
623 colorize(str(self.not_executed), RED)))
625 print(' Failures: {}'.format(
626 colorize(str(self[FAIL]), RED)))
628 print(' Errors: {}'.format(
629 colorize(str(self[ERROR]), RED)))
631 if self.all_failed > 0:
632 print('FAILURES AND ERRORS IN TESTS:')
633 for result in self.results_per_suite:
634 failed_testcase_ids = result[FAIL]
635 errored_testcase_ids = result[ERROR]
636 old_testcase_name = None
637 if failed_testcase_ids or errored_testcase_ids:
638 for failed_test_id in failed_testcase_ids:
639 new_testcase_name, test_name = \
640 result.get_testcase_names(failed_test_id)
641 if new_testcase_name != old_testcase_name:
642 print(' Testcase name: {}'.format(
643 colorize(new_testcase_name, RED)))
644 old_testcase_name = new_testcase_name
645 print(' FAILURE: {} [{}]'.format(
646 colorize(test_name, RED), failed_test_id))
647 for failed_test_id in errored_testcase_ids:
648 new_testcase_name, test_name = \
649 result.get_testcase_names(failed_test_id)
650 if new_testcase_name != old_testcase_name:
651 print(' Testcase name: {}'.format(
652 colorize(new_testcase_name, RED)))
653 old_testcase_name = new_testcase_name
654 print(' ERROR: {} [{}]'.format(
655 colorize(test_name, RED), failed_test_id))
656 if self.testsuites_no_tests_run:
657 print('TESTCASES WHERE NO TESTS WERE SUCCESSFULLY EXECUTED:')
659 for testsuite in self.testsuites_no_tests_run:
660 for testcase in testsuite:
661 tc_classes.add(get_testcase_doc_name(testcase))
662 for tc_class in tc_classes:
663 print(' {}'.format(colorize(tc_class, RED)))
665 print(double_line_delim)
669 def not_executed(self):
670 return self.all_testcases - self[TEST_RUN]
673 def all_failed(self):
674 return self[FAIL] + self[ERROR]
677 def parse_results(results):
679 Prints the number of scheduled, executed, not executed, passed, failed,
680 errored and skipped tests and details about failed and errored tests.
682 Also returns all suites where any test failed.
688 results_per_suite = AllResults()
691 for result in results:
692 result_code = results_per_suite.add_result(result)
695 elif result_code == -1:
698 results_per_suite.print_results()
706 return return_code, results_per_suite.rerun
709 def parse_digit_env(env_var, default):
710 value = os.getenv(env_var, default)
715 print('WARNING: unsupported value "%s" for env var "%s",'
716 'defaulting to %s' % (value, env_var, default))
721 if __name__ == '__main__':
723 verbose = parse_digit_env("V", 0)
725 test_timeout = parse_digit_env("TIMEOUT", 600) # default = 10 minutes
727 retries = parse_digit_env("RETRIES", 0)
729 debug = os.getenv("DEBUG", "n").lower() in ["gdb", "gdbserver"]
731 debug_core = os.getenv("DEBUG", "").lower() == "core"
732 compress_core = os.getenv("CORE_COMPRESS", "").lower() in ("y", "yes", "1")
734 step = os.getenv("STEP", "n").lower() in ("y", "yes", "1")
736 run_interactive = debug or step
738 test_jobs = os.getenv("TEST_JOBS", "1").lower() # default = 1 process
739 if test_jobs == 'auto':
742 print('Interactive mode required, running on one core')
744 shm_free = psutil.disk_usage('/dev/shm').free
745 shm_max_processes = 1
746 if shm_free < min_req_shm:
747 raise Exception('Not enough free space in /dev/shm. Required '
748 'free space is at least %sM.'
749 % (min_req_shm >> 20))
751 extra_shm = shm_free - min_req_shm
752 shm_max_processes += extra_shm / shm_per_process
753 concurrent_tests = min(cpu_count(), shm_max_processes)
754 print('Found enough resources to run tests with %s cores'
756 elif test_jobs.isdigit():
757 concurrent_tests = int(test_jobs)
761 if run_interactive and concurrent_tests > 1:
762 raise NotImplementedError(
763 'Running tests interactively (DEBUG is gdb or gdbserver or STEP '
764 'is set) in parallel (TEST_JOBS is more than 1) is not supported')
766 parser = argparse.ArgumentParser(description="VPP unit tests")
767 parser.add_argument("-f", "--failfast", action='store_true',
768 help="fast failure flag")
769 parser.add_argument("-d", "--dir", action='append', type=str,
770 help="directory containing test files "
771 "(may be specified multiple times)")
772 args = parser.parse_args()
773 failfast = args.failfast
776 print("Running tests using custom test runner") # debug message
777 filter_file, filter_class, filter_func = parse_test_option()
779 print("Active filters: file=%s, class=%s, function=%s" % (
780 filter_file, filter_class, filter_func))
782 filter_cb = FilterByTestOption(filter_file, filter_class, filter_func)
784 ignore_path = os.getenv("VENV_PATH", None)
785 cb = SplitToSuitesCallback(filter_cb)
787 print("Adding tests from directory tree %s" % d)
788 discover_tests(d, cb, ignore_path)
790 # suites are not hashable, need to use list
793 for testcase_suite in cb.suites.values():
794 tests_amount += testcase_suite.countTestCases()
795 suites.append(testcase_suite)
797 print("%s out of %s tests match specified filters" % (
798 tests_amount, tests_amount + cb.filtered.countTestCases()))
800 if not running_extended_tests:
801 print("Not running extended tests (some tests will be skipped)")
803 attempts = retries + 1
805 print("Perform %s attempts to pass the suite..." % attempts)
807 if run_interactive and suites:
808 # don't fork if requiring interactive terminal
809 full_suite = unittest.TestSuite()
810 map(full_suite.addTests, suites)
811 result = VppTestRunner(verbosity=verbose,
813 print_summary=True).run(full_suite)
814 was_successful = result.wasSuccessful()
815 if not was_successful:
816 for test_case_info in result.failed_test_cases_info:
817 handle_failed_suite(test_case_info.logger,
818 test_case_info.tempdir,
819 test_case_info.vpp_pid)
820 if test_case_info in result.core_crash_test_cases_info:
821 check_and_handle_core(test_case_info.vpp_bin_path,
822 test_case_info.tempdir,
823 test_case_info.core_crash_test)
825 sys.exit(not was_successful)
828 while suites and attempts > 0:
829 results = run_forked(suites)
830 exit_code, suites = parse_results(results)
833 print('Test run was successful')
835 print('%s attempt(s) left.' % attempts)