X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=test%2Frun_tests.py;h=cbca7f96e396f1dc04b4ba8bb1732fa3d535d992;hb=e88865d;hp=c42c6a820dc1357191d7bffeb7ca0f57e0c25477;hpb=40dd73bcfa7625773e1e0cc049134f9d7107bccc;p=vpp.git diff --git a/test/run_tests.py b/test/run_tests.py index c42c6a820dc..cbca7f96e39 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -68,8 +68,8 @@ class TestResult(dict): def was_successful(self): return 0 == len(self[FAIL]) == len(self[ERROR]) \ - and len(self[PASS] + self[SKIP]) \ - == self.testcase_suite.countTestCases() == len(self[TEST_RUN]) + and len(self[PASS] + self[SKIP]) \ + == self.testcase_suite.countTestCases() == len(self[TEST_RUN]) def no_tests_run(self): return 0 == len(self[TEST_RUN]) @@ -120,12 +120,13 @@ def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue, finished_pipe, result_pipe, logger): sys.stdout = stdouterr_queue sys.stderr = stdouterr_queue - VppTestCase.logger = logger + VppTestCase.parallel_handler = logger.handlers[0] result = VppTestRunner(keep_alive_pipe=keep_alive_pipe, descriptions=descriptions, verbosity=verbose, result_pipe=result_pipe, - failfast=failfast).run(suite) + failfast=failfast, + print_summary=False).run(suite) finished_pipe.send(result.wasSuccessful()) finished_pipe.close() keep_alive_pipe.close() @@ -209,13 +210,12 @@ class TestCaseWrapper(object): def stdouterr_reader_wrapper(unread_testcases, finished_unread_testcases, read_testcases): read_testcase = None - while read_testcases.is_set() or len(unread_testcases) > 0: - if not read_testcase: - if len(finished_unread_testcases) > 0: - read_testcase = finished_unread_testcases.pop() - unread_testcases.remove(read_testcase) - elif len(unread_testcases) > 0: - read_testcase = unread_testcases.pop() + while read_testcases.is_set() or len(unread_testcases): + if len(finished_unread_testcases): + read_testcase = finished_unread_testcases.pop() + unread_testcases.remove(read_testcase) + elif len(unread_testcases): + read_testcase = unread_testcases.pop() if read_testcase: data = '' while data is not None: @@ -231,15 +231,12 @@ def handle_failed_suite(logger, last_test_temp_dir, vpp_pid): if last_test_temp_dir: # Need to create link in case of a timeout or core dump without failure lttd = os.path.basename(last_test_temp_dir) - failed_dir = os.getenv('VPP_TEST_FAILED_DIR') + failed_dir = os.getenv('FAILED_DIR') link_path = '%s%s-FAILED' % (failed_dir, lttd) if not os.path.exists(link_path): - logger.error("Creating a link to the failed test: %s -> %s" % - (link_path, lttd)) os.symlink(last_test_temp_dir, link_path) - else: - logger.error("Link to the failed test already exists: %s -> %s" % - (link_path, lttd)) + logger.error("Symlink to failed testcase directory: %s -> %s" + % (link_path, lttd)) # Report core existence core_path = get_core_path(last_test_temp_dir) @@ -330,103 +327,112 @@ def run_forked(testcase_suites): failed_wrapped_testcases = set() stop_run = False - while len(wrapped_testcase_suites) > 0: - finished_testcase_suites = set() + + try: + while len(wrapped_testcase_suites) > 0: + finished_testcase_suites = set() + for wrapped_testcase_suite in wrapped_testcase_suites: + while wrapped_testcase_suite.result_parent_end.poll(): + wrapped_testcase_suite.result.process_result( + *wrapped_testcase_suite.result_parent_end.recv()) + wrapped_testcase_suite.last_heard = time.time() + + while wrapped_testcase_suite.keep_alive_parent_end.poll(): + wrapped_testcase_suite.last_test, \ + wrapped_testcase_suite.last_test_vpp_binary, \ + wrapped_testcase_suite.last_test_temp_dir, \ + wrapped_testcase_suite.vpp_pid = \ + wrapped_testcase_suite.keep_alive_parent_end.recv() + wrapped_testcase_suite.last_heard = time.time() + + if wrapped_testcase_suite.finished_parent_end.poll(): + wrapped_testcase_suite.finished_parent_end.recv() + wrapped_testcase_suite.last_heard = time.time() + stop_run = process_finished_testsuite( + wrapped_testcase_suite, + finished_testcase_suites, + failed_wrapped_testcases, + results) or stop_run + continue + + fail = False + if wrapped_testcase_suite.last_heard + test_timeout < \ + time.time(): + fail = True + wrapped_testcase_suite.logger.critical( + "Child test runner process timed out " + "(last test running was `%s' in `%s')!" % + (wrapped_testcase_suite.last_test, + wrapped_testcase_suite.last_test_temp_dir)) + elif not wrapped_testcase_suite.child.is_alive(): + fail = True + wrapped_testcase_suite.logger.critical( + "Child test runner process unexpectedly died " + "(last test running was `%s' in `%s')!" % + (wrapped_testcase_suite.last_test, + wrapped_testcase_suite.last_test_temp_dir)) + elif wrapped_testcase_suite.last_test_temp_dir and \ + wrapped_testcase_suite.last_test_vpp_binary: + if is_core_present( + wrapped_testcase_suite.last_test_temp_dir): + wrapped_testcase_suite.add_testclass_with_core() + if wrapped_testcase_suite.core_detected_at is None: + wrapped_testcase_suite.core_detected_at = \ + time.time() + elif wrapped_testcase_suite.core_detected_at + \ + core_timeout < time.time(): + wrapped_testcase_suite.logger.critical( + "Child test runner process unresponsive and " + "core-file exists in test temporary directory " + "(last test running was `%s' in `%s')!" % + (wrapped_testcase_suite.last_test, + wrapped_testcase_suite.last_test_temp_dir)) + fail = True + + if fail: + wrapped_testcase_suite.child.terminate() + try: + # terminating the child process tends to leave orphan + # VPP process around + if wrapped_testcase_suite.vpp_pid: + os.kill(wrapped_testcase_suite.vpp_pid, + signal.SIGTERM) + except OSError: + # already dead + pass + wrapped_testcase_suite.result.crashed = True + wrapped_testcase_suite.result.process_result( + wrapped_testcase_suite.last_test_id, ERROR) + stop_run = process_finished_testsuite( + wrapped_testcase_suite, + finished_testcase_suites, + failed_wrapped_testcases, + results) or stop_run + + for finished_testcase in finished_testcase_suites: + finished_testcase.child.join() + finished_testcase.close_pipes() + wrapped_testcase_suites.remove(finished_testcase) + finished_unread_testcases.add(finished_testcase) + finished_testcase.stdouterr_queue.put(None) + if stop_run: + while len(testcase_suites) > 0: + results.append(TestResult(testcase_suites.pop(0))) + elif len(testcase_suites) > 0: + new_testcase = TestCaseWrapper(testcase_suites.pop(0), + manager) + wrapped_testcase_suites.add(new_testcase) + unread_testcases.add(new_testcase) + except Exception: for wrapped_testcase_suite in wrapped_testcase_suites: - while wrapped_testcase_suite.result_parent_end.poll(): - wrapped_testcase_suite.result.process_result( - *wrapped_testcase_suite.result_parent_end.recv()) - wrapped_testcase_suite.last_heard = time.time() - - while wrapped_testcase_suite.keep_alive_parent_end.poll(): - wrapped_testcase_suite.last_test, \ - wrapped_testcase_suite.last_test_vpp_binary, \ - wrapped_testcase_suite.last_test_temp_dir, \ - wrapped_testcase_suite.vpp_pid = \ - wrapped_testcase_suite.keep_alive_parent_end.recv() - wrapped_testcase_suite.last_heard = time.time() - - if wrapped_testcase_suite.finished_parent_end.poll(): - wrapped_testcase_suite.finished_parent_end.recv() - wrapped_testcase_suite.last_heard = time.time() - stop_run = process_finished_testsuite( - wrapped_testcase_suite, - finished_testcase_suites, - failed_wrapped_testcases, - results) or stop_run - continue - - fail = False - if wrapped_testcase_suite.last_heard + test_timeout < time.time(): - fail = True - wrapped_testcase_suite.logger.critical( - "Child test runner process timed out " - "(last test running was `%s' in `%s')!" % - (wrapped_testcase_suite.last_test, - wrapped_testcase_suite.last_test_temp_dir)) - elif not wrapped_testcase_suite.child.is_alive(): - fail = True - wrapped_testcase_suite.logger.critical( - "Child test runner process unexpectedly died " - "(last test running was `%s' in `%s')!" % - (wrapped_testcase_suite.last_test, - wrapped_testcase_suite.last_test_temp_dir)) - elif wrapped_testcase_suite.last_test_temp_dir and \ - wrapped_testcase_suite.last_test_vpp_binary: - if is_core_present(wrapped_testcase_suite.last_test_temp_dir): - wrapped_testcase_suite.add_testclass_with_core() - if wrapped_testcase_suite.core_detected_at is None: - wrapped_testcase_suite.core_detected_at = time.time() - elif wrapped_testcase_suite.core_detected_at + \ - core_timeout < time.time(): - wrapped_testcase_suite.logger.critical( - "Child test runner process unresponsive and core-" - "file exists in test temporary directory " - "(last test running was `%s' in `%s')!" % - (wrapped_testcase_suite.last_test, - wrapped_testcase_suite.last_test_temp_dir)) - fail = True - - if fail: - wrapped_testcase_suite.child.terminate() - try: - # terminating the child process tends to leave orphan - # VPP process around - if wrapped_testcase_suite.vpp_pid: - os.kill(wrapped_testcase_suite.vpp_pid, signal.SIGTERM) - except OSError: - # already dead - pass - wrapped_testcase_suite.result.crashed = True - wrapped_testcase_suite.result.process_result( - wrapped_testcase_suite.last_test_id, ERROR) - stop_run = process_finished_testsuite( - wrapped_testcase_suite, - finished_testcase_suites, - failed_wrapped_testcases, - results) or stop_run - - for finished_testcase in finished_testcase_suites: - finished_testcase.child.join() - finished_testcase.close_pipes() - wrapped_testcase_suites.remove(finished_testcase) - finished_unread_testcases.add(finished_testcase) - finished_testcase.stdouterr_queue.put(None) - if stop_run: - while len(testcase_suites) > 0: - results.append(TestResult(testcase_suites.pop(0))) - elif len(testcase_suites) > 0: - new_testcase = TestCaseWrapper(testcase_suites.pop(0), manager) - wrapped_testcase_suites.add(new_testcase) - unread_testcases.add(new_testcase) - - while len(unread_testcases) > 0: - # wait for reader thread to read everything in all loggers - pass - - read_from_testcases.clear() - stdouterr_thread.join(test_timeout) - manager.shutdown() + wrapped_testcase_suite.child.terminate() + wrapped_testcase_suite.stdouterr_queue.put(None) + raise + finally: + read_from_testcases.clear() + stdouterr_thread.join(test_timeout) + manager.shutdown() + handle_cores(failed_wrapped_testcases) return results @@ -575,10 +581,7 @@ class AllResults(dict): retval = 1 if retval != 0: - if concurrent_tests == 1: - self.rerun.append(result.suite_from_failed()) - else: - self.rerun.append(result.testcase_suite) + self.rerun.append(result.testcase_suite) return retval @@ -617,8 +620,8 @@ class AllResults(dict): print(' Testcase name: {}'.format( colorize(new_testcase_name, RED))) old_testcase_name = new_testcase_name - print(' FAILURE: {}'.format( - colorize(test_name, RED))) + print(' FAILURE: {} [{}]'.format( + colorize(test_name, RED), failed_test_id)) for failed_test_id in errored_testcase_ids: new_testcase_name, test_name = \ result.get_testcase_names(failed_test_id) @@ -626,8 +629,8 @@ class AllResults(dict): print(' Testcase name: {}'.format( colorize(new_testcase_name, RED))) old_testcase_name = new_testcase_name - print(' ERROR: {}'.format( - colorize(test_name, RED))) + print(' ERROR: {} [{}]'.format( + colorize(test_name, RED), failed_test_id)) if len(self.testsuites_no_tests_run) > 0: print('TESTCASES WHERE NO TESTS WERE SUCCESSFULLY EXECUTED:') tc_classes = set() @@ -755,10 +758,11 @@ if __name__ == '__main__': filter_cb = FilterByTestOption(filter_file, filter_class, filter_func) + ignore_path = os.getenv("VENV_PATH", None) cb = SplitToSuitesCallback(filter_cb) for d in args.dir: print("Adding tests from directory tree %s" % d) - discover_tests(d, cb) + discover_tests(d, cb, ignore_path) # suites are not hashable, need to use list suites = [] @@ -767,13 +771,6 @@ if __name__ == '__main__': tests_amount += testcase_suite.countTestCases() suites.append(testcase_suite) - if concurrent_tests == 1: - new_suite = unittest.TestSuite() - for suite in suites: - new_suite.addTests(suite) - - suites = [new_suite] - print("%s out of %s tests match specified filters" % ( tests_amount, tests_amount + cb.filtered.countTestCases())) @@ -786,8 +783,9 @@ if __name__ == '__main__': if run_interactive: # don't fork if requiring interactive terminal - result = VppTestRunner(verbosity=verbose, failfast=failfast)\ - .run(suites[0]) + result = VppTestRunner(verbosity=verbose, + failfast=failfast, + print_summary=True).run(suites[0]) was_successful = result.wasSuccessful() if not was_successful: for test_case_info in result.failed_test_cases_info: