+class FilterByTestOption:
+ def __init__(self, filter_file_name, filter_class_name, filter_func_name):
+ self.filter_file_name = filter_file_name
+ self.filter_class_name = filter_class_name
+ self.filter_func_name = filter_func_name
+
+ def __call__(self, file_name, class_name, func_name):
+ if self.filter_file_name and file_name != self.filter_file_name:
+ return False
+ if self.filter_class_name and class_name != self.filter_class_name:
+ return False
+ if self.filter_func_name and func_name != self.filter_func_name:
+ return False
+ return True
+
+
+class FilterByClassList:
+ def __init__(self, classes_with_filenames):
+ self.classes_with_filenames = classes_with_filenames
+
+ def __call__(self, file_name, class_name, func_name):
+ return '.'.join([file_name, class_name]) in self.classes_with_filenames
+
+
+def suite_from_failed(suite, failed):
+ failed = {x.rsplit('.', 1)[0] for x in failed}
+ filter_cb = FilterByClassList(failed)
+ suite = filter_tests(suite, filter_cb)
+ return suite
+
+
+class NonPassedResults(dict):
+ def __init__(self):
+ super(NonPassedResults, self).__init__()
+ self.all_testcases = 0
+ self.results_per_suite = {}
+ self.failures_id = 'failures'
+ self.errors_id = 'errors'
+ self.crashes_id = 'crashes'
+ self.skipped_id = 'skipped'
+ self.expectedFailures_id = 'expectedFailures'
+ self.unexpectedSuccesses_id = 'unexpectedSuccesses'
+ self.rerun = []
+ self.passed = 0
+ self[self.failures_id] = 0
+ self[self.errors_id] = 0
+ self[self.skipped_id] = 0
+ self[self.expectedFailures_id] = 0
+ self[self.unexpectedSuccesses_id] = 0
+
+ def _add_result(self, test, result_id):
+ if isinstance(test, VppTestCase):
+ parts = test.id().split('.')
+ if len(parts) == 3:
+ tc_class = get_testcase_doc_name(test)
+ if tc_class not in self.results_per_suite:
+ # failed, errored, skipped, expectedly failed,
+ # unexpectedly passed
+ self.results_per_suite[tc_class] = \
+ {self.failures_id: [],
+ self.errors_id: [],
+ self.skipped_id: [],
+ self.expectedFailures_id: [],
+ self.unexpectedSuccesses_id: []}
+ self.results_per_suite[tc_class][result_id].append(test)
+ return True
+ return False
+
+ def add_results(self, testcases, testcase_result_id):
+ for failed_testcase, _ in testcases:
+ if self._add_result(failed_testcase, testcase_result_id):
+ self[testcase_result_id] += 1
+
+ def add_result(self, testcase_suite, result):
+ retval = 0
+ if result:
+ self.all_testcases += result.testsRun
+ self.passed += len(result.passed)
+ if not len(result.passed) + len(result.skipped) \
+ == testcase_suite.countTestCases():
+ retval = 1
+
+ self.add_results(result.failures, self.failures_id)
+ self.add_results(result.errors, self.errors_id)
+ self.add_results(result.skipped, self.skipped_id)
+ self.add_results(result.expectedFailures,
+ self.expectedFailures_id)
+ self.add_results(result.unexpectedSuccesses,
+ self.unexpectedSuccesses_id)
+ else:
+ retval = -1
+
+ if retval != 0:
+ if concurrent_tests == 1:
+ if result:
+ rerun_ids = set([])
+ skipped = [x.id() for (x, _) in result.skipped]
+ for testcase in testcase_suite:
+ tc_id = testcase.id()
+ if tc_id not in result.passed and \
+ tc_id not in skipped:
+ rerun_ids.add(tc_id)
+ if len(rerun_ids) > 0:
+ self.rerun.append(suite_from_failed(testcase_suite,
+ rerun_ids))
+ else:
+ self.rerun.append(testcase_suite)
+ else:
+ self.rerun.append(testcase_suite)
+
+ return retval
+
+ def print_results(self):
+ print('')
+ print(double_line_delim)
+ print('TEST RESULTS:')
+ print(' Executed tests: {}'.format(self.all_testcases))
+ print(' Passed tests: {}'.format(
+ colorize(str(self.passed), GREEN)))
+ if self[self.failures_id] > 0:
+ print(' Failures: {}'.format(
+ colorize(str(self[self.failures_id]), RED)))
+ if self[self.errors_id] > 0:
+ print(' Errors: {}'.format(
+ colorize(str(self[self.errors_id]), RED)))
+ if self[self.skipped_id] > 0:
+ print(' Skipped tests: {}'.format(
+ colorize(str(self[self.skipped_id]), YELLOW)))
+ if self[self.expectedFailures_id] > 0:
+ print(' Expected failures: {}'.format(
+ colorize(str(self[self.expectedFailures_id]), GREEN)))
+ if self[self.unexpectedSuccesses_id] > 0:
+ print(' Unexpected successes: {}'.format(
+ colorize(str(self[self.unexpectedSuccesses_id]), YELLOW)))
+
+ if self.all_failed > 0:
+ print('FAILED TESTS:')
+ for testcase_class, suite_results in \
+ self.results_per_suite.items():
+ failed_testcases = suite_results[
+ self.failures_id]
+ errored_testcases = suite_results[
+ self.errors_id]
+ if len(failed_testcases) or len(errored_testcases):
+ print(' Testcase name: {}'.format(
+ colorize(testcase_class, RED)))
+ for failed_test in failed_testcases:
+ print(' FAILED: {}'.format(
+ colorize(get_test_description(
+ descriptions, failed_test), RED)))
+ for failed_test in errored_testcases:
+ print(' ERRORED: {}'.format(
+ colorize(get_test_description(
+ descriptions, failed_test), RED)))
+
+ print(double_line_delim)
+ print('')
+
+ @property
+ def all_failed(self):
+ return self[self.failures_id] + self[self.errors_id]
+
+
+def parse_results(results):
+ """
+ Prints the number of executed, passed, failed, errored, skipped,
+ expectedly failed and unexpectedly passed tests and details about
+ failed, errored, expectedly failed and unexpectedly passed tests.
+
+ Also returns any suites where any test failed.
+
+ :param results:
+ :return:
+ """
+
+ results_per_suite = NonPassedResults()
+ crashed = False
+ failed = False
+ for testcase_suite, result in results:
+ result_code = results_per_suite.add_result(testcase_suite, result)
+ if result_code == 1:
+ failed = True
+ elif result_code == -1:
+ crashed = True
+
+ results_per_suite.print_results()
+
+ if crashed:
+ return_code = -1
+ elif failed:
+ return_code = 1
+ else:
+ return_code = 0
+ return return_code, results_per_suite.rerun
+
+
+def parse_digit_env(env_var, default):
+ value = os.getenv(env_var, default)
+ if value != default:
+ if value.isdigit():
+ value = int(value)
+ else:
+ print('WARNING: unsupported value "%s" for env var "%s",'
+ 'defaulting to %s' % (value, env_var, default))
+ value = default
+ return value
+
+