test/run_tests.py

   1 #!/usr/bin/env python
   2
   3 import sys
   4 import shutil
   5 import os
   6 import fnmatch
   7 import unittest
   8 import argparse
   9 import time
  10 import threading
  11 import signal
  12 import psutil
  13 import re
  14 from multiprocessing import Process, Pipe, cpu_count
  15 from multiprocessing.queues import Queue
  16 from multiprocessing.managers import BaseManager
  17 from framework import VppTestRunner, running_extended_tests, VppTestCase, \
  18     get_testcase_doc_name, get_test_description, PASS, FAIL, ERROR, SKIP, \
  19     TEST_RUN
  20 from debug import spawn_gdb
  21 from log import get_parallel_logger, double_line_delim, RED, YELLOW, GREEN, \
  22     colorize, single_line_delim
  23 from discover_tests import discover_tests
  24 from subprocess import check_output, CalledProcessError
  25 from util import check_core_path, get_core_path, is_core_present
  26
  27 # timeout which controls how long the child has to finish after seeing
  28 # a core dump in test temporary directory. If this is exceeded, parent assumes
  29 # that child process is stuck (e.g. waiting for shm mutex, which will never
  30 # get unlocked) and kill the child
  31 core_timeout = 3
  32 min_req_shm = 536870912  # min 512MB shm required
  33 # 128MB per extra process
  34 shm_per_process = 134217728
  35
  36
  37 class StreamQueue(Queue):
  38     def write(self, msg):
  39         self.put(msg)
  40
  41     def flush(self):
  42         sys.__stdout__.flush()
  43         sys.__stderr__.flush()
  44
  45     def fileno(self):
  46         return self._writer.fileno()
  47
  48
  49 class StreamQueueManager(BaseManager):
  50     pass
  51
  52
  53 StreamQueueManager.register('StreamQueue', StreamQueue)
  54
  55
  56 class TestResult(dict):
  57     def __init__(self, testcase_suite, testcases_by_id=None):
  58         super(TestResult, self).__init__()
  59         self[PASS] = []
  60         self[FAIL] = []
  61         self[ERROR] = []
  62         self[SKIP] = []
  63         self[TEST_RUN] = []
  64         self.crashed = False
  65         self.testcase_suite = testcase_suite
  66         self.testcases = [testcase for testcase in testcase_suite]
  67         self.testcases_by_id = testcases_by_id
  68
  69     def was_successful(self):
  70         return 0 == len(self[FAIL]) == len(self[ERROR]) \
  71             and len(self[PASS] + self[SKIP]) \
  72             == self.testcase_suite.countTestCases() == len(self[TEST_RUN])
  73
  74     def no_tests_run(self):
  75         return 0 == len(self[TEST_RUN])
  76
  77     def process_result(self, test_id, result):
  78         self[result].append(test_id)
  79
  80     def suite_from_failed(self):
  81         rerun_ids = set([])
  82         for testcase in self.testcase_suite:
  83             tc_id = testcase.id()
  84             if tc_id not in self[PASS] and tc_id not in self[SKIP]:
  85                 rerun_ids.add(tc_id)
  86         if rerun_ids:
  87             return suite_from_failed(self.testcase_suite, rerun_ids)
  88
  89     def get_testcase_names(self, test_id):
  90         # could be tearDownClass (test_ipsec_esp.TestIpsecEsp1)
  91         setup_teardown_match = re.match(
  92             r'((tearDownClass)|(setUpClass)) \((.+\..+)\)', test_id)
  93         if setup_teardown_match:
  94             test_name, _, _, testcase_name = setup_teardown_match.groups()
  95             if len(testcase_name.split('.')) == 2:
  96                 for key in self.testcases_by_id.keys():
  97                     if key.startswith(testcase_name):
  98                         testcase_name = key
  99                         break
 100             testcase_name = self._get_testcase_doc_name(testcase_name)
 101         else:
 102             test_name = self._get_test_description(test_id)
 103             testcase_name = self._get_testcase_doc_name(test_id)
 104
 105         return testcase_name, test_name
 106
 107     def _get_test_description(self, test_id):
 108         if test_id in self.testcases_by_id:
 109             desc = get_test_description(descriptions,
 110                                         self.testcases_by_id[test_id])
 111         else:
 112             desc = test_id
 113         return desc
 114
 115     def _get_testcase_doc_name(self, test_id):
 116         if test_id in self.testcases_by_id:
 117             doc_name = get_testcase_doc_name(self.testcases_by_id[test_id])
 118         else:
 119             doc_name = test_id
 120         return doc_name
 121
 122
 123 def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue,
 124                         finished_pipe, result_pipe, logger):
 125     sys.stdout = stdouterr_queue
 126     sys.stderr = stdouterr_queue
 127     VppTestCase.parallel_handler = logger.handlers[0]
 128     result = VppTestRunner(keep_alive_pipe=keep_alive_pipe,
 129                            descriptions=descriptions,
 130                            verbosity=verbose,
 131                            result_pipe=result_pipe,
 132                            failfast=failfast,
 133                            print_summary=False).run(suite)
 134     finished_pipe.send(result.wasSuccessful())
 135     finished_pipe.close()
 136     keep_alive_pipe.close()
 137
 138
 139 class TestCaseWrapper(object):
 140     def __init__(self, testcase_suite, manager):
 141         self.keep_alive_parent_end, self.keep_alive_child_end = Pipe(
 142             duplex=False)
 143         self.finished_parent_end, self.finished_child_end = Pipe(duplex=False)
 144         self.result_parent_end, self.result_child_end = Pipe(duplex=False)
 145         self.testcase_suite = testcase_suite
 146         if sys.version[0] == '2':
 147             self.stdouterr_queue = manager.StreamQueue()
 148         else:
 149             from multiprocessing import get_context
 150             self.stdouterr_queue = manager.StreamQueue(ctx=get_context())
 151         self.logger = get_parallel_logger(self.stdouterr_queue)
 152         self.child = Process(target=test_runner_wrapper,
 153                              args=(testcase_suite,
 154                                    self.keep_alive_child_end,
 155                                    self.stdouterr_queue,
 156                                    self.finished_child_end,
 157                                    self.result_child_end,
 158                                    self.logger)
 159                              )
 160         self.child.start()
 161         self.last_test_temp_dir = None
 162         self.last_test_vpp_binary = None
 163         self._last_test = None
 164         self.last_test_id = None
 165         self.vpp_pid = None
 166         self.last_heard = time.time()
 167         self.core_detected_at = None
 168         self.testcases_by_id = {}
 169         self.testclasess_with_core = {}
 170         for testcase in self.testcase_suite:
 171             self.testcases_by_id[testcase.id()] = testcase
 172         self.result = TestResult(testcase_suite, self.testcases_by_id)
 173
 174     @property
 175     def last_test(self):
 176         return self._last_test
 177
 178     @last_test.setter
 179     def last_test(self, test_id):
 180         self.last_test_id = test_id
 181         if test_id in self.testcases_by_id:
 182             testcase = self.testcases_by_id[test_id]
 183             self._last_test = testcase.shortDescription()
 184             if not self._last_test:
 185                 self._last_test = str(testcase)
 186         else:
 187             self._last_test = test_id
 188
 189     def add_testclass_with_core(self):
 190         if self.last_test_id in self.testcases_by_id:
 191             test = self.testcases_by_id[self.last_test_id]
 192             class_name = unittest.util.strclass(test.__class__)
 193             test_name = "'{}' ({})".format(get_test_description(descriptions,
 194                                                                 test),
 195                                            self.last_test_id)
 196         else:
 197             test_name = self.last_test_id
 198             class_name = re.match(r'((tearDownClass)|(setUpClass)) '
 199                                   r'\((.+\..+)\)', test_name).groups()[3]
 200         if class_name not in self.testclasess_with_core:
 201             self.testclasess_with_core[class_name] = (
 202                 test_name,
 203                 self.last_test_vpp_binary,
 204                 self.last_test_temp_dir)
 205
 206     def close_pipes(self):
 207         self.keep_alive_child_end.close()
 208         self.finished_child_end.close()
 209         self.result_child_end.close()
 210         self.keep_alive_parent_end.close()
 211         self.finished_parent_end.close()
 212         self.result_parent_end.close()
 213
 214     def was_successful(self):
 215         return self.result.was_successful()
 216
 217
 218 def stdouterr_reader_wrapper(unread_testcases, finished_unread_testcases,
 219                              read_testcases):
 220     read_testcase = None
 221     while read_testcases.is_set() or unread_testcases:
 222         if finished_unread_testcases:
 223             read_testcase = finished_unread_testcases.pop()
 224             unread_testcases.remove(read_testcase)
 225         elif unread_testcases:
 226             read_testcase = unread_testcases.pop()
 227         if read_testcase:
 228             data = ''
 229             while data is not None:
 230                 sys.stdout.write(data)
 231                 data = read_testcase.stdouterr_queue.get()
 232
 233             read_testcase.stdouterr_queue.close()
 234             finished_unread_testcases.discard(read_testcase)
 235             read_testcase = None
 236
 237
 238 def handle_failed_suite(logger, last_test_temp_dir, vpp_pid):
 239     if last_test_temp_dir:
 240         # Need to create link in case of a timeout or core dump without failure
 241         lttd = os.path.basename(last_test_temp_dir)
 242         failed_dir = os.getenv('FAILED_DIR')
 243         link_path = '%s%s-FAILED' % (failed_dir, lttd)
 244         if not os.path.exists(link_path):
 245             os.symlink(last_test_temp_dir, link_path)
 246         logger.error("Symlink to failed testcase directory: %s -> %s"
 247                      % (link_path, lttd))
 248
 249         # Report core existence
 250         core_path = get_core_path(last_test_temp_dir)
 251         if os.path.exists(core_path):
 252             logger.error(
 253                 "Core-file exists in test temporary directory: %s!" %
 254                 core_path)
 255             check_core_path(logger, core_path)
 256             logger.debug("Running 'file %s':" % core_path)
 257             try:
 258                 info = check_output(["file", core_path])
 259                 logger.debug(info)
 260             except CalledProcessError as e:
 261                 logger.error("Subprocess returned with return code "
 262                              "while running `file' utility on core-file "
 263                              "returned: "
 264                              "rc=%s", e.returncode)
 265             except OSError as e:
 266                 logger.error("Subprocess returned with OS error while "
 267                              "running 'file' utility "
 268                              "on core-file: "
 269                              "(%s) %s", e.errno, e.strerror)
 270             except Exception as e:
 271                 logger.exception("Unexpected error running `file' utility "
 272                                  "on core-file")
 273
 274     if vpp_pid:
 275         # Copy api post mortem
 276         api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
 277         if os.path.isfile(api_post_mortem_path):
 278             logger.error("Copying api_post_mortem.%d to %s" %
 279                          (vpp_pid, last_test_temp_dir))
 280             shutil.copy2(api_post_mortem_path, last_test_temp_dir)
 281
 282
 283 def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
 284     if is_core_present(tempdir):
 285         print('VPP core detected in %s. Last test running was %s' %
 286               (tempdir, core_crash_test))
 287         print(single_line_delim)
 288         spawn_gdb(vpp_binary, get_core_path(tempdir))
 289         print(single_line_delim)
 290
 291
 292 def handle_cores(failed_testcases):
 293     if debug_core:
 294         for failed_testcase in failed_testcases:
 295             tcs_with_core = failed_testcase.testclasess_with_core
 296             if tcs_with_core:
 297                 for test, vpp_binary, tempdir in tcs_with_core.values():
 298                     check_and_handle_core(vpp_binary, tempdir, test)
 299
 300
 301 def process_finished_testsuite(wrapped_testcase_suite,
 302                                finished_testcase_suites,
 303                                failed_wrapped_testcases,
 304                                results):
 305     results.append(wrapped_testcase_suite.result)
 306     finished_testcase_suites.add(wrapped_testcase_suite)
 307     stop_run = False
 308     if failfast and not wrapped_testcase_suite.was_successful():
 309         stop_run = True
 310
 311     if not wrapped_testcase_suite.was_successful():
 312         failed_wrapped_testcases.add(wrapped_testcase_suite)
 313         handle_failed_suite(wrapped_testcase_suite.logger,
 314                             wrapped_testcase_suite.last_test_temp_dir,
 315                             wrapped_testcase_suite.vpp_pid)
 316
 317     return stop_run
 318
 319
 320 def run_forked(testcase_suites):
 321     wrapped_testcase_suites = set()
 322
 323     # suites are unhashable, need to use list
 324     results = []
 325     unread_testcases = set()
 326     finished_unread_testcases = set()
 327     manager = StreamQueueManager()
 328     manager.start()
 329     for i in range(concurrent_tests):
 330         if testcase_suites:
 331             wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0),
 332                                                      manager)
 333             wrapped_testcase_suites.add(wrapped_testcase_suite)
 334             unread_testcases.add(wrapped_testcase_suite)
 335         else:
 336             break
 337
 338     read_from_testcases = threading.Event()
 339     read_from_testcases.set()
 340     stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper,
 341                                         args=(unread_testcases,
 342                                               finished_unread_testcases,
 343                                               read_from_testcases))
 344     stdouterr_thread.start()
 345
 346     failed_wrapped_testcases = set()
 347     stop_run = False
 348
 349     try:
 350         while wrapped_testcase_suites:
 351             finished_testcase_suites = set()
 352             for wrapped_testcase_suite in wrapped_testcase_suites:
 353                 while wrapped_testcase_suite.result_parent_end.poll():
 354                     wrapped_testcase_suite.result.process_result(
 355                         *wrapped_testcase_suite.result_parent_end.recv())
 356                     wrapped_testcase_suite.last_heard = time.time()
 357
 358                 while wrapped_testcase_suite.keep_alive_parent_end.poll():
 359                     wrapped_testcase_suite.last_test, \
 360                         wrapped_testcase_suite.last_test_vpp_binary, \
 361                         wrapped_testcase_suite.last_test_temp_dir, \
 362                         wrapped_testcase_suite.vpp_pid = \
 363                         wrapped_testcase_suite.keep_alive_parent_end.recv()
 364                     wrapped_testcase_suite.last_heard = time.time()
 365
 366                 if wrapped_testcase_suite.finished_parent_end.poll():
 367                     wrapped_testcase_suite.finished_parent_end.recv()
 368                     wrapped_testcase_suite.last_heard = time.time()
 369                     stop_run = process_finished_testsuite(
 370                         wrapped_testcase_suite,
 371                         finished_testcase_suites,
 372                         failed_wrapped_testcases,
 373                         results) or stop_run
 374                     continue
 375
 376                 fail = False
 377                 if wrapped_testcase_suite.last_heard + test_timeout < \
 378                         time.time():
 379                     fail = True
 380                     wrapped_testcase_suite.logger.critical(
 381                         "Child test runner process timed out "
 382                         "(last test running was `%s' in `%s')!" %
 383                         (wrapped_testcase_suite.last_test,
 384                          wrapped_testcase_suite.last_test_temp_dir))
 385                 elif not wrapped_testcase_suite.child.is_alive():
 386                     fail = True
 387                     wrapped_testcase_suite.logger.critical(
 388                         "Child test runner process unexpectedly died "
 389                         "(last test running was `%s' in `%s')!" %
 390                         (wrapped_testcase_suite.last_test,
 391                          wrapped_testcase_suite.last_test_temp_dir))
 392                 elif wrapped_testcase_suite.last_test_temp_dir and \
 393                         wrapped_testcase_suite.last_test_vpp_binary:
 394                     if is_core_present(
 395                             wrapped_testcase_suite.last_test_temp_dir):
 396                         wrapped_testcase_suite.add_testclass_with_core()
 397                         if wrapped_testcase_suite.core_detected_at is None:
 398                             wrapped_testcase_suite.core_detected_at = \
 399                                 time.time()
 400                         elif wrapped_testcase_suite.core_detected_at + \
 401                                 core_timeout < time.time():
 402                             wrapped_testcase_suite.logger.critical(
 403                                 "Child test runner process unresponsive and "
 404                                 "core-file exists in test temporary directory "
 405                                 "(last test running was `%s' in `%s')!" %
 406                                 (wrapped_testcase_suite.last_test,
 407                                  wrapped_testcase_suite.last_test_temp_dir))
 408                             fail = True
 409
 410                 if fail:
 411                     wrapped_testcase_suite.child.terminate()
 412                     try:
 413                         # terminating the child process tends to leave orphan
 414                         # VPP process around
 415                         if wrapped_testcase_suite.vpp_pid:
 416                             os.kill(wrapped_testcase_suite.vpp_pid,
 417                                     signal.SIGTERM)
 418                     except OSError:
 419                         # already dead
 420                         pass
 421                     wrapped_testcase_suite.result.crashed = True
 422                     wrapped_testcase_suite.result.process_result(
 423                         wrapped_testcase_suite.last_test_id, ERROR)
 424                     stop_run = process_finished_testsuite(
 425                         wrapped_testcase_suite,
 426                         finished_testcase_suites,
 427                         failed_wrapped_testcases,
 428                         results) or stop_run
 429
 430             for finished_testcase in finished_testcase_suites:
 431                 finished_testcase.child.join()
 432                 finished_testcase.close_pipes()
 433                 wrapped_testcase_suites.remove(finished_testcase)
 434                 finished_unread_testcases.add(finished_testcase)
 435                 finished_testcase.stdouterr_queue.put(None)
 436                 if stop_run:
 437                     while testcase_suites:
 438                         results.append(TestResult(testcase_suites.pop(0)))
 439                 elif testcase_suites:
 440                     new_testcase = TestCaseWrapper(testcase_suites.pop(0),
 441                                                    manager)
 442                     wrapped_testcase_suites.add(new_testcase)
 443                     unread_testcases.add(new_testcase)
 444     except Exception:
 445         for wrapped_testcase_suite in wrapped_testcase_suites:
 446             wrapped_testcase_suite.child.terminate()
 447             wrapped_testcase_suite.stdouterr_queue.put(None)
 448         raise
 449     finally:
 450         read_from_testcases.clear()
 451         stdouterr_thread.join(test_timeout)
 452         manager.shutdown()
 453
 454     handle_cores(failed_wrapped_testcases)
 455     return results
 456
 457
 458 class SplitToSuitesCallback:
 459     def __init__(self, filter_callback):
 460         self.suites = {}
 461         self.suite_name = 'default'
 462         self.filter_callback = filter_callback
 463         self.filtered = unittest.TestSuite()
 464
 465     def __call__(self, file_name, cls, method):
 466         test_method = cls(method)
 467         if self.filter_callback(file_name, cls.__name__, method):
 468             self.suite_name = file_name + cls.__name__
 469             if self.suite_name not in self.suites:
 470                 self.suites[self.suite_name] = unittest.TestSuite()
 471             self.suites[self.suite_name].addTest(test_method)
 472
 473         else:
 474             self.filtered.addTest(test_method)
 475
 476
 477 test_option = "TEST"
 478
 479
 480 def parse_test_option():
 481     f = os.getenv(test_option, None)
 482     filter_file_name = None
 483     filter_class_name = None
 484     filter_func_name = None
 485     if f:
 486         if '.' in f:
 487             parts = f.split('.')
 488             if len(parts) > 3:
 489                 raise Exception("Unrecognized %s option: %s" %
 490                                 (test_option, f))
 491             if len(parts) > 2:
 492                 if parts[2] not in ('*', ''):
 493                     filter_func_name = parts[2]
 494             if parts[1] not in ('*', ''):
 495                 filter_class_name = parts[1]
 496             if parts[0] not in ('*', ''):
 497                 if parts[0].startswith('test_'):
 498                     filter_file_name = parts[0]
 499                 else:
 500                     filter_file_name = 'test_%s' % parts[0]
 501         else:
 502             if f.startswith('test_'):
 503                 filter_file_name = f
 504             else:
 505                 filter_file_name = 'test_%s' % f
 506     if filter_file_name:
 507         filter_file_name = '%s.py' % filter_file_name
 508     return filter_file_name, filter_class_name, filter_func_name
 509
 510
 511 def filter_tests(tests, filter_cb):
 512     result = unittest.suite.TestSuite()
 513     for t in tests:
 514         if isinstance(t, unittest.suite.TestSuite):
 515             # this is a bunch of tests, recursively filter...
 516             x = filter_tests(t, filter_cb)
 517             if x.countTestCases() > 0:
 518                 result.addTest(x)
 519         elif isinstance(t, unittest.TestCase):
 520             # this is a single test
 521             parts = t.id().split('.')
 522             # t.id() for common cases like this:
 523             # test_classifier.TestClassifier.test_acl_ip
 524             # apply filtering only if it is so
 525             if len(parts) == 3:
 526                 if not filter_cb(parts[0], parts[1], parts[2]):
 527                     continue
 528             result.addTest(t)
 529         else:
 530             # unexpected object, don't touch it
 531             result.addTest(t)
 532     return result
 533
 534
 535 class FilterByTestOption:
 536     def __init__(self, filter_file_name, filter_class_name, filter_func_name):
 537         self.filter_file_name = filter_file_name
 538         self.filter_class_name = filter_class_name
 539         self.filter_func_name = filter_func_name
 540
 541     def __call__(self, file_name, class_name, func_name):
 542         if self.filter_file_name:
 543             fn_match = fnmatch.fnmatch(file_name, self.filter_file_name)
 544             if not fn_match:
 545                 return False
 546         if self.filter_class_name and class_name != self.filter_class_name:
 547             return False
 548         if self.filter_func_name and func_name != self.filter_func_name:
 549             return False
 550         return True
 551
 552
 553 class FilterByClassList:
 554     def __init__(self, classes_with_filenames):
 555         self.classes_with_filenames = classes_with_filenames
 556
 557     def __call__(self, file_name, class_name, func_name):
 558         return '.'.join([file_name, class_name]) in self.classes_with_filenames
 559
 560
 561 def suite_from_failed(suite, failed):
 562     failed = {x.rsplit('.', 1)[0] for x in failed}
 563     filter_cb = FilterByClassList(failed)
 564     suite = filter_tests(suite, filter_cb)
 565     return suite
 566
 567
 568 class AllResults(dict):
 569     def __init__(self):
 570         super(AllResults, self).__init__()
 571         self.all_testcases = 0
 572         self.results_per_suite = []
 573         self[PASS] = 0
 574         self[FAIL] = 0
 575         self[ERROR] = 0
 576         self[SKIP] = 0
 577         self[TEST_RUN] = 0
 578         self.rerun = []
 579         self.testsuites_no_tests_run = []
 580
 581     def add_results(self, result):
 582         self.results_per_suite.append(result)
 583         result_types = [PASS, FAIL, ERROR, SKIP, TEST_RUN]
 584         for result_type in result_types:
 585             self[result_type] += len(result[result_type])
 586
 587     def add_result(self, result):
 588         retval = 0
 589         self.all_testcases += result.testcase_suite.countTestCases()
 590         self.add_results(result)
 591
 592         if result.no_tests_run():
 593             self.testsuites_no_tests_run.append(result.testcase_suite)
 594             if result.crashed:
 595                 retval = -1
 596             else:
 597                 retval = 1
 598         elif not result.was_successful():
 599             retval = 1
 600
 601         if retval != 0:
 602             self.rerun.append(result.testcase_suite)
 603
 604         return retval
 605
 606     def print_results(self):
 607         print('')
 608         print(double_line_delim)
 609         print('TEST RESULTS:')
 610         print('     Scheduled tests: {}'.format(self.all_testcases))
 611         print('      Executed tests: {}'.format(self[TEST_RUN]))
 612         print('        Passed tests: {}'.format(
 613             colorize(str(self[PASS]), GREEN)))
 614         if self[SKIP] > 0:
 615             print('       Skipped tests: {}'.format(
 616                 colorize(str(self[SKIP]), YELLOW)))
 617         if self.not_executed > 0:
 618             print('  Not Executed tests: {}'.format(
 619                 colorize(str(self.not_executed), RED)))
 620         if self[FAIL] > 0:
 621             print('            Failures: {}'.format(
 622                 colorize(str(self[FAIL]), RED)))
 623         if self[ERROR] > 0:
 624             print('              Errors: {}'.format(
 625                 colorize(str(self[ERROR]), RED)))
 626
 627         if self.all_failed > 0:
 628             print('FAILURES AND ERRORS IN TESTS:')
 629             for result in self.results_per_suite:
 630                 failed_testcase_ids = result[FAIL]
 631                 errored_testcase_ids = result[ERROR]
 632                 old_testcase_name = None
 633                 if failed_testcase_ids or errored_testcase_ids:
 634                     for failed_test_id in failed_testcase_ids:
 635                         new_testcase_name, test_name = \
 636                             result.get_testcase_names(failed_test_id)
 637                         if new_testcase_name != old_testcase_name:
 638                             print('  Testcase name: {}'.format(
 639                                 colorize(new_testcase_name, RED)))
 640                             old_testcase_name = new_testcase_name
 641                         print('    FAILURE: {} [{}]'.format(
 642                             colorize(test_name, RED), failed_test_id))
 643                     for failed_test_id in errored_testcase_ids:
 644                         new_testcase_name, test_name = \
 645                             result.get_testcase_names(failed_test_id)
 646                         if new_testcase_name != old_testcase_name:
 647                             print('  Testcase name: {}'.format(
 648                                 colorize(new_testcase_name, RED)))
 649                             old_testcase_name = new_testcase_name
 650                         print('      ERROR: {} [{}]'.format(
 651                             colorize(test_name, RED), failed_test_id))
 652         if self.testsuites_no_tests_run:
 653             print('TESTCASES WHERE NO TESTS WERE SUCCESSFULLY EXECUTED:')
 654             tc_classes = set()
 655             for testsuite in self.testsuites_no_tests_run:
 656                 for testcase in testsuite:
 657                     tc_classes.add(get_testcase_doc_name(testcase))
 658             for tc_class in tc_classes:
 659                 print('  {}'.format(colorize(tc_class, RED)))
 660
 661         print(double_line_delim)
 662         print('')
 663
 664     @property
 665     def not_executed(self):
 666         return self.all_testcases - self[TEST_RUN]
 667
 668     @property
 669     def all_failed(self):
 670         return self[FAIL] + self[ERROR]
 671
 672
 673 def parse_results(results):
 674     """
 675     Prints the number of scheduled, executed, not executed, passed, failed,
 676     errored and skipped tests and details about failed and errored tests.
 677
 678     Also returns all suites where any test failed.
 679
 680     :param results:
 681     :return:
 682     """
 683
 684     results_per_suite = AllResults()
 685     crashed = False
 686     failed = False
 687     for result in results:
 688         result_code = results_per_suite.add_result(result)
 689         if result_code == 1:
 690             failed = True
 691         elif result_code == -1:
 692             crashed = True
 693
 694     results_per_suite.print_results()
 695
 696     if crashed:
 697         return_code = -1
 698     elif failed:
 699         return_code = 1
 700     else:
 701         return_code = 0
 702     return return_code, results_per_suite.rerun
 703
 704
 705 def parse_digit_env(env_var, default):
 706     value = os.getenv(env_var, default)
 707     if value != default:
 708         if value.isdigit():
 709             value = int(value)
 710         else:
 711             print('WARNING: unsupported value "%s" for env var "%s",'
 712                   'defaulting to %s' % (value, env_var, default))
 713             value = default
 714     return value
 715
 716
 717 if __name__ == '__main__':
 718
 719     verbose = parse_digit_env("V", 0)
 720
 721     test_timeout = parse_digit_env("TIMEOUT", 600)  # default = 10 minutes
 722
 723     retries = parse_digit_env("RETRIES", 0)
 724
 725     debug = os.getenv("DEBUG", "n").lower() in ["gdb", "gdbserver"]
 726
 727     debug_core = os.getenv("DEBUG", "").lower() == "core"
 728
 729     step = os.getenv("STEP", "n").lower() in ("y", "yes", "1")
 730
 731     run_interactive = debug or step
 732
 733     test_jobs = os.getenv("TEST_JOBS", "1").lower()  # default = 1 process
 734     if test_jobs == 'auto':
 735         if run_interactive:
 736             concurrent_tests = 1
 737             print('Interactive mode required, running on one core')
 738         else:
 739             shm_free = psutil.disk_usage('/dev/shm').free
 740             shm_max_processes = 1
 741             if shm_free < min_req_shm:
 742                 raise Exception('Not enough free space in /dev/shm. Required '
 743                                 'free space is at least %sM.'
 744                                 % (min_req_shm >> 20))
 745             else:
 746                 extra_shm = shm_free - min_req_shm
 747                 shm_max_processes += extra_shm / shm_per_process
 748             concurrent_tests = min(cpu_count(), shm_max_processes)
 749             print('Found enough resources to run tests with %s cores'
 750                   % concurrent_tests)
 751     elif test_jobs.isdigit():
 752         concurrent_tests = int(test_jobs)
 753     else:
 754         concurrent_tests = 1
 755
 756     if run_interactive and concurrent_tests > 1:
 757         raise NotImplementedError(
 758             'Running tests interactively (DEBUG is gdb or gdbserver or STEP '
 759             'is set) in parallel (TEST_JOBS is more than 1) is not supported')
 760
 761     parser = argparse.ArgumentParser(description="VPP unit tests")
 762     parser.add_argument("-f", "--failfast", action='store_true',
 763                         help="fast failure flag")
 764     parser.add_argument("-d", "--dir", action='append', type=str,
 765                         help="directory containing test files "
 766                              "(may be specified multiple times)")
 767     args = parser.parse_args()
 768     failfast = args.failfast
 769     descriptions = True
 770
 771     print("Running tests using custom test runner")  # debug message
 772     filter_file, filter_class, filter_func = parse_test_option()
 773
 774     print("Active filters: file=%s, class=%s, function=%s" % (
 775         filter_file, filter_class, filter_func))
 776
 777     filter_cb = FilterByTestOption(filter_file, filter_class, filter_func)
 778
 779     ignore_path = os.getenv("VENV_PATH", None)
 780     cb = SplitToSuitesCallback(filter_cb)
 781     for d in args.dir:
 782         print("Adding tests from directory tree %s" % d)
 783         discover_tests(d, cb, ignore_path)
 784
 785     # suites are not hashable, need to use list
 786     suites = []
 787     tests_amount = 0
 788     for testcase_suite in cb.suites.values():
 789         tests_amount += testcase_suite.countTestCases()
 790         suites.append(testcase_suite)
 791
 792     print("%s out of %s tests match specified filters" % (
 793         tests_amount, tests_amount + cb.filtered.countTestCases()))
 794
 795     if not running_extended_tests:
 796         print("Not running extended tests (some tests will be skipped)")
 797
 798     attempts = retries + 1
 799     if attempts > 1:
 800         print("Perform %s attempts to pass the suite..." % attempts)
 801
 802     if run_interactive and suites:
 803         # don't fork if requiring interactive terminal
 804         full_suite = unittest.TestSuite()
 805         map(full_suite.addTests, suites)
 806         result = VppTestRunner(verbosity=verbose,
 807                                failfast=failfast,
 808                                print_summary=True).run(full_suite)
 809         was_successful = result.wasSuccessful()
 810         if not was_successful:
 811             for test_case_info in result.failed_test_cases_info:
 812                 handle_failed_suite(test_case_info.logger,
 813                                     test_case_info.tempdir,
 814                                     test_case_info.vpp_pid)
 815                 if debug_core and \
 816                         test_case_info in result.core_crash_test_cases_info:
 817                     check_and_handle_core(test_case_info.vpp_bin_path,
 818                                           test_case_info.tempdir,
 819                                           test_case_info.core_crash_test)
 820
 821         sys.exit(not was_successful)
 822     else:
 823         exit_code = 0
 824         while suites and attempts > 0:
 825             results = run_forked(suites)
 826             exit_code, suites = parse_results(results)
 827             attempts -= 1
 828             if exit_code == 0:
 829                 print('Test run was successful')
 830             else:
 831                 print('%s attempt(s) left.' % attempts)
 832         sys.exit(exit_code)