test/run_tests.py

   1 #!/usr/bin/env python
   2
   3 import sys
   4 import shutil
   5 import os
   6 import fnmatch
   7 import unittest
   8 import argparse
   9 import time
  10 import threading
  11 import signal
  12 import psutil
  13 import re
  14 from multiprocessing import Process, Pipe, cpu_count
  15 from multiprocessing.queues import Queue
  16 from multiprocessing.managers import BaseManager
  17 from framework import VppTestRunner, running_extended_tests, VppTestCase, \
  18     get_testcase_doc_name, get_test_description, PASS, FAIL, ERROR, SKIP, \
  19     TEST_RUN
  20 from debug import spawn_gdb
  21 from log import get_parallel_logger, double_line_delim, RED, YELLOW, GREEN, \
  22     colorize, single_line_delim
  23 from discover_tests import discover_tests
  24 from subprocess import check_output, CalledProcessError
  25 from util import check_core_path, get_core_path, is_core_present
  26
  27 # timeout which controls how long the child has to finish after seeing
  28 # a core dump in test temporary directory. If this is exceeded, parent assumes
  29 # that child process is stuck (e.g. waiting for shm mutex, which will never
  30 # get unlocked) and kill the child
  31 core_timeout = 3
  32 min_req_shm = 536870912  # min 512MB shm required
  33 # 128MB per extra process
  34 shm_per_process = 134217728
  35
  36
  37 class StreamQueue(Queue):
  38     def write(self, msg):
  39         self.put(msg)
  40
  41     def flush(self):
  42         sys.__stdout__.flush()
  43         sys.__stderr__.flush()
  44
  45     def fileno(self):
  46         return self._writer.fileno()
  47
  48
  49 class StreamQueueManager(BaseManager):
  50     pass
  51
  52
  53 StreamQueueManager.register('StreamQueue', StreamQueue)
  54
  55
  56 class TestResult(dict):
  57     def __init__(self, testcase_suite, testcases_by_id=None):
  58         super(TestResult, self).__init__()
  59         self[PASS] = []
  60         self[FAIL] = []
  61         self[ERROR] = []
  62         self[SKIP] = []
  63         self[TEST_RUN] = []
  64         self.crashed = False
  65         self.testcase_suite = testcase_suite
  66         self.testcases = [testcase for testcase in testcase_suite]
  67         self.testcases_by_id = testcases_by_id
  68
  69     def was_successful(self):
  70         return 0 == len(self[FAIL]) == len(self[ERROR]) \
  71             and len(self[PASS] + self[SKIP]) \
  72             == self.testcase_suite.countTestCases() == len(self[TEST_RUN])
  73
  74     def no_tests_run(self):
  75         return 0 == len(self[TEST_RUN])
  76
  77     def process_result(self, test_id, result):
  78         self[result].append(test_id)
  79
  80     def suite_from_failed(self):
  81         rerun_ids = set([])
  82         for testcase in self.testcase_suite:
  83             tc_id = testcase.id()
  84             if tc_id not in self[PASS] and tc_id not in self[SKIP]:
  85                 rerun_ids.add(tc_id)
  86         if rerun_ids:
  87             return suite_from_failed(self.testcase_suite, rerun_ids)
  88
  89     def get_testcase_names(self, test_id):
  90         # could be tearDownClass (test_ipsec_esp.TestIpsecEsp1)
  91         setup_teardown_match = re.match(
  92             r'((tearDownClass)|(setUpClass)) \((.+\..+)\)', test_id)
  93         if setup_teardown_match:
  94             test_name, _, _, testcase_name = setup_teardown_match.groups()
  95             if len(testcase_name.split('.')) == 2:
  96                 for key in self.testcases_by_id.keys():
  97                     if key.startswith(testcase_name):
  98                         testcase_name = key
  99                         break
 100             testcase_name = self._get_testcase_doc_name(testcase_name)
 101         else:
 102             test_name = self._get_test_description(test_id)
 103             testcase_name = self._get_testcase_doc_name(test_id)
 104
 105         return testcase_name, test_name
 106
 107     def _get_test_description(self, test_id):
 108         if test_id in self.testcases_by_id:
 109             desc = get_test_description(descriptions,
 110                                         self.testcases_by_id[test_id])
 111         else:
 112             desc = test_id
 113         return desc
 114
 115     def _get_testcase_doc_name(self, test_id):
 116         if test_id in self.testcases_by_id:
 117             doc_name = get_testcase_doc_name(self.testcases_by_id[test_id])
 118         else:
 119             doc_name = test_id
 120         return doc_name
 121
 122
 123 def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue,
 124                         finished_pipe, result_pipe, logger):
 125     sys.stdout = stdouterr_queue
 126     sys.stderr = stdouterr_queue
 127     VppTestCase.parallel_handler = logger.handlers[0]
 128     result = VppTestRunner(keep_alive_pipe=keep_alive_pipe,
 129                            descriptions=descriptions,
 130                            verbosity=verbose,
 131                            result_pipe=result_pipe,
 132                            failfast=failfast,
 133                            print_summary=False).run(suite)
 134     finished_pipe.send(result.wasSuccessful())
 135     finished_pipe.close()
 136     keep_alive_pipe.close()
 137
 138
 139 class TestCaseWrapper(object):
 140     def __init__(self, testcase_suite, manager):
 141         self.keep_alive_parent_end, self.keep_alive_child_end = Pipe(
 142             duplex=False)
 143         self.finished_parent_end, self.finished_child_end = Pipe(duplex=False)
 144         self.result_parent_end, self.result_child_end = Pipe(duplex=False)
 145         self.testcase_suite = testcase_suite
 146         if sys.version[0] == '2':
 147             self.stdouterr_queue = manager.StreamQueue()
 148         else:
 149             from multiprocessing import get_context
 150             self.stdouterr_queue = manager.StreamQueue(ctx=get_context())
 151         self.logger = get_parallel_logger(self.stdouterr_queue)
 152         self.child = Process(target=test_runner_wrapper,
 153                              args=(testcase_suite,
 154                                    self.keep_alive_child_end,
 155                                    self.stdouterr_queue,
 156                                    self.finished_child_end,
 157                                    self.result_child_end,
 158                                    self.logger)
 159                              )
 160         self.child.start()
 161         self.last_test_temp_dir = None
 162         self.last_test_vpp_binary = None
 163         self._last_test = None
 164         self.last_test_id = None
 165         self.vpp_pid = None
 166         self.last_heard = time.time()
 167         self.core_detected_at = None
 168         self.testcases_by_id = {}
 169         self.testclasess_with_core = {}
 170         for testcase in self.testcase_suite:
 171             self.testcases_by_id[testcase.id()] = testcase
 172         self.result = TestResult(testcase_suite, self.testcases_by_id)
 173
 174     @property
 175     def last_test(self):
 176         return self._last_test
 177
 178     @last_test.setter
 179     def last_test(self, test_id):
 180         self.last_test_id = test_id
 181         if test_id in self.testcases_by_id:
 182             testcase = self.testcases_by_id[test_id]
 183             self._last_test = testcase.shortDescription()
 184             if not self._last_test:
 185                 self._last_test = str(testcase)
 186         else:
 187             self._last_test = test_id
 188
 189     def add_testclass_with_core(self):
 190         if self.last_test_id in self.testcases_by_id:
 191             test = self.testcases_by_id[self.last_test_id]
 192             class_name = unittest.util.strclass(test.__class__)
 193             test_name = "'{}' ({})".format(get_test_description(descriptions,
 194                                                                 test),
 195                                            self.last_test_id)
 196         else:
 197             test_name = self.last_test_id
 198             class_name = re.match(r'((tearDownClass)|(setUpClass)) '
 199                                   r'\((.+\..+)\)', test_name).groups()[3]
 200         if class_name not in self.testclasess_with_core:
 201             self.testclasess_with_core[class_name] = (
 202                 test_name,
 203                 self.last_test_vpp_binary,
 204                 self.last_test_temp_dir)
 205
 206     def close_pipes(self):
 207         self.keep_alive_child_end.close()
 208         self.finished_child_end.close()
 209         self.result_child_end.close()
 210         self.keep_alive_parent_end.close()
 211         self.finished_parent_end.close()
 212         self.result_parent_end.close()
 213
 214     def was_successful(self):
 215         return self.result.was_successful()
 216
 217
 218 def stdouterr_reader_wrapper(unread_testcases, finished_unread_testcases,
 219                              read_testcases):
 220     read_testcase = None
 221     while read_testcases.is_set() or unread_testcases:
 222         if finished_unread_testcases:
 223             read_testcase = finished_unread_testcases.pop()
 224             unread_testcases.remove(read_testcase)
 225         elif unread_testcases:
 226             read_testcase = unread_testcases.pop()
 227         if read_testcase:
 228             data = ''
 229             while data is not None:
 230                 sys.stdout.write(data)
 231                 data = read_testcase.stdouterr_queue.get()
 232
 233             read_testcase.stdouterr_queue.close()
 234             finished_unread_testcases.discard(read_testcase)
 235             read_testcase = None
 236
 237
 238 def handle_failed_suite(logger, last_test_temp_dir, vpp_pid):
 239     if last_test_temp_dir:
 240         # Need to create link in case of a timeout or core dump without failure
 241         lttd = os.path.basename(last_test_temp_dir)
 242         failed_dir = os.getenv('FAILED_DIR')
 243         link_path = '%s%s-FAILED' % (failed_dir, lttd)
 244         if not os.path.exists(link_path):
 245             os.symlink(last_test_temp_dir, link_path)
 246         logger.error("Symlink to failed testcase directory: %s -> %s"
 247                      % (link_path, lttd))
 248
 249         # Report core existence
 250         core_path = get_core_path(last_test_temp_dir)
 251         if os.path.exists(core_path):
 252             logger.error(
 253                 "Core-file exists in test temporary directory: %s!" %
 254                 core_path)
 255             check_core_path(logger, core_path)
 256             logger.debug("Running 'file %s':" % core_path)
 257             try:
 258                 info = check_output(["file", core_path])
 259                 logger.debug(info)
 260             except CalledProcessError as e:
 261                 logger.error("Subprocess returned with return code "
 262                              "while running `file' utility on core-file "
 263                              "returned: "
 264                              "rc=%s", e.returncode)
 265             except OSError as e:
 266                 logger.error("Subprocess returned with OS error while "
 267                              "running 'file' utility "
 268                              "on core-file: "
 269                              "(%s) %s", e.errno, e.strerror)
 270             except Exception as e:
 271                 logger.exception("Unexpected error running `file' utility "
 272                                  "on core-file")
 273
 274     if vpp_pid:
 275         # Copy api post mortem
 276         api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
 277         if os.path.isfile(api_post_mortem_path):
 278             logger.error("Copying api_post_mortem.%d to %s" %
 279                          (vpp_pid, last_test_temp_dir))
 280             shutil.copy2(api_post_mortem_path, last_test_temp_dir)
 281
 282
 283 def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
 284     if is_core_present(tempdir):
 285         if debug_core:
 286             print('VPP core detected in %s. Last test running was %s' %
 287                   (tempdir, core_crash_test))
 288             print(single_line_delim)
 289             spawn_gdb(vpp_binary, get_core_path(tempdir))
 290             print(single_line_delim)
 291         elif compress_core:
 292             print("Compressing core-file in test directory `%s'" % tempdir)
 293             os.system("gzip %s" % get_core_path(tempdir))
 294
 295
 296 def handle_cores(failed_testcases):
 297     for failed_testcase in failed_testcases:
 298         tcs_with_core = failed_testcase.testclasess_with_core
 299         if tcs_with_core:
 300             for test, vpp_binary, tempdir in tcs_with_core.values():
 301                 check_and_handle_core(vpp_binary, tempdir, test)
 302
 303
 304 def process_finished_testsuite(wrapped_testcase_suite,
 305                                finished_testcase_suites,
 306                                failed_wrapped_testcases,
 307                                results):
 308     results.append(wrapped_testcase_suite.result)
 309     finished_testcase_suites.add(wrapped_testcase_suite)
 310     stop_run = False
 311     if failfast and not wrapped_testcase_suite.was_successful():
 312         stop_run = True
 313
 314     if not wrapped_testcase_suite.was_successful():
 315         failed_wrapped_testcases.add(wrapped_testcase_suite)
 316         handle_failed_suite(wrapped_testcase_suite.logger,
 317                             wrapped_testcase_suite.last_test_temp_dir,
 318                             wrapped_testcase_suite.vpp_pid)
 319
 320     return stop_run
 321
 322
 323 def run_forked(testcase_suites):
 324     wrapped_testcase_suites = set()
 325
 326     # suites are unhashable, need to use list
 327     results = []
 328     unread_testcases = set()
 329     finished_unread_testcases = set()
 330     manager = StreamQueueManager()
 331     manager.start()
 332     for i in range(concurrent_tests):
 333         if testcase_suites:
 334             wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0),
 335                                                      manager)
 336             wrapped_testcase_suites.add(wrapped_testcase_suite)
 337             unread_testcases.add(wrapped_testcase_suite)
 338         else:
 339             break
 340
 341     read_from_testcases = threading.Event()
 342     read_from_testcases.set()
 343     stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper,
 344                                         args=(unread_testcases,
 345                                               finished_unread_testcases,
 346                                               read_from_testcases))
 347     stdouterr_thread.start()
 348
 349     failed_wrapped_testcases = set()
 350     stop_run = False
 351
 352     try:
 353         while wrapped_testcase_suites:
 354             finished_testcase_suites = set()
 355             for wrapped_testcase_suite in wrapped_testcase_suites:
 356                 while wrapped_testcase_suite.result_parent_end.poll():
 357                     wrapped_testcase_suite.result.process_result(
 358                         *wrapped_testcase_suite.result_parent_end.recv())
 359                     wrapped_testcase_suite.last_heard = time.time()
 360
 361                 while wrapped_testcase_suite.keep_alive_parent_end.poll():
 362                     wrapped_testcase_suite.last_test, \
 363                         wrapped_testcase_suite.last_test_vpp_binary, \
 364                         wrapped_testcase_suite.last_test_temp_dir, \
 365                         wrapped_testcase_suite.vpp_pid = \
 366                         wrapped_testcase_suite.keep_alive_parent_end.recv()
 367                     wrapped_testcase_suite.last_heard = time.time()
 368
 369                 if wrapped_testcase_suite.finished_parent_end.poll():
 370                     wrapped_testcase_suite.finished_parent_end.recv()
 371                     wrapped_testcase_suite.last_heard = time.time()
 372                     stop_run = process_finished_testsuite(
 373                         wrapped_testcase_suite,
 374                         finished_testcase_suites,
 375                         failed_wrapped_testcases,
 376                         results) or stop_run
 377                     continue
 378
 379                 fail = False
 380                 if wrapped_testcase_suite.last_heard + test_timeout < \
 381                         time.time():
 382                     fail = True
 383                     wrapped_testcase_suite.logger.critical(
 384                         "Child test runner process timed out "
 385                         "(last test running was `%s' in `%s')!" %
 386                         (wrapped_testcase_suite.last_test,
 387                          wrapped_testcase_suite.last_test_temp_dir))
 388                 elif not wrapped_testcase_suite.child.is_alive():
 389                     fail = True
 390                     wrapped_testcase_suite.logger.critical(
 391                         "Child test runner process unexpectedly died "
 392                         "(last test running was `%s' in `%s')!" %
 393                         (wrapped_testcase_suite.last_test,
 394                          wrapped_testcase_suite.last_test_temp_dir))
 395                 elif wrapped_testcase_suite.last_test_temp_dir and \
 396                         wrapped_testcase_suite.last_test_vpp_binary:
 397                     if is_core_present(
 398                             wrapped_testcase_suite.last_test_temp_dir):
 399                         wrapped_testcase_suite.add_testclass_with_core()
 400                         if wrapped_testcase_suite.core_detected_at is None:
 401                             wrapped_testcase_suite.core_detected_at = \
 402                                 time.time()
 403                         elif wrapped_testcase_suite.core_detected_at + \
 404                                 core_timeout < time.time():
 405                             wrapped_testcase_suite.logger.critical(
 406                                 "Child test runner process unresponsive and "
 407                                 "core-file exists in test temporary directory "
 408                                 "(last test running was `%s' in `%s')!" %
 409                                 (wrapped_testcase_suite.last_test,
 410                                  wrapped_testcase_suite.last_test_temp_dir))
 411                             fail = True
 412
 413                 if fail:
 414                     wrapped_testcase_suite.child.terminate()
 415                     try:
 416                         # terminating the child process tends to leave orphan
 417                         # VPP process around
 418                         if wrapped_testcase_suite.vpp_pid:
 419                             os.kill(wrapped_testcase_suite.vpp_pid,
 420                                     signal.SIGTERM)
 421                     except OSError:
 422                         # already dead
 423                         pass
 424                     wrapped_testcase_suite.result.crashed = True
 425                     wrapped_testcase_suite.result.process_result(
 426                         wrapped_testcase_suite.last_test_id, ERROR)
 427                     stop_run = process_finished_testsuite(
 428                         wrapped_testcase_suite,
 429                         finished_testcase_suites,
 430                         failed_wrapped_testcases,
 431                         results) or stop_run
 432
 433             for finished_testcase in finished_testcase_suites:
 434                 finished_testcase.child.join()
 435                 finished_testcase.close_pipes()
 436                 wrapped_testcase_suites.remove(finished_testcase)
 437                 finished_unread_testcases.add(finished_testcase)
 438                 finished_testcase.stdouterr_queue.put(None)
 439                 if stop_run:
 440                     while testcase_suites:
 441                         results.append(TestResult(testcase_suites.pop(0)))
 442                 elif testcase_suites:
 443                     new_testcase = TestCaseWrapper(testcase_suites.pop(0),
 444                                                    manager)
 445                     wrapped_testcase_suites.add(new_testcase)
 446                     unread_testcases.add(new_testcase)
 447             time.sleep(0.1)
 448     except Exception:
 449         for wrapped_testcase_suite in wrapped_testcase_suites:
 450             wrapped_testcase_suite.child.terminate()
 451             wrapped_testcase_suite.stdouterr_queue.put(None)
 452         raise
 453     finally:
 454         read_from_testcases.clear()
 455         stdouterr_thread.join(test_timeout)
 456         manager.shutdown()
 457
 458     handle_cores(failed_wrapped_testcases)
 459     return results
 460
 461
 462 class SplitToSuitesCallback:
 463     def __init__(self, filter_callback):
 464         self.suites = {}
 465         self.suite_name = 'default'
 466         self.filter_callback = filter_callback
 467         self.filtered = unittest.TestSuite()
 468
 469     def __call__(self, file_name, cls, method):
 470         test_method = cls(method)
 471         if self.filter_callback(file_name, cls.__name__, method):
 472             self.suite_name = file_name + cls.__name__
 473             if self.suite_name not in self.suites:
 474                 self.suites[self.suite_name] = unittest.TestSuite()
 475             self.suites[self.suite_name].addTest(test_method)
 476
 477         else:
 478             self.filtered.addTest(test_method)
 479
 480
 481 test_option = "TEST"
 482
 483
 484 def parse_test_option():
 485     f = os.getenv(test_option, None)
 486     filter_file_name = None
 487     filter_class_name = None
 488     filter_func_name = None
 489     if f:
 490         if '.' in f:
 491             parts = f.split('.')
 492             if len(parts) > 3:
 493                 raise Exception("Unrecognized %s option: %s" %
 494                                 (test_option, f))
 495             if len(parts) > 2:
 496                 if parts[2] not in ('*', ''):
 497                     filter_func_name = parts[2]
 498             if parts[1] not in ('*', ''):
 499                 filter_class_name = parts[1]
 500             if parts[0] not in ('*', ''):
 501                 if parts[0].startswith('test_'):
 502                     filter_file_name = parts[0]
 503                 else:
 504                     filter_file_name = 'test_%s' % parts[0]
 505         else:
 506             if f.startswith('test_'):
 507                 filter_file_name = f
 508             else:
 509                 filter_file_name = 'test_%s' % f
 510     if filter_file_name:
 511         filter_file_name = '%s.py' % filter_file_name
 512     return filter_file_name, filter_class_name, filter_func_name
 513
 514
 515 def filter_tests(tests, filter_cb):
 516     result = unittest.suite.TestSuite()
 517     for t in tests:
 518         if isinstance(t, unittest.suite.TestSuite):
 519             # this is a bunch of tests, recursively filter...
 520             x = filter_tests(t, filter_cb)
 521             if x.countTestCases() > 0:
 522                 result.addTest(x)
 523         elif isinstance(t, unittest.TestCase):
 524             # this is a single test
 525             parts = t.id().split('.')
 526             # t.id() for common cases like this:
 527             # test_classifier.TestClassifier.test_acl_ip
 528             # apply filtering only if it is so
 529             if len(parts) == 3:
 530                 if not filter_cb(parts[0], parts[1], parts[2]):
 531                     continue
 532             result.addTest(t)
 533         else:
 534             # unexpected object, don't touch it
 535             result.addTest(t)
 536     return result
 537
 538
 539 class FilterByTestOption:
 540     def __init__(self, filter_file_name, filter_class_name, filter_func_name):
 541         self.filter_file_name = filter_file_name
 542         self.filter_class_name = filter_class_name
 543         self.filter_func_name = filter_func_name
 544
 545     def __call__(self, file_name, class_name, func_name):
 546         if self.filter_file_name:
 547             fn_match = fnmatch.fnmatch(file_name, self.filter_file_name)
 548             if not fn_match:
 549                 return False
 550         if self.filter_class_name and class_name != self.filter_class_name:
 551             return False
 552         if self.filter_func_name and func_name != self.filter_func_name:
 553             return False
 554         return True
 555
 556
 557 class FilterByClassList:
 558     def __init__(self, classes_with_filenames):
 559         self.classes_with_filenames = classes_with_filenames
 560
 561     def __call__(self, file_name, class_name, func_name):
 562         return '.'.join([file_name, class_name]) in self.classes_with_filenames
 563
 564
 565 def suite_from_failed(suite, failed):
 566     failed = {x.rsplit('.', 1)[0] for x in failed}
 567     filter_cb = FilterByClassList(failed)
 568     suite = filter_tests(suite, filter_cb)
 569     return suite
 570
 571
 572 class AllResults(dict):
 573     def __init__(self):
 574         super(AllResults, self).__init__()
 575         self.all_testcases = 0
 576         self.results_per_suite = []
 577         self[PASS] = 0
 578         self[FAIL] = 0
 579         self[ERROR] = 0
 580         self[SKIP] = 0
 581         self[TEST_RUN] = 0
 582         self.rerun = []
 583         self.testsuites_no_tests_run = []
 584
 585     def add_results(self, result):
 586         self.results_per_suite.append(result)
 587         result_types = [PASS, FAIL, ERROR, SKIP, TEST_RUN]
 588         for result_type in result_types:
 589             self[result_type] += len(result[result_type])
 590
 591     def add_result(self, result):
 592         retval = 0
 593         self.all_testcases += result.testcase_suite.countTestCases()
 594         self.add_results(result)
 595
 596         if result.no_tests_run():
 597             self.testsuites_no_tests_run.append(result.testcase_suite)
 598             if result.crashed:
 599                 retval = -1
 600             else:
 601                 retval = 1
 602         elif not result.was_successful():
 603             retval = 1
 604
 605         if retval != 0:
 606             self.rerun.append(result.testcase_suite)
 607
 608         return retval
 609
 610     def print_results(self):
 611         print('')
 612         print(double_line_delim)
 613         print('TEST RESULTS:')
 614         print('     Scheduled tests: {}'.format(self.all_testcases))
 615         print('      Executed tests: {}'.format(self[TEST_RUN]))
 616         print('        Passed tests: {}'.format(
 617             colorize(str(self[PASS]), GREEN)))
 618         if self[SKIP] > 0:
 619             print('       Skipped tests: {}'.format(
 620                 colorize(str(self[SKIP]), YELLOW)))
 621         if self.not_executed > 0:
 622             print('  Not Executed tests: {}'.format(
 623                 colorize(str(self.not_executed), RED)))
 624         if self[FAIL] > 0:
 625             print('            Failures: {}'.format(
 626                 colorize(str(self[FAIL]), RED)))
 627         if self[ERROR] > 0:
 628             print('              Errors: {}'.format(
 629                 colorize(str(self[ERROR]), RED)))
 630
 631         if self.all_failed > 0:
 632             print('FAILURES AND ERRORS IN TESTS:')
 633             for result in self.results_per_suite:
 634                 failed_testcase_ids = result[FAIL]
 635                 errored_testcase_ids = result[ERROR]
 636                 old_testcase_name = None
 637                 if failed_testcase_ids or errored_testcase_ids:
 638                     for failed_test_id in failed_testcase_ids:
 639                         new_testcase_name, test_name = \
 640                             result.get_testcase_names(failed_test_id)
 641                         if new_testcase_name != old_testcase_name:
 642                             print('  Testcase name: {}'.format(
 643                                 colorize(new_testcase_name, RED)))
 644                             old_testcase_name = new_testcase_name
 645                         print('    FAILURE: {} [{}]'.format(
 646                             colorize(test_name, RED), failed_test_id))
 647                     for failed_test_id in errored_testcase_ids:
 648                         new_testcase_name, test_name = \
 649                             result.get_testcase_names(failed_test_id)
 650                         if new_testcase_name != old_testcase_name:
 651                             print('  Testcase name: {}'.format(
 652                                 colorize(new_testcase_name, RED)))
 653                             old_testcase_name = new_testcase_name
 654                         print('      ERROR: {} [{}]'.format(
 655                             colorize(test_name, RED), failed_test_id))
 656         if self.testsuites_no_tests_run:
 657             print('TESTCASES WHERE NO TESTS WERE SUCCESSFULLY EXECUTED:')
 658             tc_classes = set()
 659             for testsuite in self.testsuites_no_tests_run:
 660                 for testcase in testsuite:
 661                     tc_classes.add(get_testcase_doc_name(testcase))
 662             for tc_class in tc_classes:
 663                 print('  {}'.format(colorize(tc_class, RED)))
 664
 665         print(double_line_delim)
 666         print('')
 667
 668     @property
 669     def not_executed(self):
 670         return self.all_testcases - self[TEST_RUN]
 671
 672     @property
 673     def all_failed(self):
 674         return self[FAIL] + self[ERROR]
 675
 676
 677 def parse_results(results):
 678     """
 679     Prints the number of scheduled, executed, not executed, passed, failed,
 680     errored and skipped tests and details about failed and errored tests.
 681
 682     Also returns all suites where any test failed.
 683
 684     :param results:
 685     :return:
 686     """
 687
 688     results_per_suite = AllResults()
 689     crashed = False
 690     failed = False
 691     for result in results:
 692         result_code = results_per_suite.add_result(result)
 693         if result_code == 1:
 694             failed = True
 695         elif result_code == -1:
 696             crashed = True
 697
 698     results_per_suite.print_results()
 699
 700     if crashed:
 701         return_code = -1
 702     elif failed:
 703         return_code = 1
 704     else:
 705         return_code = 0
 706     return return_code, results_per_suite.rerun
 707
 708
 709 def parse_digit_env(env_var, default):
 710     value = os.getenv(env_var, default)
 711     if value != default:
 712         if value.isdigit():
 713             value = int(value)
 714         else:
 715             print('WARNING: unsupported value "%s" for env var "%s",'
 716                   'defaulting to %s' % (value, env_var, default))
 717             value = default
 718     return value
 719
 720
 721 if __name__ == '__main__':
 722
 723     verbose = parse_digit_env("V", 0)
 724
 725     test_timeout = parse_digit_env("TIMEOUT", 600)  # default = 10 minutes
 726
 727     retries = parse_digit_env("RETRIES", 0)
 728
 729     debug = os.getenv("DEBUG", "n").lower() in ["gdb", "gdbserver"]
 730
 731     debug_core = os.getenv("DEBUG", "").lower() == "core"
 732     compress_core = os.getenv("CORE_COMPRESS", "").lower() in ("y", "yes", "1")
 733
 734     step = os.getenv("STEP", "n").lower() in ("y", "yes", "1")
 735
 736     run_interactive = debug or step
 737
 738     test_jobs = os.getenv("TEST_JOBS", "1").lower()  # default = 1 process
 739     if test_jobs == 'auto':
 740         if run_interactive:
 741             concurrent_tests = 1
 742             print('Interactive mode required, running on one core')
 743         else:
 744             shm_free = psutil.disk_usage('/dev/shm').free
 745             shm_max_processes = 1
 746             if shm_free < min_req_shm:
 747                 raise Exception('Not enough free space in /dev/shm. Required '
 748                                 'free space is at least %sM.'
 749                                 % (min_req_shm >> 20))
 750             else:
 751                 extra_shm = shm_free - min_req_shm
 752                 shm_max_processes += extra_shm / shm_per_process
 753             concurrent_tests = min(cpu_count(), shm_max_processes)
 754             print('Found enough resources to run tests with %s cores'
 755                   % concurrent_tests)
 756     elif test_jobs.isdigit():
 757         concurrent_tests = int(test_jobs)
 758     else:
 759         concurrent_tests = 1
 760
 761     if run_interactive and concurrent_tests > 1:
 762         raise NotImplementedError(
 763             'Running tests interactively (DEBUG is gdb or gdbserver or STEP '
 764             'is set) in parallel (TEST_JOBS is more than 1) is not supported')
 765
 766     parser = argparse.ArgumentParser(description="VPP unit tests")
 767     parser.add_argument("-f", "--failfast", action='store_true',
 768                         help="fast failure flag")
 769     parser.add_argument("-d", "--dir", action='append', type=str,
 770                         help="directory containing test files "
 771                              "(may be specified multiple times)")
 772     args = parser.parse_args()
 773     failfast = args.failfast
 774     descriptions = True
 775
 776     print("Running tests using custom test runner")  # debug message
 777     filter_file, filter_class, filter_func = parse_test_option()
 778
 779     print("Active filters: file=%s, class=%s, function=%s" % (
 780         filter_file, filter_class, filter_func))
 781
 782     filter_cb = FilterByTestOption(filter_file, filter_class, filter_func)
 783
 784     ignore_path = os.getenv("VENV_PATH", None)
 785     cb = SplitToSuitesCallback(filter_cb)
 786     for d in args.dir:
 787         print("Adding tests from directory tree %s" % d)
 788         discover_tests(d, cb, ignore_path)
 789
 790     # suites are not hashable, need to use list
 791     suites = []
 792     tests_amount = 0
 793     for testcase_suite in cb.suites.values():
 794         tests_amount += testcase_suite.countTestCases()
 795         suites.append(testcase_suite)
 796
 797     print("%s out of %s tests match specified filters" % (
 798         tests_amount, tests_amount + cb.filtered.countTestCases()))
 799
 800     if not running_extended_tests:
 801         print("Not running extended tests (some tests will be skipped)")
 802
 803     attempts = retries + 1
 804     if attempts > 1:
 805         print("Perform %s attempts to pass the suite..." % attempts)
 806
 807     if run_interactive and suites:
 808         # don't fork if requiring interactive terminal
 809         full_suite = unittest.TestSuite()
 810         map(full_suite.addTests, suites)
 811         result = VppTestRunner(verbosity=verbose,
 812                                failfast=failfast,
 813                                print_summary=True).run(full_suite)
 814         was_successful = result.wasSuccessful()
 815         if not was_successful:
 816             for test_case_info in result.failed_test_cases_info:
 817                 handle_failed_suite(test_case_info.logger,
 818                                     test_case_info.tempdir,
 819                                     test_case_info.vpp_pid)
 820                 if test_case_info in result.core_crash_test_cases_info:
 821                     check_and_handle_core(test_case_info.vpp_bin_path,
 822                                           test_case_info.tempdir,
 823                                           test_case_info.core_crash_test)
 824
 825         sys.exit(not was_successful)
 826     else:
 827         exit_code = 0
 828         while suites and attempts > 0:
 829             results = run_forked(suites)
 830             exit_code, suites = parse_results(results)
 831             attempts -= 1
 832             if exit_code == 0:
 833                 print('Test run was successful')
 834             else:
 835                 print('%s attempt(s) left.' % attempts)
 836         sys.exit(exit_code)