test/run_tests.py

   1 #!/usr/bin/env python
   2
   3 import sys
   4 import shutil
   5 import os
   6 import fnmatch
   7 import unittest
   8 import argparse
   9 import time
  10 import threading
  11 import signal
  12 import psutil
  13 import re
  14 from multiprocessing import Process, Pipe, cpu_count
  15 from multiprocessing.queues import Queue
  16 from multiprocessing.managers import BaseManager
  17 from framework import VppTestRunner, running_extended_tests, VppTestCase, \
  18     get_testcase_doc_name, get_test_description, PASS, FAIL, ERROR, SKIP, \
  19     TEST_RUN
  20 from debug import spawn_gdb
  21 from log import get_parallel_logger, double_line_delim, RED, YELLOW, GREEN, \
  22     colorize, single_line_delim
  23 from discover_tests import discover_tests
  24 from subprocess import check_output, CalledProcessError
  25 from util import check_core_path, get_core_path, is_core_present
  26
  27 # timeout which controls how long the child has to finish after seeing
  28 # a core dump in test temporary directory. If this is exceeded, parent assumes
  29 # that child process is stuck (e.g. waiting for shm mutex, which will never
  30 # get unlocked) and kill the child
  31 core_timeout = 3
  32 min_req_shm = 536870912  # min 512MB shm required
  33 # 128MB per extra process
  34 shm_per_process = 134217728
  35
  36
  37 class StreamQueue(Queue):
  38     def write(self, msg):
  39         self.put(msg)
  40
  41     def flush(self):
  42         sys.__stdout__.flush()
  43         sys.__stderr__.flush()
  44
  45     def fileno(self):
  46         return self._writer.fileno()
  47
  48
  49 class StreamQueueManager(BaseManager):
  50     pass
  51
  52
  53 StreamQueueManager.register('StreamQueue', StreamQueue)
  54
  55
  56 class TestResult(dict):
  57     def __init__(self, testcase_suite, testcases_by_id=None):
  58         super(TestResult, self).__init__()
  59         self[PASS] = []
  60         self[FAIL] = []
  61         self[ERROR] = []
  62         self[SKIP] = []
  63         self[TEST_RUN] = []
  64         self.crashed = False
  65         self.testcase_suite = testcase_suite
  66         self.testcases = [testcase for testcase in testcase_suite]
  67         self.testcases_by_id = testcases_by_id
  68
  69     def was_successful(self):
  70         return 0 == len(self[FAIL]) == len(self[ERROR]) \
  71             and len(self[PASS] + self[SKIP]) \
  72             == self.testcase_suite.countTestCases() == len(self[TEST_RUN])
  73
  74     def no_tests_run(self):
  75         return 0 == len(self[TEST_RUN])
  76
  77     def process_result(self, test_id, result):
  78         self[result].append(test_id)
  79
  80     def suite_from_failed(self):
  81         rerun_ids = set([])
  82         for testcase in self.testcase_suite:
  83             tc_id = testcase.id()
  84             if tc_id not in self[PASS] and tc_id not in self[SKIP]:
  85                 rerun_ids.add(tc_id)
  86         if len(rerun_ids) > 0:
  87             return suite_from_failed(self.testcase_suite, rerun_ids)
  88
  89     def get_testcase_names(self, test_id):
  90         if re.match(r'.+\..+\..+', test_id):
  91             test_name = self._get_test_description(test_id)
  92             testcase_name = self._get_testcase_doc_name(test_id)
  93         else:
  94             # could be tearDownClass (test_ipsec_esp.TestIpsecEsp1)
  95             setup_teardown_match = re.match(
  96                 r'((tearDownClass)|(setUpClass)) \((.+\..+)\)', test_id)
  97             if setup_teardown_match:
  98                 test_name, _, _, testcase_name = setup_teardown_match.groups()
  99                 if len(testcase_name.split('.')) == 2:
 100                     for key in self.testcases_by_id.keys():
 101                         if key.startswith(testcase_name):
 102                             testcase_name = key
 103                             break
 104                 testcase_name = self._get_testcase_doc_name(testcase_name)
 105             else:
 106                 test_name = test_id
 107                 testcase_name = test_id
 108
 109         return testcase_name, test_name
 110
 111     def _get_test_description(self, test_id):
 112         return get_test_description(descriptions,
 113                                     self.testcases_by_id[test_id])
 114
 115     def _get_testcase_doc_name(self, test_id):
 116         return get_testcase_doc_name(self.testcases_by_id[test_id])
 117
 118
 119 def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue,
 120                         finished_pipe, result_pipe, logger):
 121     sys.stdout = stdouterr_queue
 122     sys.stderr = stdouterr_queue
 123     VppTestCase.parallel_handler = logger.handlers[0]
 124     result = VppTestRunner(keep_alive_pipe=keep_alive_pipe,
 125                            descriptions=descriptions,
 126                            verbosity=verbose,
 127                            result_pipe=result_pipe,
 128                            failfast=failfast,
 129                            print_summary=False).run(suite)
 130     finished_pipe.send(result.wasSuccessful())
 131     finished_pipe.close()
 132     keep_alive_pipe.close()
 133
 134
 135 class TestCaseWrapper(object):
 136     def __init__(self, testcase_suite, manager):
 137         self.keep_alive_parent_end, self.keep_alive_child_end = Pipe(
 138             duplex=False)
 139         self.finished_parent_end, self.finished_child_end = Pipe(duplex=False)
 140         self.result_parent_end, self.result_child_end = Pipe(duplex=False)
 141         self.testcase_suite = testcase_suite
 142         self.stdouterr_queue = manager.StreamQueue()
 143         self.logger = get_parallel_logger(self.stdouterr_queue)
 144         self.child = Process(target=test_runner_wrapper,
 145                              args=(testcase_suite,
 146                                    self.keep_alive_child_end,
 147                                    self.stdouterr_queue,
 148                                    self.finished_child_end,
 149                                    self.result_child_end,
 150                                    self.logger)
 151                              )
 152         self.child.start()
 153         self.last_test_temp_dir = None
 154         self.last_test_vpp_binary = None
 155         self._last_test = None
 156         self.last_test_id = None
 157         self.vpp_pid = None
 158         self.last_heard = time.time()
 159         self.core_detected_at = None
 160         self.testcases_by_id = {}
 161         self.testclasess_with_core = {}
 162         for testcase in self.testcase_suite:
 163             self.testcases_by_id[testcase.id()] = testcase
 164         self.result = TestResult(testcase_suite, self.testcases_by_id)
 165
 166     @property
 167     def last_test(self):
 168         return self._last_test
 169
 170     @last_test.setter
 171     def last_test(self, test_id):
 172         self.last_test_id = test_id
 173         if test_id in self.testcases_by_id:
 174             testcase = self.testcases_by_id[test_id]
 175             self._last_test = testcase.shortDescription()
 176             if not self._last_test:
 177                 self._last_test = str(testcase)
 178         else:
 179             self._last_test = test_id
 180
 181     def add_testclass_with_core(self):
 182         if self.last_test_id in self.testcases_by_id:
 183             test = self.testcases_by_id[self.last_test_id]
 184             class_name = unittest.util.strclass(test.__class__)
 185             test_name = "'{}' ({})".format(get_test_description(descriptions,
 186                                                                 test),
 187                                            self.last_test_id)
 188         else:
 189             test_name = self.last_test_id
 190             class_name = re.match(r'((tearDownClass)|(setUpClass)) '
 191                                   r'\((.+\..+)\)', test_name).groups()[3]
 192         if class_name not in self.testclasess_with_core:
 193             self.testclasess_with_core[class_name] = (
 194                 test_name,
 195                 self.last_test_vpp_binary,
 196                 self.last_test_temp_dir)
 197
 198     def close_pipes(self):
 199         self.keep_alive_child_end.close()
 200         self.finished_child_end.close()
 201         self.result_child_end.close()
 202         self.keep_alive_parent_end.close()
 203         self.finished_parent_end.close()
 204         self.result_parent_end.close()
 205
 206     def was_successful(self):
 207         return self.result.was_successful()
 208
 209
 210 def stdouterr_reader_wrapper(unread_testcases, finished_unread_testcases,
 211                              read_testcases):
 212     read_testcase = None
 213     while read_testcases.is_set() or len(unread_testcases):
 214         if len(finished_unread_testcases):
 215             read_testcase = finished_unread_testcases.pop()
 216             unread_testcases.remove(read_testcase)
 217         elif len(unread_testcases):
 218             read_testcase = unread_testcases.pop()
 219         if read_testcase:
 220             data = ''
 221             while data is not None:
 222                 sys.stdout.write(data)
 223                 data = read_testcase.stdouterr_queue.get()
 224
 225             read_testcase.stdouterr_queue.close()
 226             finished_unread_testcases.discard(read_testcase)
 227             read_testcase = None
 228
 229
 230 def handle_failed_suite(logger, last_test_temp_dir, vpp_pid):
 231     if last_test_temp_dir:
 232         # Need to create link in case of a timeout or core dump without failure
 233         lttd = os.path.basename(last_test_temp_dir)
 234         failed_dir = os.getenv('FAILED_DIR')
 235         link_path = '%s%s-FAILED' % (failed_dir, lttd)
 236         if not os.path.exists(link_path):
 237             os.symlink(last_test_temp_dir, link_path)
 238         logger.error("Symlink to failed testcase directory: %s -> %s"
 239                      % (link_path, lttd))
 240
 241         # Report core existence
 242         core_path = get_core_path(last_test_temp_dir)
 243         if os.path.exists(core_path):
 244             logger.error(
 245                 "Core-file exists in test temporary directory: %s!" %
 246                 core_path)
 247             check_core_path(logger, core_path)
 248             logger.debug("Running `file %s':" % core_path)
 249             try:
 250                 info = check_output(["file", core_path])
 251                 logger.debug(info)
 252             except CalledProcessError as e:
 253                 logger.error("Could not run `file' utility on core-file, "
 254                              "rc=%s" % e.returncode)
 255
 256     if vpp_pid:
 257         # Copy api post mortem
 258         api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
 259         if os.path.isfile(api_post_mortem_path):
 260             logger.error("Copying api_post_mortem.%d to %s" %
 261                          (vpp_pid, last_test_temp_dir))
 262             shutil.copy2(api_post_mortem_path, last_test_temp_dir)
 263
 264
 265 def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
 266     if is_core_present(tempdir):
 267         print('VPP core detected in %s. Last test running was %s' %
 268               (tempdir, core_crash_test))
 269         print(single_line_delim)
 270         spawn_gdb(vpp_binary, get_core_path(tempdir))
 271         print(single_line_delim)
 272
 273
 274 def handle_cores(failed_testcases):
 275     if debug_core:
 276         for failed_testcase in failed_testcases:
 277             tcs_with_core = failed_testcase.testclasess_with_core
 278             if len(tcs_with_core) > 0:
 279                 for test, vpp_binary, tempdir in tcs_with_core.values():
 280                     check_and_handle_core(vpp_binary, tempdir, test)
 281
 282
 283 def process_finished_testsuite(wrapped_testcase_suite,
 284                                finished_testcase_suites,
 285                                failed_wrapped_testcases,
 286                                results):
 287     results.append(wrapped_testcase_suite.result)
 288     finished_testcase_suites.add(wrapped_testcase_suite)
 289     stop_run = False
 290     if failfast and not wrapped_testcase_suite.was_successful():
 291         stop_run = True
 292
 293     if not wrapped_testcase_suite.was_successful():
 294         failed_wrapped_testcases.add(wrapped_testcase_suite)
 295         handle_failed_suite(wrapped_testcase_suite.logger,
 296                             wrapped_testcase_suite.last_test_temp_dir,
 297                             wrapped_testcase_suite.vpp_pid)
 298
 299     return stop_run
 300
 301
 302 def run_forked(testcase_suites):
 303     wrapped_testcase_suites = set()
 304
 305     # suites are unhashable, need to use list
 306     results = []
 307     unread_testcases = set()
 308     finished_unread_testcases = set()
 309     manager = StreamQueueManager()
 310     manager.start()
 311     for i in range(concurrent_tests):
 312         if len(testcase_suites) > 0:
 313             wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0),
 314                                                      manager)
 315             wrapped_testcase_suites.add(wrapped_testcase_suite)
 316             unread_testcases.add(wrapped_testcase_suite)
 317         else:
 318             break
 319
 320     read_from_testcases = threading.Event()
 321     read_from_testcases.set()
 322     stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper,
 323                                         args=(unread_testcases,
 324                                               finished_unread_testcases,
 325                                               read_from_testcases))
 326     stdouterr_thread.start()
 327
 328     failed_wrapped_testcases = set()
 329     stop_run = False
 330
 331     try:
 332         while len(wrapped_testcase_suites) > 0:
 333             finished_testcase_suites = set()
 334             for wrapped_testcase_suite in wrapped_testcase_suites:
 335                 while wrapped_testcase_suite.result_parent_end.poll():
 336                     wrapped_testcase_suite.result.process_result(
 337                         *wrapped_testcase_suite.result_parent_end.recv())
 338                     wrapped_testcase_suite.last_heard = time.time()
 339
 340                 while wrapped_testcase_suite.keep_alive_parent_end.poll():
 341                     wrapped_testcase_suite.last_test, \
 342                         wrapped_testcase_suite.last_test_vpp_binary, \
 343                         wrapped_testcase_suite.last_test_temp_dir, \
 344                         wrapped_testcase_suite.vpp_pid = \
 345                         wrapped_testcase_suite.keep_alive_parent_end.recv()
 346                     wrapped_testcase_suite.last_heard = time.time()
 347
 348                 if wrapped_testcase_suite.finished_parent_end.poll():
 349                     wrapped_testcase_suite.finished_parent_end.recv()
 350                     wrapped_testcase_suite.last_heard = time.time()
 351                     stop_run = process_finished_testsuite(
 352                         wrapped_testcase_suite,
 353                         finished_testcase_suites,
 354                         failed_wrapped_testcases,
 355                         results) or stop_run
 356                     continue
 357
 358                 fail = False
 359                 if wrapped_testcase_suite.last_heard + test_timeout < \
 360                         time.time():
 361                     fail = True
 362                     wrapped_testcase_suite.logger.critical(
 363                         "Child test runner process timed out "
 364                         "(last test running was `%s' in `%s')!" %
 365                         (wrapped_testcase_suite.last_test,
 366                          wrapped_testcase_suite.last_test_temp_dir))
 367                 elif not wrapped_testcase_suite.child.is_alive():
 368                     fail = True
 369                     wrapped_testcase_suite.logger.critical(
 370                         "Child test runner process unexpectedly died "
 371                         "(last test running was `%s' in `%s')!" %
 372                         (wrapped_testcase_suite.last_test,
 373                          wrapped_testcase_suite.last_test_temp_dir))
 374                 elif wrapped_testcase_suite.last_test_temp_dir and \
 375                         wrapped_testcase_suite.last_test_vpp_binary:
 376                     if is_core_present(
 377                             wrapped_testcase_suite.last_test_temp_dir):
 378                         wrapped_testcase_suite.add_testclass_with_core()
 379                         if wrapped_testcase_suite.core_detected_at is None:
 380                             wrapped_testcase_suite.core_detected_at = \
 381                                 time.time()
 382                         elif wrapped_testcase_suite.core_detected_at + \
 383                                 core_timeout < time.time():
 384                             wrapped_testcase_suite.logger.critical(
 385                                 "Child test runner process unresponsive and "
 386                                 "core-file exists in test temporary directory "
 387                                 "(last test running was `%s' in `%s')!" %
 388                                 (wrapped_testcase_suite.last_test,
 389                                  wrapped_testcase_suite.last_test_temp_dir))
 390                             fail = True
 391
 392                 if fail:
 393                     wrapped_testcase_suite.child.terminate()
 394                     try:
 395                         # terminating the child process tends to leave orphan
 396                         # VPP process around
 397                         if wrapped_testcase_suite.vpp_pid:
 398                             os.kill(wrapped_testcase_suite.vpp_pid,
 399                                     signal.SIGTERM)
 400                     except OSError:
 401                         # already dead
 402                         pass
 403                     wrapped_testcase_suite.result.crashed = True
 404                     wrapped_testcase_suite.result.process_result(
 405                         wrapped_testcase_suite.last_test_id, ERROR)
 406                     stop_run = process_finished_testsuite(
 407                         wrapped_testcase_suite,
 408                         finished_testcase_suites,
 409                         failed_wrapped_testcases,
 410                         results) or stop_run
 411
 412             for finished_testcase in finished_testcase_suites:
 413                 finished_testcase.child.join()
 414                 finished_testcase.close_pipes()
 415                 wrapped_testcase_suites.remove(finished_testcase)
 416                 finished_unread_testcases.add(finished_testcase)
 417                 finished_testcase.stdouterr_queue.put(None)
 418                 if stop_run:
 419                     while len(testcase_suites) > 0:
 420                         results.append(TestResult(testcase_suites.pop(0)))
 421                 elif len(testcase_suites) > 0:
 422                     new_testcase = TestCaseWrapper(testcase_suites.pop(0),
 423                                                    manager)
 424                     wrapped_testcase_suites.add(new_testcase)
 425                     unread_testcases.add(new_testcase)
 426     except Exception:
 427         for wrapped_testcase_suite in wrapped_testcase_suites:
 428             wrapped_testcase_suite.child.terminate()
 429             wrapped_testcase_suite.stdouterr_queue.put(None)
 430         raise
 431     finally:
 432         read_from_testcases.clear()
 433         stdouterr_thread.join(test_timeout)
 434         manager.shutdown()
 435
 436     handle_cores(failed_wrapped_testcases)
 437     return results
 438
 439
 440 class SplitToSuitesCallback:
 441     def __init__(self, filter_callback):
 442         self.suites = {}
 443         self.suite_name = 'default'
 444         self.filter_callback = filter_callback
 445         self.filtered = unittest.TestSuite()
 446
 447     def __call__(self, file_name, cls, method):
 448         test_method = cls(method)
 449         if self.filter_callback(file_name, cls.__name__, method):
 450             self.suite_name = file_name + cls.__name__
 451             if self.suite_name not in self.suites:
 452                 self.suites[self.suite_name] = unittest.TestSuite()
 453             self.suites[self.suite_name].addTest(test_method)
 454
 455         else:
 456             self.filtered.addTest(test_method)
 457
 458
 459 test_option = "TEST"
 460
 461
 462 def parse_test_option():
 463     f = os.getenv(test_option, None)
 464     filter_file_name = None
 465     filter_class_name = None
 466     filter_func_name = None
 467     if f:
 468         if '.' in f:
 469             parts = f.split('.')
 470             if len(parts) > 3:
 471                 raise Exception("Unrecognized %s option: %s" %
 472                                 (test_option, f))
 473             if len(parts) > 2:
 474                 if parts[2] not in ('*', ''):
 475                     filter_func_name = parts[2]
 476             if parts[1] not in ('*', ''):
 477                 filter_class_name = parts[1]
 478             if parts[0] not in ('*', ''):
 479                 if parts[0].startswith('test_'):
 480                     filter_file_name = parts[0]
 481                 else:
 482                     filter_file_name = 'test_%s' % parts[0]
 483         else:
 484             if f.startswith('test_'):
 485                 filter_file_name = f
 486             else:
 487                 filter_file_name = 'test_%s' % f
 488     if filter_file_name:
 489         filter_file_name = '%s.py' % filter_file_name
 490     return filter_file_name, filter_class_name, filter_func_name
 491
 492
 493 def filter_tests(tests, filter_cb):
 494     result = unittest.suite.TestSuite()
 495     for t in tests:
 496         if isinstance(t, unittest.suite.TestSuite):
 497             # this is a bunch of tests, recursively filter...
 498             x = filter_tests(t, filter_cb)
 499             if x.countTestCases() > 0:
 500                 result.addTest(x)
 501         elif isinstance(t, unittest.TestCase):
 502             # this is a single test
 503             parts = t.id().split('.')
 504             # t.id() for common cases like this:
 505             # test_classifier.TestClassifier.test_acl_ip
 506             # apply filtering only if it is so
 507             if len(parts) == 3:
 508                 if not filter_cb(parts[0], parts[1], parts[2]):
 509                     continue
 510             result.addTest(t)
 511         else:
 512             # unexpected object, don't touch it
 513             result.addTest(t)
 514     return result
 515
 516
 517 class FilterByTestOption:
 518     def __init__(self, filter_file_name, filter_class_name, filter_func_name):
 519         self.filter_file_name = filter_file_name
 520         self.filter_class_name = filter_class_name
 521         self.filter_func_name = filter_func_name
 522
 523     def __call__(self, file_name, class_name, func_name):
 524         if self.filter_file_name:
 525             fn_match = fnmatch.fnmatch(file_name, self.filter_file_name)
 526             if not fn_match:
 527                 return False
 528         if self.filter_class_name and class_name != self.filter_class_name:
 529             return False
 530         if self.filter_func_name and func_name != self.filter_func_name:
 531             return False
 532         return True
 533
 534
 535 class FilterByClassList:
 536     def __init__(self, classes_with_filenames):
 537         self.classes_with_filenames = classes_with_filenames
 538
 539     def __call__(self, file_name, class_name, func_name):
 540         return '.'.join([file_name, class_name]) in self.classes_with_filenames
 541
 542
 543 def suite_from_failed(suite, failed):
 544     failed = {x.rsplit('.', 1)[0] for x in failed}
 545     filter_cb = FilterByClassList(failed)
 546     suite = filter_tests(suite, filter_cb)
 547     return suite
 548
 549
 550 class AllResults(dict):
 551     def __init__(self):
 552         super(AllResults, self).__init__()
 553         self.all_testcases = 0
 554         self.results_per_suite = []
 555         self[PASS] = 0
 556         self[FAIL] = 0
 557         self[ERROR] = 0
 558         self[SKIP] = 0
 559         self[TEST_RUN] = 0
 560         self.rerun = []
 561         self.testsuites_no_tests_run = []
 562
 563     def add_results(self, result):
 564         self.results_per_suite.append(result)
 565         result_types = [PASS, FAIL, ERROR, SKIP, TEST_RUN]
 566         for result_type in result_types:
 567             self[result_type] += len(result[result_type])
 568
 569     def add_result(self, result):
 570         retval = 0
 571         self.all_testcases += result.testcase_suite.countTestCases()
 572         self.add_results(result)
 573
 574         if result.no_tests_run():
 575             self.testsuites_no_tests_run.append(result.testcase_suite)
 576             if result.crashed:
 577                 retval = -1
 578             else:
 579                 retval = 1
 580         elif not result.was_successful():
 581             retval = 1
 582
 583         if retval != 0:
 584             self.rerun.append(result.testcase_suite)
 585
 586         return retval
 587
 588     def print_results(self):
 589         print('')
 590         print(double_line_delim)
 591         print('TEST RESULTS:')
 592         print('     Scheduled tests: {}'.format(self.all_testcases))
 593         print('      Executed tests: {}'.format(self[TEST_RUN]))
 594         print('        Passed tests: {}'.format(
 595             colorize(str(self[PASS]), GREEN)))
 596         if self[SKIP] > 0:
 597             print('       Skipped tests: {}'.format(
 598                 colorize(str(self[SKIP]), YELLOW)))
 599         if self.not_executed > 0:
 600             print('  Not Executed tests: {}'.format(
 601                 colorize(str(self.not_executed), RED)))
 602         if self[FAIL] > 0:
 603             print('            Failures: {}'.format(
 604                 colorize(str(self[FAIL]), RED)))
 605         if self[ERROR] > 0:
 606             print('              Errors: {}'.format(
 607                 colorize(str(self[ERROR]), RED)))
 608
 609         if self.all_failed > 0:
 610             print('FAILURES AND ERRORS IN TESTS:')
 611             for result in self.results_per_suite:
 612                 failed_testcase_ids = result[FAIL]
 613                 errored_testcase_ids = result[ERROR]
 614                 old_testcase_name = None
 615                 if len(failed_testcase_ids) or len(errored_testcase_ids):
 616                     for failed_test_id in failed_testcase_ids:
 617                         new_testcase_name, test_name = \
 618                             result.get_testcase_names(failed_test_id)
 619                         if new_testcase_name != old_testcase_name:
 620                             print('  Testcase name: {}'.format(
 621                                 colorize(new_testcase_name, RED)))
 622                             old_testcase_name = new_testcase_name
 623                         print('    FAILURE: {} [{}]'.format(
 624                             colorize(test_name, RED), failed_test_id))
 625                     for failed_test_id in errored_testcase_ids:
 626                         new_testcase_name, test_name = \
 627                             result.get_testcase_names(failed_test_id)
 628                         if new_testcase_name != old_testcase_name:
 629                             print('  Testcase name: {}'.format(
 630                                 colorize(new_testcase_name, RED)))
 631                             old_testcase_name = new_testcase_name
 632                         print('      ERROR: {} [{}]'.format(
 633                             colorize(test_name, RED), failed_test_id))
 634         if len(self.testsuites_no_tests_run) > 0:
 635             print('TESTCASES WHERE NO TESTS WERE SUCCESSFULLY EXECUTED:')
 636             tc_classes = set()
 637             for testsuite in self.testsuites_no_tests_run:
 638                 for testcase in testsuite:
 639                     tc_classes.add(get_testcase_doc_name(testcase))
 640             for tc_class in tc_classes:
 641                 print('  {}'.format(colorize(tc_class, RED)))
 642
 643         print(double_line_delim)
 644         print('')
 645
 646     @property
 647     def not_executed(self):
 648         return self.all_testcases - self[TEST_RUN]
 649
 650     @property
 651     def all_failed(self):
 652         return self[FAIL] + self[ERROR]
 653
 654
 655 def parse_results(results):
 656     """
 657     Prints the number of scheduled, executed, not executed, passed, failed,
 658     errored and skipped tests and details about failed and errored tests.
 659
 660     Also returns all suites where any test failed.
 661
 662     :param results:
 663     :return:
 664     """
 665
 666     results_per_suite = AllResults()
 667     crashed = False
 668     failed = False
 669     for result in results:
 670         result_code = results_per_suite.add_result(result)
 671         if result_code == 1:
 672             failed = True
 673         elif result_code == -1:
 674             crashed = True
 675
 676     results_per_suite.print_results()
 677
 678     if crashed:
 679         return_code = -1
 680     elif failed:
 681         return_code = 1
 682     else:
 683         return_code = 0
 684     return return_code, results_per_suite.rerun
 685
 686
 687 def parse_digit_env(env_var, default):
 688     value = os.getenv(env_var, default)
 689     if value != default:
 690         if value.isdigit():
 691             value = int(value)
 692         else:
 693             print('WARNING: unsupported value "%s" for env var "%s",'
 694                   'defaulting to %s' % (value, env_var, default))
 695             value = default
 696     return value
 697
 698
 699 if __name__ == '__main__':
 700
 701     verbose = parse_digit_env("V", 0)
 702
 703     test_timeout = parse_digit_env("TIMEOUT", 600)  # default = 10 minutes
 704
 705     retries = parse_digit_env("RETRIES", 0)
 706
 707     debug = os.getenv("DEBUG", "n").lower() in ["gdb", "gdbserver"]
 708
 709     debug_core = os.getenv("DEBUG", "").lower() == "core"
 710
 711     step = os.getenv("STEP", "n").lower() in ("y", "yes", "1")
 712
 713     run_interactive = debug or step
 714
 715     test_jobs = os.getenv("TEST_JOBS", "1").lower()  # default = 1 process
 716     if test_jobs == 'auto':
 717         if run_interactive:
 718             concurrent_tests = 1
 719             print('Interactive mode required, running on one core')
 720         else:
 721             shm_free = psutil.disk_usage('/dev/shm').free
 722             shm_max_processes = 1
 723             if shm_free < min_req_shm:
 724                 raise Exception('Not enough free space in /dev/shm. Required '
 725                                 'free space is at least %sM.'
 726                                 % (min_req_shm >> 20))
 727             else:
 728                 extra_shm = shm_free - min_req_shm
 729                 shm_max_processes += extra_shm / shm_per_process
 730             concurrent_tests = min(cpu_count(), shm_max_processes)
 731             print('Found enough resources to run tests with %s cores'
 732                   % concurrent_tests)
 733     elif test_jobs.isdigit():
 734         concurrent_tests = int(test_jobs)
 735     else:
 736         concurrent_tests = 1
 737
 738     if run_interactive and concurrent_tests > 1:
 739         raise NotImplementedError(
 740             'Running tests interactively (DEBUG is gdb or gdbserver or STEP '
 741             'is set) in parallel (TEST_JOBS is more than 1) is not supported')
 742
 743     parser = argparse.ArgumentParser(description="VPP unit tests")
 744     parser.add_argument("-f", "--failfast", action='store_true',
 745                         help="fast failure flag")
 746     parser.add_argument("-d", "--dir", action='append', type=str,
 747                         help="directory containing test files "
 748                              "(may be specified multiple times)")
 749     args = parser.parse_args()
 750     failfast = args.failfast
 751     descriptions = True
 752
 753     print("Running tests using custom test runner")  # debug message
 754     filter_file, filter_class, filter_func = parse_test_option()
 755
 756     print("Active filters: file=%s, class=%s, function=%s" % (
 757         filter_file, filter_class, filter_func))
 758
 759     filter_cb = FilterByTestOption(filter_file, filter_class, filter_func)
 760
 761     ignore_path = os.getenv("VENV_PATH", None)
 762     cb = SplitToSuitesCallback(filter_cb)
 763     for d in args.dir:
 764         print("Adding tests from directory tree %s" % d)
 765         discover_tests(d, cb, ignore_path)
 766
 767     # suites are not hashable, need to use list
 768     suites = []
 769     tests_amount = 0
 770     for testcase_suite in cb.suites.values():
 771         tests_amount += testcase_suite.countTestCases()
 772         suites.append(testcase_suite)
 773
 774     print("%s out of %s tests match specified filters" % (
 775         tests_amount, tests_amount + cb.filtered.countTestCases()))
 776
 777     if not running_extended_tests():
 778         print("Not running extended tests (some tests will be skipped)")
 779
 780     attempts = retries + 1
 781     if attempts > 1:
 782         print("Perform %s attempts to pass the suite..." % attempts)
 783
 784     if run_interactive:
 785         # don't fork if requiring interactive terminal
 786         result = VppTestRunner(verbosity=verbose,
 787                                failfast=failfast,
 788                                print_summary=True).run(suites[0])
 789         was_successful = result.wasSuccessful()
 790         if not was_successful:
 791             for test_case_info in result.failed_test_cases_info:
 792                 handle_failed_suite(test_case_info.logger,
 793                                     test_case_info.tempdir,
 794                                     test_case_info.vpp_pid)
 795                 if debug_core and \
 796                         test_case_info in result.core_crash_test_cases_info:
 797                     check_and_handle_core(test_case_info.vpp_bin_path,
 798                                           test_case_info.tempdir,
 799                                           test_case_info.core_crash_test)
 800
 801         sys.exit(not was_successful)
 802     else:
 803         exit_code = 0
 804         while len(suites) > 0 and attempts > 0:
 805             results = run_forked(suites)
 806             exit_code, suites = parse_results(results)
 807             attempts -= 1
 808             if exit_code == 0:
 809                 print('Test run was successful')
 810             else:
 811                 print('%s attempt(s) left.' % attempts)
 812         sys.exit(exit_code)