test/run_tests.py

   1 #!/usr/bin/env python3
   2
   3 import sys
   4 import shutil
   5 import os
   6 import fnmatch
   7 import unittest
   8 import argparse
   9 import time
  10 import threading
  11 import signal
  12 import psutil
  13 import re
  14 import multiprocessing
  15 from multiprocessing import Process, Pipe, cpu_count
  16 from multiprocessing.queues import Queue
  17 from multiprocessing.managers import BaseManager
  18 import framework
  19 from framework import VppTestRunner, running_extended_tests, VppTestCase, \
  20     get_testcase_doc_name, get_test_description, PASS, FAIL, ERROR, SKIP, \
  21     TEST_RUN
  22 from debug import spawn_gdb
  23 from log import get_parallel_logger, double_line_delim, RED, YELLOW, GREEN, \
  24     colorize, single_line_delim
  25 from discover_tests import discover_tests
  26 from subprocess import check_output, CalledProcessError
  27 from util import check_core_path, get_core_path, is_core_present
  28
  29 # timeout which controls how long the child has to finish after seeing
  30 # a core dump in test temporary directory. If this is exceeded, parent assumes
  31 # that child process is stuck (e.g. waiting for event from vpp) and kill
  32 # the child
  33 core_timeout = 3
  34
  35
  36 class StreamQueue(Queue):
  37     def write(self, msg):
  38         self.put(msg)
  39
  40     def flush(self):
  41         sys.__stdout__.flush()
  42         sys.__stderr__.flush()
  43
  44     def fileno(self):
  45         return self._writer.fileno()
  46
  47
  48 class StreamQueueManager(BaseManager):
  49     pass
  50
  51
  52 StreamQueueManager.register('StreamQueue', StreamQueue)
  53
  54
  55 class TestResult(dict):
  56     def __init__(self, testcase_suite, testcases_by_id=None):
  57         super(TestResult, self).__init__()
  58         self[PASS] = []
  59         self[FAIL] = []
  60         self[ERROR] = []
  61         self[SKIP] = []
  62         self[TEST_RUN] = []
  63         self.crashed = False
  64         self.testcase_suite = testcase_suite
  65         self.testcases = [testcase for testcase in testcase_suite]
  66         self.testcases_by_id = testcases_by_id
  67
  68     def was_successful(self):
  69         return 0 == len(self[FAIL]) == len(self[ERROR]) \
  70             and len(self[PASS] + self[SKIP]) \
  71             == self.testcase_suite.countTestCases() == len(self[TEST_RUN])
  72
  73     def no_tests_run(self):
  74         return 0 == len(self[TEST_RUN])
  75
  76     def process_result(self, test_id, result):
  77         self[result].append(test_id)
  78
  79     def suite_from_failed(self):
  80         rerun_ids = set([])
  81         for testcase in self.testcase_suite:
  82             tc_id = testcase.id()
  83             if tc_id not in self[PASS] and tc_id not in self[SKIP]:
  84                 rerun_ids.add(tc_id)
  85         if rerun_ids:
  86             return suite_from_failed(self.testcase_suite, rerun_ids)
  87
  88     def get_testcase_names(self, test_id):
  89         # could be tearDownClass (test_ipsec_esp.TestIpsecEsp1)
  90         setup_teardown_match = re.match(
  91             r'((tearDownClass)|(setUpClass)) \((.+\..+)\)', test_id)
  92         if setup_teardown_match:
  93             test_name, _, _, testcase_name = setup_teardown_match.groups()
  94             if len(testcase_name.split('.')) == 2:
  95                 for key in self.testcases_by_id.keys():
  96                     if key.startswith(testcase_name):
  97                         testcase_name = key
  98                         break
  99             testcase_name = self._get_testcase_doc_name(testcase_name)
 100         else:
 101             test_name = self._get_test_description(test_id)
 102             testcase_name = self._get_testcase_doc_name(test_id)
 103
 104         return testcase_name, test_name
 105
 106     def _get_test_description(self, test_id):
 107         if test_id in self.testcases_by_id:
 108             desc = get_test_description(descriptions,
 109                                         self.testcases_by_id[test_id])
 110         else:
 111             desc = test_id
 112         return desc
 113
 114     def _get_testcase_doc_name(self, test_id):
 115         if test_id in self.testcases_by_id:
 116             doc_name = get_testcase_doc_name(self.testcases_by_id[test_id])
 117         else:
 118             doc_name = test_id
 119         return doc_name
 120
 121
 122 def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue,
 123                         finished_pipe, result_pipe, logger):
 124     sys.stdout = stdouterr_queue
 125     sys.stderr = stdouterr_queue
 126     VppTestCase.parallel_handler = logger.handlers[0]
 127     result = VppTestRunner(keep_alive_pipe=keep_alive_pipe,
 128                            descriptions=descriptions,
 129                            verbosity=verbose,
 130                            result_pipe=result_pipe,
 131                            failfast=failfast,
 132                            print_summary=False).run(suite)
 133     finished_pipe.send(result.wasSuccessful())
 134     finished_pipe.close()
 135     keep_alive_pipe.close()
 136
 137
 138 class TestCaseWrapper(object):
 139     def __init__(self, testcase_suite, manager):
 140         self.keep_alive_parent_end, self.keep_alive_child_end = Pipe(
 141             duplex=False)
 142         self.finished_parent_end, self.finished_child_end = Pipe(duplex=False)
 143         self.result_parent_end, self.result_child_end = Pipe(duplex=False)
 144         self.testcase_suite = testcase_suite
 145         if sys.version[0] == '2':
 146             self.stdouterr_queue = manager.StreamQueue()
 147         else:
 148             from multiprocessing import get_context
 149             self.stdouterr_queue = manager.StreamQueue(ctx=get_context())
 150         self.logger = get_parallel_logger(self.stdouterr_queue)
 151         self.child = Process(target=test_runner_wrapper,
 152                              args=(testcase_suite,
 153                                    self.keep_alive_child_end,
 154                                    self.stdouterr_queue,
 155                                    self.finished_child_end,
 156                                    self.result_child_end,
 157                                    self.logger)
 158                              )
 159         self.child.start()
 160         self.last_test_temp_dir = None
 161         self.last_test_vpp_binary = None
 162         self._last_test = None
 163         self.last_test_id = None
 164         self.vpp_pid = None
 165         self.last_heard = time.time()
 166         self.core_detected_at = None
 167         self.testcases_by_id = {}
 168         self.testclasess_with_core = {}
 169         for testcase in self.testcase_suite:
 170             self.testcases_by_id[testcase.id()] = testcase
 171         self.result = TestResult(testcase_suite, self.testcases_by_id)
 172
 173     @property
 174     def last_test(self):
 175         return self._last_test
 176
 177     @last_test.setter
 178     def last_test(self, test_id):
 179         self.last_test_id = test_id
 180         if test_id in self.testcases_by_id:
 181             testcase = self.testcases_by_id[test_id]
 182             self._last_test = testcase.shortDescription()
 183             if not self._last_test:
 184                 self._last_test = str(testcase)
 185         else:
 186             self._last_test = test_id
 187
 188     def add_testclass_with_core(self):
 189         if self.last_test_id in self.testcases_by_id:
 190             test = self.testcases_by_id[self.last_test_id]
 191             class_name = unittest.util.strclass(test.__class__)
 192             test_name = "'{}' ({})".format(get_test_description(descriptions,
 193                                                                 test),
 194                                            self.last_test_id)
 195         else:
 196             test_name = self.last_test_id
 197             class_name = re.match(r'((tearDownClass)|(setUpClass)) '
 198                                   r'\((.+\..+)\)', test_name).groups()[3]
 199         if class_name not in self.testclasess_with_core:
 200             self.testclasess_with_core[class_name] = (
 201                 test_name,
 202                 self.last_test_vpp_binary,
 203                 self.last_test_temp_dir)
 204
 205     def close_pipes(self):
 206         self.keep_alive_child_end.close()
 207         self.finished_child_end.close()
 208         self.result_child_end.close()
 209         self.keep_alive_parent_end.close()
 210         self.finished_parent_end.close()
 211         self.result_parent_end.close()
 212
 213     def was_successful(self):
 214         return self.result.was_successful()
 215
 216
 217 def stdouterr_reader_wrapper(unread_testcases, finished_unread_testcases,
 218                              read_testcases):
 219     read_testcase = None
 220     while read_testcases.is_set() or unread_testcases:
 221         if finished_unread_testcases:
 222             read_testcase = finished_unread_testcases.pop()
 223             unread_testcases.remove(read_testcase)
 224         elif unread_testcases:
 225             read_testcase = unread_testcases.pop()
 226         if read_testcase:
 227             data = ''
 228             while data is not None:
 229                 sys.stdout.write(data)
 230                 data = read_testcase.stdouterr_queue.get()
 231
 232             read_testcase.stdouterr_queue.close()
 233             finished_unread_testcases.discard(read_testcase)
 234             read_testcase = None
 235
 236
 237 def handle_failed_suite(logger, last_test_temp_dir, vpp_pid):
 238     if last_test_temp_dir:
 239         # Need to create link in case of a timeout or core dump without failure
 240         lttd = os.path.basename(last_test_temp_dir)
 241         failed_dir = os.getenv('FAILED_DIR')
 242         link_path = '%s%s-FAILED' % (failed_dir, lttd)
 243         if not os.path.exists(link_path):
 244             os.symlink(last_test_temp_dir, link_path)
 245         logger.error("Symlink to failed testcase directory: %s -> %s"
 246                      % (link_path, lttd))
 247
 248         # Report core existence
 249         core_path = get_core_path(last_test_temp_dir)
 250         if os.path.exists(core_path):
 251             logger.error(
 252                 "Core-file exists in test temporary directory: %s!" %
 253                 core_path)
 254             check_core_path(logger, core_path)
 255             logger.debug("Running 'file %s':" % core_path)
 256             try:
 257                 info = check_output(["file", core_path])
 258                 logger.debug(info)
 259             except CalledProcessError as e:
 260                 logger.error("Subprocess returned with return code "
 261                              "while running `file' utility on core-file "
 262                              "returned: "
 263                              "rc=%s", e.returncode)
 264             except OSError as e:
 265                 logger.error("Subprocess returned with OS error while "
 266                              "running 'file' utility "
 267                              "on core-file: "
 268                              "(%s) %s", e.errno, e.strerror)
 269             except Exception as e:
 270                 logger.exception("Unexpected error running `file' utility "
 271                                  "on core-file")
 272             logger.error("gdb %s %s" %
 273                          (os.getenv('VPP_BIN', 'vpp'), core_path))
 274
 275     if vpp_pid:
 276         # Copy api post mortem
 277         api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
 278         if os.path.isfile(api_post_mortem_path):
 279             logger.error("Copying api_post_mortem.%d to %s" %
 280                          (vpp_pid, last_test_temp_dir))
 281             shutil.copy2(api_post_mortem_path, last_test_temp_dir)
 282
 283
 284 def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
 285     if is_core_present(tempdir):
 286         if debug_core:
 287             print('VPP core detected in %s. Last test running was %s' %
 288                   (tempdir, core_crash_test))
 289             print(single_line_delim)
 290             spawn_gdb(vpp_binary, get_core_path(tempdir))
 291             print(single_line_delim)
 292         elif compress_core:
 293             print("Compressing core-file in test directory `%s'" % tempdir)
 294             os.system("gzip %s" % get_core_path(tempdir))
 295
 296
 297 def handle_cores(failed_testcases):
 298     for failed_testcase in failed_testcases:
 299         tcs_with_core = failed_testcase.testclasess_with_core
 300         if tcs_with_core:
 301             for test, vpp_binary, tempdir in tcs_with_core.values():
 302                 check_and_handle_core(vpp_binary, tempdir, test)
 303
 304
 305 def process_finished_testsuite(wrapped_testcase_suite,
 306                                finished_testcase_suites,
 307                                failed_wrapped_testcases,
 308                                results):
 309     results.append(wrapped_testcase_suite.result)
 310     finished_testcase_suites.add(wrapped_testcase_suite)
 311     stop_run = False
 312     if failfast and not wrapped_testcase_suite.was_successful():
 313         stop_run = True
 314
 315     if not wrapped_testcase_suite.was_successful():
 316         failed_wrapped_testcases.add(wrapped_testcase_suite)
 317         handle_failed_suite(wrapped_testcase_suite.logger,
 318                             wrapped_testcase_suite.last_test_temp_dir,
 319                             wrapped_testcase_suite.vpp_pid)
 320
 321     return stop_run
 322
 323
 324 def run_forked(testcase_suites):
 325     wrapped_testcase_suites = set()
 326     solo_testcase_suites = []
 327     total_test_runners = 0
 328
 329     # suites are unhashable, need to use list
 330     results = []
 331     unread_testcases = set()
 332     finished_unread_testcases = set()
 333     manager = StreamQueueManager()
 334     manager.start()
 335     total_test_runners = 0
 336     while total_test_runners < concurrent_tests:
 337         if testcase_suites:
 338             a_suite = testcase_suites.pop(0)
 339             if a_suite.is_tagged_run_solo:
 340                 solo_testcase_suites.append(a_suite)
 341                 continue
 342             wrapped_testcase_suite = TestCaseWrapper(a_suite,
 343                                                      manager)
 344             wrapped_testcase_suites.add(wrapped_testcase_suite)
 345             unread_testcases.add(wrapped_testcase_suite)
 346             total_test_runners = total_test_runners + 1
 347         else:
 348             break
 349
 350     while total_test_runners < 1 and solo_testcase_suites:
 351         if solo_testcase_suites:
 352             a_suite = solo_testcase_suites.pop(0)
 353             wrapped_testcase_suite = TestCaseWrapper(a_suite,
 354                                                      manager)
 355             wrapped_testcase_suites.add(wrapped_testcase_suite)
 356             unread_testcases.add(wrapped_testcase_suite)
 357             total_test_runners = total_test_runners + 1
 358         else:
 359             break
 360
 361     read_from_testcases = threading.Event()
 362     read_from_testcases.set()
 363     stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper,
 364                                         args=(unread_testcases,
 365                                               finished_unread_testcases,
 366                                               read_from_testcases))
 367     stdouterr_thread.start()
 368
 369     failed_wrapped_testcases = set()
 370     stop_run = False
 371
 372     try:
 373         while wrapped_testcase_suites:
 374             finished_testcase_suites = set()
 375             for wrapped_testcase_suite in wrapped_testcase_suites:
 376                 while wrapped_testcase_suite.result_parent_end.poll():
 377                     wrapped_testcase_suite.result.process_result(
 378                         *wrapped_testcase_suite.result_parent_end.recv())
 379                     wrapped_testcase_suite.last_heard = time.time()
 380
 381                 while wrapped_testcase_suite.keep_alive_parent_end.poll():
 382                     wrapped_testcase_suite.last_test, \
 383                         wrapped_testcase_suite.last_test_vpp_binary, \
 384                         wrapped_testcase_suite.last_test_temp_dir, \
 385                         wrapped_testcase_suite.vpp_pid = \
 386                         wrapped_testcase_suite.keep_alive_parent_end.recv()
 387                     wrapped_testcase_suite.last_heard = time.time()
 388
 389                 if wrapped_testcase_suite.finished_parent_end.poll():
 390                     wrapped_testcase_suite.finished_parent_end.recv()
 391                     wrapped_testcase_suite.last_heard = time.time()
 392                     stop_run = process_finished_testsuite(
 393                         wrapped_testcase_suite,
 394                         finished_testcase_suites,
 395                         failed_wrapped_testcases,
 396                         results) or stop_run
 397                     continue
 398
 399                 fail = False
 400                 if wrapped_testcase_suite.last_heard + test_timeout < \
 401                         time.time():
 402                     fail = True
 403                     wrapped_testcase_suite.logger.critical(
 404                         "Child test runner process timed out "
 405                         "(last test running was `%s' in `%s')!" %
 406                         (wrapped_testcase_suite.last_test,
 407                          wrapped_testcase_suite.last_test_temp_dir))
 408                 elif not wrapped_testcase_suite.child.is_alive():
 409                     fail = True
 410                     wrapped_testcase_suite.logger.critical(
 411                         "Child test runner process unexpectedly died "
 412                         "(last test running was `%s' in `%s')!" %
 413                         (wrapped_testcase_suite.last_test,
 414                          wrapped_testcase_suite.last_test_temp_dir))
 415                 elif wrapped_testcase_suite.last_test_temp_dir and \
 416                         wrapped_testcase_suite.last_test_vpp_binary:
 417                     if is_core_present(
 418                             wrapped_testcase_suite.last_test_temp_dir):
 419                         wrapped_testcase_suite.add_testclass_with_core()
 420                         if wrapped_testcase_suite.core_detected_at is None:
 421                             wrapped_testcase_suite.core_detected_at = \
 422                                 time.time()
 423                         elif wrapped_testcase_suite.core_detected_at + \
 424                                 core_timeout < time.time():
 425                             wrapped_testcase_suite.logger.critical(
 426                                 "Child test runner process unresponsive and "
 427                                 "core-file exists in test temporary directory "
 428                                 "(last test running was `%s' in `%s')!" %
 429                                 (wrapped_testcase_suite.last_test,
 430                                  wrapped_testcase_suite.last_test_temp_dir))
 431                             fail = True
 432
 433                 if fail:
 434                     wrapped_testcase_suite.child.terminate()
 435                     try:
 436                         # terminating the child process tends to leave orphan
 437                         # VPP process around
 438                         if wrapped_testcase_suite.vpp_pid:
 439                             os.kill(wrapped_testcase_suite.vpp_pid,
 440                                     signal.SIGTERM)
 441                     except OSError:
 442                         # already dead
 443                         pass
 444                     wrapped_testcase_suite.result.crashed = True
 445                     wrapped_testcase_suite.result.process_result(
 446                         wrapped_testcase_suite.last_test_id, ERROR)
 447                     stop_run = process_finished_testsuite(
 448                         wrapped_testcase_suite,
 449                         finished_testcase_suites,
 450                         failed_wrapped_testcases,
 451                         results) or stop_run
 452
 453             for finished_testcase in finished_testcase_suites:
 454                 # Somewhat surprisingly, the join below may
 455                 # timeout, even if client signaled that
 456                 # it finished - so we note it just in case.
 457                 join_start = time.time()
 458                 finished_testcase.child.join(test_finished_join_timeout)
 459                 join_end = time.time()
 460                 if join_end - join_start >= test_finished_join_timeout:
 461                     finished_testcase.logger.error(
 462                         "Timeout joining finished test: %s (pid %d)" %
 463                         (finished_testcase.last_test,
 464                          finished_testcase.child.pid))
 465                 finished_testcase.close_pipes()
 466                 wrapped_testcase_suites.remove(finished_testcase)
 467                 finished_unread_testcases.add(finished_testcase)
 468                 finished_testcase.stdouterr_queue.put(None)
 469                 total_test_runners = total_test_runners - 1
 470                 if stop_run:
 471                     while testcase_suites:
 472                         results.append(TestResult(testcase_suites.pop(0)))
 473                 elif testcase_suites:
 474                     a_testcase = testcase_suites.pop(0)
 475                     while a_testcase and a_testcase.is_tagged_run_solo:
 476                         solo_testcase_suites.append(a_testcase)
 477                         if testcase_suites:
 478                             a_testcase = testcase_suites.pop(0)
 479                         else:
 480                             a_testcase = None
 481                     if a_testcase:
 482                         new_testcase = TestCaseWrapper(a_testcase,
 483                                                        manager)
 484                         wrapped_testcase_suites.add(new_testcase)
 485                         total_test_runners = total_test_runners + 1
 486                         unread_testcases.add(new_testcase)
 487                 if solo_testcase_suites and total_test_runners == 0:
 488                     a_testcase = solo_testcase_suites.pop(0)
 489                     new_testcase = TestCaseWrapper(a_testcase,
 490                                                    manager)
 491                     wrapped_testcase_suites.add(new_testcase)
 492                     total_test_runners = total_test_runners + 1
 493                     unread_testcases.add(new_testcase)
 494             time.sleep(0.1)
 495     except Exception:
 496         for wrapped_testcase_suite in wrapped_testcase_suites:
 497             wrapped_testcase_suite.child.terminate()
 498             wrapped_testcase_suite.stdouterr_queue.put(None)
 499         raise
 500     finally:
 501         read_from_testcases.clear()
 502         stdouterr_thread.join(test_timeout)
 503         manager.shutdown()
 504
 505     handle_cores(failed_wrapped_testcases)
 506     return results
 507
 508
 509 class SplitToSuitesCallback:
 510     def __init__(self, filter_callback):
 511         self.suites = {}
 512         self.suite_name = 'default'
 513         self.filter_callback = filter_callback
 514         self.filtered = unittest.TestSuite()
 515
 516     def __call__(self, file_name, cls, method):
 517         test_method = cls(method)
 518         if self.filter_callback(file_name, cls.__name__, method):
 519             self.suite_name = file_name + cls.__name__
 520             if self.suite_name not in self.suites:
 521                 self.suites[self.suite_name] = unittest.TestSuite()
 522                 self.suites[self.suite_name].is_tagged_run_solo = False
 523             self.suites[self.suite_name].addTest(test_method)
 524             if test_method.is_tagged_run_solo():
 525                 self.suites[self.suite_name].is_tagged_run_solo = True
 526
 527         else:
 528             self.filtered.addTest(test_method)
 529
 530
 531 test_option = "TEST"
 532
 533
 534 def parse_test_option():
 535     f = os.getenv(test_option, None)
 536     filter_file_name = None
 537     filter_class_name = None
 538     filter_func_name = None
 539     if f:
 540         if '.' in f:
 541             parts = f.split('.')
 542             if len(parts) > 3:
 543                 raise Exception("Unrecognized %s option: %s" %
 544                                 (test_option, f))
 545             if len(parts) > 2:
 546                 if parts[2] not in ('*', ''):
 547                     filter_func_name = parts[2]
 548             if parts[1] not in ('*', ''):
 549                 filter_class_name = parts[1]
 550             if parts[0] not in ('*', ''):
 551                 if parts[0].startswith('test_'):
 552                     filter_file_name = parts[0]
 553                 else:
 554                     filter_file_name = 'test_%s' % parts[0]
 555         else:
 556             if f.startswith('test_'):
 557                 filter_file_name = f
 558             else:
 559                 filter_file_name = 'test_%s' % f
 560     if filter_file_name:
 561         filter_file_name = '%s.py' % filter_file_name
 562     return filter_file_name, filter_class_name, filter_func_name
 563
 564
 565 def filter_tests(tests, filter_cb):
 566     result = unittest.suite.TestSuite()
 567     for t in tests:
 568         if isinstance(t, unittest.suite.TestSuite):
 569             # this is a bunch of tests, recursively filter...
 570             x = filter_tests(t, filter_cb)
 571             if x.countTestCases() > 0:
 572                 result.addTest(x)
 573         elif isinstance(t, unittest.TestCase):
 574             # this is a single test
 575             parts = t.id().split('.')
 576             # t.id() for common cases like this:
 577             # test_classifier.TestClassifier.test_acl_ip
 578             # apply filtering only if it is so
 579             if len(parts) == 3:
 580                 if not filter_cb(parts[0], parts[1], parts[2]):
 581                     continue
 582             result.addTest(t)
 583         else:
 584             # unexpected object, don't touch it
 585             result.addTest(t)
 586     return result
 587
 588
 589 class FilterByTestOption:
 590     def __init__(self, filter_file_name, filter_class_name, filter_func_name):
 591         self.filter_file_name = filter_file_name
 592         self.filter_class_name = filter_class_name
 593         self.filter_func_name = filter_func_name
 594
 595     def __call__(self, file_name, class_name, func_name):
 596         if self.filter_file_name:
 597             fn_match = fnmatch.fnmatch(file_name, self.filter_file_name)
 598             if not fn_match:
 599                 return False
 600         if self.filter_class_name and class_name != self.filter_class_name:
 601             return False
 602         if self.filter_func_name and func_name != self.filter_func_name:
 603             return False
 604         return True
 605
 606
 607 class FilterByClassList:
 608     def __init__(self, classes_with_filenames):
 609         self.classes_with_filenames = classes_with_filenames
 610
 611     def __call__(self, file_name, class_name, func_name):
 612         return '.'.join([file_name, class_name]) in self.classes_with_filenames
 613
 614
 615 def suite_from_failed(suite, failed):
 616     failed = {x.rsplit('.', 1)[0] for x in failed}
 617     filter_cb = FilterByClassList(failed)
 618     suite = filter_tests(suite, filter_cb)
 619     return suite
 620
 621
 622 class AllResults(dict):
 623     def __init__(self):
 624         super(AllResults, self).__init__()
 625         self.all_testcases = 0
 626         self.results_per_suite = []
 627         self[PASS] = 0
 628         self[FAIL] = 0
 629         self[ERROR] = 0
 630         self[SKIP] = 0
 631         self[TEST_RUN] = 0
 632         self.rerun = []
 633         self.testsuites_no_tests_run = []
 634
 635     def add_results(self, result):
 636         self.results_per_suite.append(result)
 637         result_types = [PASS, FAIL, ERROR, SKIP, TEST_RUN]
 638         for result_type in result_types:
 639             self[result_type] += len(result[result_type])
 640
 641     def add_result(self, result):
 642         retval = 0
 643         self.all_testcases += result.testcase_suite.countTestCases()
 644         self.add_results(result)
 645
 646         if result.no_tests_run():
 647             self.testsuites_no_tests_run.append(result.testcase_suite)
 648             if result.crashed:
 649                 retval = -1
 650             else:
 651                 retval = 1
 652         elif not result.was_successful():
 653             retval = 1
 654
 655         if retval != 0:
 656             self.rerun.append(result.testcase_suite)
 657
 658         return retval
 659
 660     def print_results(self):
 661         print('')
 662         print(double_line_delim)
 663         print('TEST RESULTS:')
 664         print('     Scheduled tests: {}'.format(self.all_testcases))
 665         print('      Executed tests: {}'.format(self[TEST_RUN]))
 666         print('        Passed tests: {}'.format(
 667             colorize(str(self[PASS]), GREEN)))
 668         if self[SKIP] > 0:
 669             print('       Skipped tests: {}'.format(
 670                 colorize(str(self[SKIP]), YELLOW)))
 671         if self.not_executed > 0:
 672             print('  Not Executed tests: {}'.format(
 673                 colorize(str(self.not_executed), RED)))
 674         if self[FAIL] > 0:
 675             print('            Failures: {}'.format(
 676                 colorize(str(self[FAIL]), RED)))
 677         if self[ERROR] > 0:
 678             print('              Errors: {}'.format(
 679                 colorize(str(self[ERROR]), RED)))
 680
 681         if self.all_failed > 0:
 682             print('FAILURES AND ERRORS IN TESTS:')
 683             for result in self.results_per_suite:
 684                 failed_testcase_ids = result[FAIL]
 685                 errored_testcase_ids = result[ERROR]
 686                 old_testcase_name = None
 687                 if failed_testcase_ids:
 688                     for failed_test_id in failed_testcase_ids:
 689                         new_testcase_name, test_name = \
 690                             result.get_testcase_names(failed_test_id)
 691                         if new_testcase_name != old_testcase_name:
 692                             print('  Testcase name: {}'.format(
 693                                 colorize(new_testcase_name, RED)))
 694                             old_testcase_name = new_testcase_name
 695                         print('    FAILURE: {} [{}]'.format(
 696                             colorize(test_name, RED), failed_test_id))
 697                 if errored_testcase_ids:
 698                     for errored_test_id in errored_testcase_ids:
 699                         new_testcase_name, test_name = \
 700                             result.get_testcase_names(errored_test_id)
 701                         if new_testcase_name != old_testcase_name:
 702                             print('  Testcase name: {}'.format(
 703                                 colorize(new_testcase_name, RED)))
 704                             old_testcase_name = new_testcase_name
 705                         print('      ERROR: {} [{}]'.format(
 706                             colorize(test_name, RED), errored_test_id))
 707         if self.testsuites_no_tests_run:
 708             print('TESTCASES WHERE NO TESTS WERE SUCCESSFULLY EXECUTED:')
 709             tc_classes = set()
 710             for testsuite in self.testsuites_no_tests_run:
 711                 for testcase in testsuite:
 712                     tc_classes.add(get_testcase_doc_name(testcase))
 713             for tc_class in tc_classes:
 714                 print('  {}'.format(colorize(tc_class, RED)))
 715
 716         print(double_line_delim)
 717         print('')
 718
 719     @property
 720     def not_executed(self):
 721         return self.all_testcases - self[TEST_RUN]
 722
 723     @property
 724     def all_failed(self):
 725         return self[FAIL] + self[ERROR]
 726
 727
 728 def parse_results(results):
 729     """
 730     Prints the number of scheduled, executed, not executed, passed, failed,
 731     errored and skipped tests and details about failed and errored tests.
 732
 733     Also returns all suites where any test failed.
 734
 735     :param results:
 736     :return:
 737     """
 738
 739     results_per_suite = AllResults()
 740     crashed = False
 741     failed = False
 742     for result in results:
 743         result_code = results_per_suite.add_result(result)
 744         if result_code == 1:
 745             failed = True
 746         elif result_code == -1:
 747             crashed = True
 748
 749     results_per_suite.print_results()
 750
 751     if crashed:
 752         return_code = -1
 753     elif failed:
 754         return_code = 1
 755     else:
 756         return_code = 0
 757     return return_code, results_per_suite.rerun
 758
 759
 760 def parse_digit_env(env_var, default):
 761     value = os.getenv(env_var, default)
 762     if value != default:
 763         if value.isdigit():
 764             value = int(value)
 765         else:
 766             print('WARNING: unsupported value "%s" for env var "%s",'
 767                   'defaulting to %s' % (value, env_var, default))
 768             value = default
 769     return value
 770
 771
 772 if __name__ == '__main__':
 773
 774     verbose = parse_digit_env("V", 0)
 775
 776     test_timeout = parse_digit_env("TIMEOUT", 600)  # default = 10 minutes
 777
 778     test_finished_join_timeout = 15
 779
 780     retries = parse_digit_env("RETRIES", 0)
 781
 782     debug = os.getenv("DEBUG", "n").lower() in ["gdb", "gdbserver"]
 783
 784     debug_core = os.getenv("DEBUG", "").lower() == "core"
 785     compress_core = framework.BoolEnvironmentVariable("CORE_COMPRESS")
 786
 787     step = framework.BoolEnvironmentVariable("STEP")
 788     force_foreground = framework.BoolEnvironmentVariable("FORCE_FOREGROUND")
 789
 790     run_interactive = debug or step or force_foreground
 791
 792     try:
 793         num_cpus = len(os.sched_getaffinity(0))
 794     except AttributeError:
 795         num_cpus = multiprocessing.cpu_count()
 796
 797     print("OS reports %s available cpu(s)." % num_cpus)
 798
 799     test_jobs = os.getenv("TEST_JOBS", "1").lower()  # default = 1 process
 800     if test_jobs == 'auto':
 801         if run_interactive:
 802             concurrent_tests = 1
 803             print('Interactive mode required, running on one core')
 804         else:
 805             concurrent_tests = num_cpus
 806             print('Found enough resources to run tests with %s cores'
 807                   % concurrent_tests)
 808     elif test_jobs.isdigit():
 809         concurrent_tests = int(test_jobs)
 810         print("Running on %s core(s) as set by 'TEST_JOBS'." %
 811               concurrent_tests)
 812     else:
 813         concurrent_tests = 1
 814         print('Running on one core.')
 815
 816     if run_interactive and concurrent_tests > 1:
 817         raise NotImplementedError(
 818             'Running tests interactively (DEBUG is gdb or gdbserver or STEP '
 819             'is set) in parallel (TEST_JOBS is more than 1) is not supported')
 820
 821     parser = argparse.ArgumentParser(description="VPP unit tests")
 822     parser.add_argument("-f", "--failfast", action='store_true',
 823                         help="fast failure flag")
 824     parser.add_argument("-d", "--dir", action='append', type=str,
 825                         help="directory containing test files "
 826                              "(may be specified multiple times)")
 827     args = parser.parse_args()
 828     failfast = args.failfast
 829     descriptions = True
 830
 831     print("Running tests using custom test runner")  # debug message
 832     filter_file, filter_class, filter_func = parse_test_option()
 833
 834     print("Active filters: file=%s, class=%s, function=%s" % (
 835         filter_file, filter_class, filter_func))
 836
 837     filter_cb = FilterByTestOption(filter_file, filter_class, filter_func)
 838
 839     ignore_path = os.getenv("VENV_PATH", None)
 840     cb = SplitToSuitesCallback(filter_cb)
 841     for d in args.dir:
 842         print("Adding tests from directory tree %s" % d)
 843         discover_tests(d, cb, ignore_path)
 844
 845     # suites are not hashable, need to use list
 846     suites = []
 847     tests_amount = 0
 848     for testcase_suite in cb.suites.values():
 849         tests_amount += testcase_suite.countTestCases()
 850         suites.append(testcase_suite)
 851
 852     print("%s out of %s tests match specified filters" % (
 853         tests_amount, tests_amount + cb.filtered.countTestCases()))
 854
 855     if not running_extended_tests:
 856         print("Not running extended tests (some tests will be skipped)")
 857
 858     attempts = retries + 1
 859     if attempts > 1:
 860         print("Perform %s attempts to pass the suite..." % attempts)
 861
 862     if run_interactive and suites:
 863         # don't fork if requiring interactive terminal
 864         print('Running tests in foreground in the current process')
 865         full_suite = unittest.TestSuite()
 866         full_suite.addTests(suites)
 867         result = VppTestRunner(verbosity=verbose,
 868                                failfast=failfast,
 869                                print_summary=True).run(full_suite)
 870         was_successful = result.wasSuccessful()
 871         if not was_successful:
 872             for test_case_info in result.failed_test_cases_info:
 873                 handle_failed_suite(test_case_info.logger,
 874                                     test_case_info.tempdir,
 875                                     test_case_info.vpp_pid)
 876                 if test_case_info in result.core_crash_test_cases_info:
 877                     check_and_handle_core(test_case_info.vpp_bin_path,
 878                                           test_case_info.tempdir,
 879                                           test_case_info.core_crash_test)
 880
 881         sys.exit(not was_successful)
 882     else:
 883         print('Running each VPPTestCase in a separate background process'
 884               ' with {} parallel process(es)'.format(concurrent_tests))
 885         exit_code = 0
 886         while suites and attempts > 0:
 887             results = run_forked(suites)
 888             exit_code, suites = parse_results(results)
 889             attempts -= 1
 890             if exit_code == 0:
 891                 print('Test run was successful')
 892             else:
 893                 print('%s attempt(s) left.' % attempts)
 894         sys.exit(exit_code)