test/run_tests.py

   1 #!/usr/bin/env python
   2
   3 import sys
   4 import shutil
   5 import os
   6 import fnmatch
   7 import unittest
   8 import argparse
   9 import time
  10 import threading
  11 import signal
  12 import psutil
  13 import re
  14 from multiprocessing import Process, Pipe, cpu_count
  15 from multiprocessing.queues import Queue
  16 from multiprocessing.managers import BaseManager
  17 import framework
  18 from framework import VppTestRunner, running_extended_tests, VppTestCase, \
  19     get_testcase_doc_name, get_test_description, PASS, FAIL, ERROR, SKIP, \
  20     TEST_RUN
  21 from debug import spawn_gdb
  22 from log import get_parallel_logger, double_line_delim, RED, YELLOW, GREEN, \
  23     colorize, single_line_delim
  24 from discover_tests import discover_tests
  25 from subprocess import check_output, CalledProcessError
  26 from util import check_core_path, get_core_path, is_core_present
  27
  28 # timeout which controls how long the child has to finish after seeing
  29 # a core dump in test temporary directory. If this is exceeded, parent assumes
  30 # that child process is stuck (e.g. waiting for shm mutex, which will never
  31 # get unlocked) and kill the child
  32 core_timeout = 3
  33 min_req_shm = 536870912  # min 512MB shm required
  34 # 128MB per extra process
  35 shm_per_process = 134217728
  36
  37
  38 class StreamQueue(Queue):
  39     def write(self, msg):
  40         self.put(msg)
  41
  42     def flush(self):
  43         sys.__stdout__.flush()
  44         sys.__stderr__.flush()
  45
  46     def fileno(self):
  47         return self._writer.fileno()
  48
  49
  50 class StreamQueueManager(BaseManager):
  51     pass
  52
  53
  54 StreamQueueManager.register('StreamQueue', StreamQueue)
  55
  56
  57 class TestResult(dict):
  58     def __init__(self, testcase_suite, testcases_by_id=None):
  59         super(TestResult, self).__init__()
  60         self[PASS] = []
  61         self[FAIL] = []
  62         self[ERROR] = []
  63         self[SKIP] = []
  64         self[TEST_RUN] = []
  65         self.crashed = False
  66         self.testcase_suite = testcase_suite
  67         self.testcases = [testcase for testcase in testcase_suite]
  68         self.testcases_by_id = testcases_by_id
  69
  70     def was_successful(self):
  71         return 0 == len(self[FAIL]) == len(self[ERROR]) \
  72             and len(self[PASS] + self[SKIP]) \
  73             == self.testcase_suite.countTestCases() == len(self[TEST_RUN])
  74
  75     def no_tests_run(self):
  76         return 0 == len(self[TEST_RUN])
  77
  78     def process_result(self, test_id, result):
  79         self[result].append(test_id)
  80
  81     def suite_from_failed(self):
  82         rerun_ids = set([])
  83         for testcase in self.testcase_suite:
  84             tc_id = testcase.id()
  85             if tc_id not in self[PASS] and tc_id not in self[SKIP]:
  86                 rerun_ids.add(tc_id)
  87         if rerun_ids:
  88             return suite_from_failed(self.testcase_suite, rerun_ids)
  89
  90     def get_testcase_names(self, test_id):
  91         # could be tearDownClass (test_ipsec_esp.TestIpsecEsp1)
  92         setup_teardown_match = re.match(
  93             r'((tearDownClass)|(setUpClass)) \((.+\..+)\)', test_id)
  94         if setup_teardown_match:
  95             test_name, _, _, testcase_name = setup_teardown_match.groups()
  96             if len(testcase_name.split('.')) == 2:
  97                 for key in self.testcases_by_id.keys():
  98                     if key.startswith(testcase_name):
  99                         testcase_name = key
 100                         break
 101             testcase_name = self._get_testcase_doc_name(testcase_name)
 102         else:
 103             test_name = self._get_test_description(test_id)
 104             testcase_name = self._get_testcase_doc_name(test_id)
 105
 106         return testcase_name, test_name
 107
 108     def _get_test_description(self, test_id):
 109         if test_id in self.testcases_by_id:
 110             desc = get_test_description(descriptions,
 111                                         self.testcases_by_id[test_id])
 112         else:
 113             desc = test_id
 114         return desc
 115
 116     def _get_testcase_doc_name(self, test_id):
 117         if test_id in self.testcases_by_id:
 118             doc_name = get_testcase_doc_name(self.testcases_by_id[test_id])
 119         else:
 120             doc_name = test_id
 121         return doc_name
 122
 123
 124 def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue,
 125                         finished_pipe, result_pipe, logger):
 126     sys.stdout = stdouterr_queue
 127     sys.stderr = stdouterr_queue
 128     VppTestCase.parallel_handler = logger.handlers[0]
 129     result = VppTestRunner(keep_alive_pipe=keep_alive_pipe,
 130                            descriptions=descriptions,
 131                            verbosity=verbose,
 132                            result_pipe=result_pipe,
 133                            failfast=failfast,
 134                            print_summary=False).run(suite)
 135     finished_pipe.send(result.wasSuccessful())
 136     finished_pipe.close()
 137     keep_alive_pipe.close()
 138
 139
 140 class TestCaseWrapper(object):
 141     def __init__(self, testcase_suite, manager):
 142         self.keep_alive_parent_end, self.keep_alive_child_end = Pipe(
 143             duplex=False)
 144         self.finished_parent_end, self.finished_child_end = Pipe(duplex=False)
 145         self.result_parent_end, self.result_child_end = Pipe(duplex=False)
 146         self.testcase_suite = testcase_suite
 147         if sys.version[0] == '2':
 148             self.stdouterr_queue = manager.StreamQueue()
 149         else:
 150             from multiprocessing import get_context
 151             self.stdouterr_queue = manager.StreamQueue(ctx=get_context())
 152         self.logger = get_parallel_logger(self.stdouterr_queue)
 153         self.child = Process(target=test_runner_wrapper,
 154                              args=(testcase_suite,
 155                                    self.keep_alive_child_end,
 156                                    self.stdouterr_queue,
 157                                    self.finished_child_end,
 158                                    self.result_child_end,
 159                                    self.logger)
 160                              )
 161         self.child.start()
 162         self.last_test_temp_dir = None
 163         self.last_test_vpp_binary = None
 164         self._last_test = None
 165         self.last_test_id = None
 166         self.vpp_pid = None
 167         self.last_heard = time.time()
 168         self.core_detected_at = None
 169         self.testcases_by_id = {}
 170         self.testclasess_with_core = {}
 171         for testcase in self.testcase_suite:
 172             self.testcases_by_id[testcase.id()] = testcase
 173         self.result = TestResult(testcase_suite, self.testcases_by_id)
 174
 175     @property
 176     def last_test(self):
 177         return self._last_test
 178
 179     @last_test.setter
 180     def last_test(self, test_id):
 181         self.last_test_id = test_id
 182         if test_id in self.testcases_by_id:
 183             testcase = self.testcases_by_id[test_id]
 184             self._last_test = testcase.shortDescription()
 185             if not self._last_test:
 186                 self._last_test = str(testcase)
 187         else:
 188             self._last_test = test_id
 189
 190     def add_testclass_with_core(self):
 191         if self.last_test_id in self.testcases_by_id:
 192             test = self.testcases_by_id[self.last_test_id]
 193             class_name = unittest.util.strclass(test.__class__)
 194             test_name = "'{}' ({})".format(get_test_description(descriptions,
 195                                                                 test),
 196                                            self.last_test_id)
 197         else:
 198             test_name = self.last_test_id
 199             class_name = re.match(r'((tearDownClass)|(setUpClass)) '
 200                                   r'\((.+\..+)\)', test_name).groups()[3]
 201         if class_name not in self.testclasess_with_core:
 202             self.testclasess_with_core[class_name] = (
 203                 test_name,
 204                 self.last_test_vpp_binary,
 205                 self.last_test_temp_dir)
 206
 207     def close_pipes(self):
 208         self.keep_alive_child_end.close()
 209         self.finished_child_end.close()
 210         self.result_child_end.close()
 211         self.keep_alive_parent_end.close()
 212         self.finished_parent_end.close()
 213         self.result_parent_end.close()
 214
 215     def was_successful(self):
 216         return self.result.was_successful()
 217
 218
 219 def stdouterr_reader_wrapper(unread_testcases, finished_unread_testcases,
 220                              read_testcases):
 221     read_testcase = None
 222     while read_testcases.is_set() or unread_testcases:
 223         if finished_unread_testcases:
 224             read_testcase = finished_unread_testcases.pop()
 225             unread_testcases.remove(read_testcase)
 226         elif unread_testcases:
 227             read_testcase = unread_testcases.pop()
 228         if read_testcase:
 229             data = ''
 230             while data is not None:
 231                 sys.stdout.write(data)
 232                 data = read_testcase.stdouterr_queue.get()
 233
 234             read_testcase.stdouterr_queue.close()
 235             finished_unread_testcases.discard(read_testcase)
 236             read_testcase = None
 237
 238
 239 def handle_failed_suite(logger, last_test_temp_dir, vpp_pid):
 240     if last_test_temp_dir:
 241         # Need to create link in case of a timeout or core dump without failure
 242         lttd = os.path.basename(last_test_temp_dir)
 243         failed_dir = os.getenv('FAILED_DIR')
 244         link_path = '%s%s-FAILED' % (failed_dir, lttd)
 245         if not os.path.exists(link_path):
 246             os.symlink(last_test_temp_dir, link_path)
 247         logger.error("Symlink to failed testcase directory: %s -> %s"
 248                      % (link_path, lttd))
 249
 250         # Report core existence
 251         core_path = get_core_path(last_test_temp_dir)
 252         if os.path.exists(core_path):
 253             logger.error(
 254                 "Core-file exists in test temporary directory: %s!" %
 255                 core_path)
 256             check_core_path(logger, core_path)
 257             logger.debug("Running 'file %s':" % core_path)
 258             try:
 259                 info = check_output(["file", core_path])
 260                 logger.debug(info)
 261             except CalledProcessError as e:
 262                 logger.error("Subprocess returned with return code "
 263                              "while running `file' utility on core-file "
 264                              "returned: "
 265                              "rc=%s", e.returncode)
 266             except OSError as e:
 267                 logger.error("Subprocess returned with OS error while "
 268                              "running 'file' utility "
 269                              "on core-file: "
 270                              "(%s) %s", e.errno, e.strerror)
 271             except Exception as e:
 272                 logger.exception("Unexpected error running `file' utility "
 273                                  "on core-file")
 274
 275     if vpp_pid:
 276         # Copy api post mortem
 277         api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
 278         if os.path.isfile(api_post_mortem_path):
 279             logger.error("Copying api_post_mortem.%d to %s" %
 280                          (vpp_pid, last_test_temp_dir))
 281             shutil.copy2(api_post_mortem_path, last_test_temp_dir)
 282
 283
 284 def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
 285     if is_core_present(tempdir):
 286         if debug_core:
 287             print('VPP core detected in %s. Last test running was %s' %
 288                   (tempdir, core_crash_test))
 289             print(single_line_delim)
 290             spawn_gdb(vpp_binary, get_core_path(tempdir))
 291             print(single_line_delim)
 292         elif compress_core:
 293             print("Compressing core-file in test directory `%s'" % tempdir)
 294             os.system("gzip %s" % get_core_path(tempdir))
 295
 296
 297 def handle_cores(failed_testcases):
 298     for failed_testcase in failed_testcases:
 299         tcs_with_core = failed_testcase.testclasess_with_core
 300         if tcs_with_core:
 301             for test, vpp_binary, tempdir in tcs_with_core.values():
 302                 check_and_handle_core(vpp_binary, tempdir, test)
 303
 304
 305 def process_finished_testsuite(wrapped_testcase_suite,
 306                                finished_testcase_suites,
 307                                failed_wrapped_testcases,
 308                                results):
 309     results.append(wrapped_testcase_suite.result)
 310     finished_testcase_suites.add(wrapped_testcase_suite)
 311     stop_run = False
 312     if failfast and not wrapped_testcase_suite.was_successful():
 313         stop_run = True
 314
 315     if not wrapped_testcase_suite.was_successful():
 316         failed_wrapped_testcases.add(wrapped_testcase_suite)
 317         handle_failed_suite(wrapped_testcase_suite.logger,
 318                             wrapped_testcase_suite.last_test_temp_dir,
 319                             wrapped_testcase_suite.vpp_pid)
 320
 321     return stop_run
 322
 323
 324 def run_forked(testcase_suites):
 325     wrapped_testcase_suites = set()
 326
 327     # suites are unhashable, need to use list
 328     results = []
 329     unread_testcases = set()
 330     finished_unread_testcases = set()
 331     manager = StreamQueueManager()
 332     manager.start()
 333     for i in range(concurrent_tests):
 334         if testcase_suites:
 335             wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0),
 336                                                      manager)
 337             wrapped_testcase_suites.add(wrapped_testcase_suite)
 338             unread_testcases.add(wrapped_testcase_suite)
 339         else:
 340             break
 341
 342     read_from_testcases = threading.Event()
 343     read_from_testcases.set()
 344     stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper,
 345                                         args=(unread_testcases,
 346                                               finished_unread_testcases,
 347                                               read_from_testcases))
 348     stdouterr_thread.start()
 349
 350     failed_wrapped_testcases = set()
 351     stop_run = False
 352
 353     try:
 354         while wrapped_testcase_suites:
 355             finished_testcase_suites = set()
 356             for wrapped_testcase_suite in wrapped_testcase_suites:
 357                 while wrapped_testcase_suite.result_parent_end.poll():
 358                     wrapped_testcase_suite.result.process_result(
 359                         *wrapped_testcase_suite.result_parent_end.recv())
 360                     wrapped_testcase_suite.last_heard = time.time()
 361
 362                 while wrapped_testcase_suite.keep_alive_parent_end.poll():
 363                     wrapped_testcase_suite.last_test, \
 364                         wrapped_testcase_suite.last_test_vpp_binary, \
 365                         wrapped_testcase_suite.last_test_temp_dir, \
 366                         wrapped_testcase_suite.vpp_pid = \
 367                         wrapped_testcase_suite.keep_alive_parent_end.recv()
 368                     wrapped_testcase_suite.last_heard = time.time()
 369
 370                 if wrapped_testcase_suite.finished_parent_end.poll():
 371                     wrapped_testcase_suite.finished_parent_end.recv()
 372                     wrapped_testcase_suite.last_heard = time.time()
 373                     stop_run = process_finished_testsuite(
 374                         wrapped_testcase_suite,
 375                         finished_testcase_suites,
 376                         failed_wrapped_testcases,
 377                         results) or stop_run
 378                     continue
 379
 380                 fail = False
 381                 if wrapped_testcase_suite.last_heard + test_timeout < \
 382                         time.time():
 383                     fail = True
 384                     wrapped_testcase_suite.logger.critical(
 385                         "Child test runner process timed out "
 386                         "(last test running was `%s' in `%s')!" %
 387                         (wrapped_testcase_suite.last_test,
 388                          wrapped_testcase_suite.last_test_temp_dir))
 389                 elif not wrapped_testcase_suite.child.is_alive():
 390                     fail = True
 391                     wrapped_testcase_suite.logger.critical(
 392                         "Child test runner process unexpectedly died "
 393                         "(last test running was `%s' in `%s')!" %
 394                         (wrapped_testcase_suite.last_test,
 395                          wrapped_testcase_suite.last_test_temp_dir))
 396                 elif wrapped_testcase_suite.last_test_temp_dir and \
 397                         wrapped_testcase_suite.last_test_vpp_binary:
 398                     if is_core_present(
 399                             wrapped_testcase_suite.last_test_temp_dir):
 400                         wrapped_testcase_suite.add_testclass_with_core()
 401                         if wrapped_testcase_suite.core_detected_at is None:
 402                             wrapped_testcase_suite.core_detected_at = \
 403                                 time.time()
 404                         elif wrapped_testcase_suite.core_detected_at + \
 405                                 core_timeout < time.time():
 406                             wrapped_testcase_suite.logger.critical(
 407                                 "Child test runner process unresponsive and "
 408                                 "core-file exists in test temporary directory "
 409                                 "(last test running was `%s' in `%s')!" %
 410                                 (wrapped_testcase_suite.last_test,
 411                                  wrapped_testcase_suite.last_test_temp_dir))
 412                             fail = True
 413
 414                 if fail:
 415                     wrapped_testcase_suite.child.terminate()
 416                     try:
 417                         # terminating the child process tends to leave orphan
 418                         # VPP process around
 419                         if wrapped_testcase_suite.vpp_pid:
 420                             os.kill(wrapped_testcase_suite.vpp_pid,
 421                                     signal.SIGTERM)
 422                     except OSError:
 423                         # already dead
 424                         pass
 425                     wrapped_testcase_suite.result.crashed = True
 426                     wrapped_testcase_suite.result.process_result(
 427                         wrapped_testcase_suite.last_test_id, ERROR)
 428                     stop_run = process_finished_testsuite(
 429                         wrapped_testcase_suite,
 430                         finished_testcase_suites,
 431                         failed_wrapped_testcases,
 432                         results) or stop_run
 433
 434             for finished_testcase in finished_testcase_suites:
 435                 finished_testcase.child.join()
 436                 finished_testcase.close_pipes()
 437                 wrapped_testcase_suites.remove(finished_testcase)
 438                 finished_unread_testcases.add(finished_testcase)
 439                 finished_testcase.stdouterr_queue.put(None)
 440                 if stop_run:
 441                     while testcase_suites:
 442                         results.append(TestResult(testcase_suites.pop(0)))
 443                 elif testcase_suites:
 444                     new_testcase = TestCaseWrapper(testcase_suites.pop(0),
 445                                                    manager)
 446                     wrapped_testcase_suites.add(new_testcase)
 447                     unread_testcases.add(new_testcase)
 448             time.sleep(0.1)
 449     except Exception:
 450         for wrapped_testcase_suite in wrapped_testcase_suites:
 451             wrapped_testcase_suite.child.terminate()
 452             wrapped_testcase_suite.stdouterr_queue.put(None)
 453         raise
 454     finally:
 455         read_from_testcases.clear()
 456         stdouterr_thread.join(test_timeout)
 457         manager.shutdown()
 458
 459     handle_cores(failed_wrapped_testcases)
 460     return results
 461
 462
 463 class SplitToSuitesCallback:
 464     def __init__(self, filter_callback):
 465         self.suites = {}
 466         self.suite_name = 'default'
 467         self.filter_callback = filter_callback
 468         self.filtered = unittest.TestSuite()
 469
 470     def __call__(self, file_name, cls, method):
 471         test_method = cls(method)
 472         if self.filter_callback(file_name, cls.__name__, method):
 473             self.suite_name = file_name + cls.__name__
 474             if self.suite_name not in self.suites:
 475                 self.suites[self.suite_name] = unittest.TestSuite()
 476             self.suites[self.suite_name].addTest(test_method)
 477
 478         else:
 479             self.filtered.addTest(test_method)
 480
 481
 482 test_option = "TEST"
 483
 484
 485 def parse_test_option():
 486     f = os.getenv(test_option, None)
 487     filter_file_name = None
 488     filter_class_name = None
 489     filter_func_name = None
 490     if f:
 491         if '.' in f:
 492             parts = f.split('.')
 493             if len(parts) > 3:
 494                 raise Exception("Unrecognized %s option: %s" %
 495                                 (test_option, f))
 496             if len(parts) > 2:
 497                 if parts[2] not in ('*', ''):
 498                     filter_func_name = parts[2]
 499             if parts[1] not in ('*', ''):
 500                 filter_class_name = parts[1]
 501             if parts[0] not in ('*', ''):
 502                 if parts[0].startswith('test_'):
 503                     filter_file_name = parts[0]
 504                 else:
 505                     filter_file_name = 'test_%s' % parts[0]
 506         else:
 507             if f.startswith('test_'):
 508                 filter_file_name = f
 509             else:
 510                 filter_file_name = 'test_%s' % f
 511     if filter_file_name:
 512         filter_file_name = '%s.py' % filter_file_name
 513     return filter_file_name, filter_class_name, filter_func_name
 514
 515
 516 def filter_tests(tests, filter_cb):
 517     result = unittest.suite.TestSuite()
 518     for t in tests:
 519         if isinstance(t, unittest.suite.TestSuite):
 520             # this is a bunch of tests, recursively filter...
 521             x = filter_tests(t, filter_cb)
 522             if x.countTestCases() > 0:
 523                 result.addTest(x)
 524         elif isinstance(t, unittest.TestCase):
 525             # this is a single test
 526             parts = t.id().split('.')
 527             # t.id() for common cases like this:
 528             # test_classifier.TestClassifier.test_acl_ip
 529             # apply filtering only if it is so
 530             if len(parts) == 3:
 531                 if not filter_cb(parts[0], parts[1], parts[2]):
 532                     continue
 533             result.addTest(t)
 534         else:
 535             # unexpected object, don't touch it
 536             result.addTest(t)
 537     return result
 538
 539
 540 class FilterByTestOption:
 541     def __init__(self, filter_file_name, filter_class_name, filter_func_name):
 542         self.filter_file_name = filter_file_name
 543         self.filter_class_name = filter_class_name
 544         self.filter_func_name = filter_func_name
 545
 546     def __call__(self, file_name, class_name, func_name):
 547         if self.filter_file_name:
 548             fn_match = fnmatch.fnmatch(file_name, self.filter_file_name)
 549             if not fn_match:
 550                 return False
 551         if self.filter_class_name and class_name != self.filter_class_name:
 552             return False
 553         if self.filter_func_name and func_name != self.filter_func_name:
 554             return False
 555         return True
 556
 557
 558 class FilterByClassList:
 559     def __init__(self, classes_with_filenames):
 560         self.classes_with_filenames = classes_with_filenames
 561
 562     def __call__(self, file_name, class_name, func_name):
 563         return '.'.join([file_name, class_name]) in self.classes_with_filenames
 564
 565
 566 def suite_from_failed(suite, failed):
 567     failed = {x.rsplit('.', 1)[0] for x in failed}
 568     filter_cb = FilterByClassList(failed)
 569     suite = filter_tests(suite, filter_cb)
 570     return suite
 571
 572
 573 class AllResults(dict):
 574     def __init__(self):
 575         super(AllResults, self).__init__()
 576         self.all_testcases = 0
 577         self.results_per_suite = []
 578         self[PASS] = 0
 579         self[FAIL] = 0
 580         self[ERROR] = 0
 581         self[SKIP] = 0
 582         self[TEST_RUN] = 0
 583         self.rerun = []
 584         self.testsuites_no_tests_run = []
 585
 586     def add_results(self, result):
 587         self.results_per_suite.append(result)
 588         result_types = [PASS, FAIL, ERROR, SKIP, TEST_RUN]
 589         for result_type in result_types:
 590             self[result_type] += len(result[result_type])
 591
 592     def add_result(self, result):
 593         retval = 0
 594         self.all_testcases += result.testcase_suite.countTestCases()
 595         self.add_results(result)
 596
 597         if result.no_tests_run():
 598             self.testsuites_no_tests_run.append(result.testcase_suite)
 599             if result.crashed:
 600                 retval = -1
 601             else:
 602                 retval = 1
 603         elif not result.was_successful():
 604             retval = 1
 605
 606         if retval != 0:
 607             self.rerun.append(result.testcase_suite)
 608
 609         return retval
 610
 611     def print_results(self):
 612         print('')
 613         print(double_line_delim)
 614         print('TEST RESULTS:')
 615         print('     Scheduled tests: {}'.format(self.all_testcases))
 616         print('      Executed tests: {}'.format(self[TEST_RUN]))
 617         print('        Passed tests: {}'.format(
 618             colorize(str(self[PASS]), GREEN)))
 619         if self[SKIP] > 0:
 620             print('       Skipped tests: {}'.format(
 621                 colorize(str(self[SKIP]), YELLOW)))
 622         if self.not_executed > 0:
 623             print('  Not Executed tests: {}'.format(
 624                 colorize(str(self.not_executed), RED)))
 625         if self[FAIL] > 0:
 626             print('            Failures: {}'.format(
 627                 colorize(str(self[FAIL]), RED)))
 628         if self[ERROR] > 0:
 629             print('              Errors: {}'.format(
 630                 colorize(str(self[ERROR]), RED)))
 631
 632         if self.all_failed > 0:
 633             print('FAILURES AND ERRORS IN TESTS:')
 634             for result in self.results_per_suite:
 635                 failed_testcase_ids = result[FAIL]
 636                 errored_testcase_ids = result[ERROR]
 637                 old_testcase_name = None
 638                 if failed_testcase_ids or errored_testcase_ids:
 639                     for failed_test_id in failed_testcase_ids:
 640                         new_testcase_name, test_name = \
 641                             result.get_testcase_names(failed_test_id)
 642                         if new_testcase_name != old_testcase_name:
 643                             print('  Testcase name: {}'.format(
 644                                 colorize(new_testcase_name, RED)))
 645                             old_testcase_name = new_testcase_name
 646                         print('    FAILURE: {} [{}]'.format(
 647                             colorize(test_name, RED), failed_test_id))
 648                     for failed_test_id in errored_testcase_ids:
 649                         new_testcase_name, test_name = \
 650                             result.get_testcase_names(failed_test_id)
 651                         if new_testcase_name != old_testcase_name:
 652                             print('  Testcase name: {}'.format(
 653                                 colorize(new_testcase_name, RED)))
 654                             old_testcase_name = new_testcase_name
 655                         print('      ERROR: {} [{}]'.format(
 656                             colorize(test_name, RED), failed_test_id))
 657         if self.testsuites_no_tests_run:
 658             print('TESTCASES WHERE NO TESTS WERE SUCCESSFULLY EXECUTED:')
 659             tc_classes = set()
 660             for testsuite in self.testsuites_no_tests_run:
 661                 for testcase in testsuite:
 662                     tc_classes.add(get_testcase_doc_name(testcase))
 663             for tc_class in tc_classes:
 664                 print('  {}'.format(colorize(tc_class, RED)))
 665
 666         print(double_line_delim)
 667         print('')
 668
 669     @property
 670     def not_executed(self):
 671         return self.all_testcases - self[TEST_RUN]
 672
 673     @property
 674     def all_failed(self):
 675         return self[FAIL] + self[ERROR]
 676
 677
 678 def parse_results(results):
 679     """
 680     Prints the number of scheduled, executed, not executed, passed, failed,
 681     errored and skipped tests and details about failed and errored tests.
 682
 683     Also returns all suites where any test failed.
 684
 685     :param results:
 686     :return:
 687     """
 688
 689     results_per_suite = AllResults()
 690     crashed = False
 691     failed = False
 692     for result in results:
 693         result_code = results_per_suite.add_result(result)
 694         if result_code == 1:
 695             failed = True
 696         elif result_code == -1:
 697             crashed = True
 698
 699     results_per_suite.print_results()
 700
 701     if crashed:
 702         return_code = -1
 703     elif failed:
 704         return_code = 1
 705     else:
 706         return_code = 0
 707     return return_code, results_per_suite.rerun
 708
 709
 710 def parse_digit_env(env_var, default):
 711     value = os.getenv(env_var, default)
 712     if value != default:
 713         if value.isdigit():
 714             value = int(value)
 715         else:
 716             print('WARNING: unsupported value "%s" for env var "%s",'
 717                   'defaulting to %s' % (value, env_var, default))
 718             value = default
 719     return value
 720
 721
 722 if __name__ == '__main__':
 723
 724     verbose = parse_digit_env("V", 0)
 725
 726     test_timeout = parse_digit_env("TIMEOUT", 600)  # default = 10 minutes
 727
 728     retries = parse_digit_env("RETRIES", 0)
 729
 730     debug = os.getenv("DEBUG", "n").lower() in ["gdb", "gdbserver"]
 731
 732     debug_core = os.getenv("DEBUG", "").lower() == "core"
 733     compress_core = framework.BoolEnvironmentVariable("CORE_COMPRESS")
 734
 735     step = framework.BoolEnvironmentVariable("STEP")
 736     force_foreground = framework.BoolEnvironmentVariable("FORCE_FOREGROUND")
 737
 738     run_interactive = debug or step or force_foreground
 739
 740     test_jobs = os.getenv("TEST_JOBS", "1").lower()  # default = 1 process
 741     if test_jobs == 'auto':
 742         if run_interactive:
 743             concurrent_tests = 1
 744             print('Interactive mode required, running on one core')
 745         else:
 746             shm_free = psutil.disk_usage('/dev/shm').free
 747             shm_max_processes = 1
 748             if shm_free < min_req_shm:
 749                 raise Exception('Not enough free space in /dev/shm. Required '
 750                                 'free space is at least %sM.'
 751                                 % (min_req_shm >> 20))
 752             else:
 753                 extra_shm = shm_free - min_req_shm
 754                 shm_max_processes += extra_shm / shm_per_process
 755             concurrent_tests = min(cpu_count(), shm_max_processes)
 756             print('Found enough resources to run tests with %s cores'
 757                   % concurrent_tests)
 758     elif test_jobs.isdigit():
 759         concurrent_tests = int(test_jobs)
 760     else:
 761         concurrent_tests = 1
 762
 763     if run_interactive and concurrent_tests > 1:
 764         raise NotImplementedError(
 765             'Running tests interactively (DEBUG is gdb or gdbserver or STEP '
 766             'is set) in parallel (TEST_JOBS is more than 1) is not supported')
 767
 768     parser = argparse.ArgumentParser(description="VPP unit tests")
 769     parser.add_argument("-f", "--failfast", action='store_true',
 770                         help="fast failure flag")
 771     parser.add_argument("-d", "--dir", action='append', type=str,
 772                         help="directory containing test files "
 773                              "(may be specified multiple times)")
 774     args = parser.parse_args()
 775     failfast = args.failfast
 776     descriptions = True
 777
 778     print("Running tests using custom test runner")  # debug message
 779     filter_file, filter_class, filter_func = parse_test_option()
 780
 781     print("Active filters: file=%s, class=%s, function=%s" % (
 782         filter_file, filter_class, filter_func))
 783
 784     filter_cb = FilterByTestOption(filter_file, filter_class, filter_func)
 785
 786     ignore_path = os.getenv("VENV_PATH", None)
 787     cb = SplitToSuitesCallback(filter_cb)
 788     for d in args.dir:
 789         print("Adding tests from directory tree %s" % d)
 790         discover_tests(d, cb, ignore_path)
 791
 792     # suites are not hashable, need to use list
 793     suites = []
 794     tests_amount = 0
 795     for testcase_suite in cb.suites.values():
 796         tests_amount += testcase_suite.countTestCases()
 797         suites.append(testcase_suite)
 798
 799     print("%s out of %s tests match specified filters" % (
 800         tests_amount, tests_amount + cb.filtered.countTestCases()))
 801
 802     if not running_extended_tests:
 803         print("Not running extended tests (some tests will be skipped)")
 804
 805     attempts = retries + 1
 806     if attempts > 1:
 807         print("Perform %s attempts to pass the suite..." % attempts)
 808
 809     if run_interactive and suites:
 810         # don't fork if requiring interactive terminal
 811         print('Running tests in foreground in the current process')
 812         full_suite = unittest.TestSuite()
 813         map(full_suite.addTests, suites)
 814         result = VppTestRunner(verbosity=verbose,
 815                                failfast=failfast,
 816                                print_summary=True).run(full_suite)
 817         was_successful = result.wasSuccessful()
 818         if not was_successful:
 819             for test_case_info in result.failed_test_cases_info:
 820                 handle_failed_suite(test_case_info.logger,
 821                                     test_case_info.tempdir,
 822                                     test_case_info.vpp_pid)
 823                 if test_case_info in result.core_crash_test_cases_info:
 824                     check_and_handle_core(test_case_info.vpp_bin_path,
 825                                           test_case_info.tempdir,
 826                                           test_case_info.core_crash_test)
 827
 828         sys.exit(not was_successful)
 829     else:
 830         print('Running each VPPTestCase in a separate background process'
 831               ' with {} parallel process(es)'.format(concurrent_tests))
 832         exit_code = 0
 833         while suites and attempts > 0:
 834             results = run_forked(suites)
 835             exit_code, suites = parse_results(results)
 836             attempts -= 1
 837             if exit_code == 0:
 838                 print('Test run was successful')
 839             else:
 840                 print('%s attempt(s) left.' % attempts)
 841         sys.exit(exit_code)