test/framework.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import print_function
   4 import gc
   5 import sys
   6 import os
   7 import select
   8 import unittest
   9 import tempfile
  10 import time
  11 import faulthandler
  12 import random
  13 import copy
  14 import psutil
  15 import platform
  16 from collections import deque
  17 from threading import Thread, Event
  18 from inspect import getdoc, isclass
  19 from traceback import format_exception
  20 from logging import FileHandler, DEBUG, Formatter
  21 from scapy.packet import Raw
  22 from hook import StepHook, PollHook, VppDiedError
  23 from vpp_pg_interface import VppPGInterface
  24 from vpp_sub_interface import VppSubInterface
  25 from vpp_lo_interface import VppLoInterface
  26 from vpp_papi_provider import VppPapiProvider
  27 from vpp_papi.vpp_stats import VPPStats
  28 from log import RED, GREEN, YELLOW, double_line_delim, single_line_delim, \
  29     get_logger, colorize
  30 from vpp_object import VppObjectRegistry
  31 from util import ppp, is_core_present
  32 from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror
  33 from scapy.layers.inet6 import ICMPv6DestUnreach, ICMPv6EchoRequest
  34 from scapy.layers.inet6 import ICMPv6EchoReply
  35
  36 if os.name == 'posix' and sys.version_info[0] < 3:
  37     # using subprocess32 is recommended by python official documentation
  38     # @ https://docs.python.org/2/library/subprocess.html
  39     import subprocess32 as subprocess
  40 else:
  41     import subprocess
  42
  43 PASS = 0
  44 FAIL = 1
  45 ERROR = 2
  46 SKIP = 3
  47 TEST_RUN = 4
  48
  49 debug_framework = False
  50 if os.getenv('TEST_DEBUG', "0") == "1":
  51     debug_framework = True
  52     import debug_internal
  53
  54 """
  55   Test framework module.
  56
  57   The module provides a set of tools for constructing and running tests and
  58   representing the results.
  59 """
  60
  61
  62 class _PacketInfo(object):
  63     """Private class to create packet info object.
  64
  65     Help process information about the next packet.
  66     Set variables to default values.
  67     """
  68     #: Store the index of the packet.
  69     index = -1
  70     #: Store the index of the source packet generator interface of the packet.
  71     src = -1
  72     #: Store the index of the destination packet generator interface
  73     #: of the packet.
  74     dst = -1
  75     #: Store expected ip version
  76     ip = -1
  77     #: Store expected upper protocol
  78     proto = -1
  79     #: Store the copy of the former packet.
  80     data = None
  81
  82     def __eq__(self, other):
  83         index = self.index == other.index
  84         src = self.src == other.src
  85         dst = self.dst == other.dst
  86         data = self.data == other.data
  87         return index and src and dst and data
  88
  89
  90 def pump_output(testclass):
  91     """ pump output from vpp stdout/stderr to proper queues """
  92     stdout_fragment = ""
  93     stderr_fragment = ""
  94     while not testclass.pump_thread_stop_flag.is_set():
  95         readable = select.select([testclass.vpp.stdout.fileno(),
  96                                   testclass.vpp.stderr.fileno(),
  97                                   testclass.pump_thread_wakeup_pipe[0]],
  98                                  [], [])[0]
  99         if testclass.vpp.stdout.fileno() in readable:
 100             read = os.read(testclass.vpp.stdout.fileno(), 102400)
 101             if len(read) > 0:
 102                 split = read.splitlines(True)
 103                 if len(stdout_fragment) > 0:
 104                     split[0] = "%s%s" % (stdout_fragment, split[0])
 105                 if len(split) > 0 and split[-1].endswith("\n"):
 106                     limit = None
 107                 else:
 108                     limit = -1
 109                     stdout_fragment = split[-1]
 110                 testclass.vpp_stdout_deque.extend(split[:limit])
 111                 if not testclass.cache_vpp_output:
 112                     for line in split[:limit]:
 113                         testclass.logger.debug(
 114                             "VPP STDOUT: %s" % line.rstrip("\n"))
 115         if testclass.vpp.stderr.fileno() in readable:
 116             read = os.read(testclass.vpp.stderr.fileno(), 102400)
 117             if len(read) > 0:
 118                 split = read.splitlines(True)
 119                 if len(stderr_fragment) > 0:
 120                     split[0] = "%s%s" % (stderr_fragment, split[0])
 121                 if len(split) > 0 and split[-1].endswith(b"\n"):
 122                     limit = None
 123                 else:
 124                     limit = -1
 125                     stderr_fragment = split[-1]
 126                 testclass.vpp_stderr_deque.extend(split[:limit])
 127                 if not testclass.cache_vpp_output:
 128                     for line in split[:limit]:
 129                         testclass.logger.debug(
 130                             "VPP STDERR: %s" % line.rstrip("\n"))
 131                         # ignoring the dummy pipe here intentionally - the
 132                         # flag will take care of properly terminating the loop
 133
 134
 135 def is_skip_aarch64_set():
 136     return os.getenv('SKIP_AARCH64', 'n').lower() in ('yes', 'y', '1')
 137
 138
 139 def is_platform_aarch64():
 140     return platform.machine() == 'aarch64'
 141
 142
 143 def running_extended_tests():
 144     s = os.getenv("EXTENDED_TESTS", "n")
 145     return True if s.lower() in ("y", "yes", "1") else False
 146
 147
 148 def running_on_centos():
 149     os_id = os.getenv("OS_ID", "")
 150     return True if "centos" in os_id.lower() else False
 151
 152
 153 class KeepAliveReporter(object):
 154     """
 155     Singleton object which reports test start to parent process
 156     """
 157     _shared_state = {}
 158
 159     def __init__(self):
 160         self.__dict__ = self._shared_state
 161         self._pipe = None
 162
 163     @property
 164     def pipe(self):
 165         return self._pipe
 166
 167     @pipe.setter
 168     def pipe(self, pipe):
 169         if self._pipe is not None:
 170             raise Exception("Internal error - pipe should only be set once.")
 171         self._pipe = pipe
 172
 173     def send_keep_alive(self, test, desc=None):
 174         """
 175         Write current test tmpdir & desc to keep-alive pipe to signal liveness
 176         """
 177         if self.pipe is None:
 178             # if not running forked..
 179             return
 180
 181         if isclass(test):
 182             desc = '%s (%s)' % (desc, unittest.util.strclass(test))
 183         else:
 184             desc = test.id()
 185
 186         self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid))
 187
 188
 189 class VppTestCase(unittest.TestCase):
 190     """This subclass is a base class for VPP test cases that are implemented as
 191     classes. It provides methods to create and run test case.
 192     """
 193
 194     extra_vpp_punt_config = []
 195
 196     @property
 197     def packet_infos(self):
 198         """List of packet infos"""
 199         return self._packet_infos
 200
 201     @classmethod
 202     def get_packet_count_for_if_idx(cls, dst_if_index):
 203         """Get the number of packet info for specified destination if index"""
 204         if dst_if_index in cls._packet_count_for_dst_if_idx:
 205             return cls._packet_count_for_dst_if_idx[dst_if_index]
 206         else:
 207             return 0
 208
 209     @classmethod
 210     def instance(cls):
 211         """Return the instance of this testcase"""
 212         return cls.test_instance
 213
 214     @classmethod
 215     def set_debug_flags(cls, d):
 216         cls.debug_core = False
 217         cls.debug_gdb = False
 218         cls.debug_gdbserver = False
 219         if d is None:
 220             return
 221         dl = d.lower()
 222         if dl == "core":
 223             cls.debug_core = True
 224         elif dl == "gdb":
 225             cls.debug_gdb = True
 226         elif dl == "gdbserver":
 227             cls.debug_gdbserver = True
 228         else:
 229             raise Exception("Unrecognized DEBUG option: '%s'" % d)
 230
 231     @staticmethod
 232     def get_least_used_cpu():
 233         cpu_usage_list = [set(range(psutil.cpu_count()))]
 234         vpp_processes = [p for p in psutil.process_iter(attrs=['pid', 'name'])
 235                          if 'vpp_main' == p.info['name']]
 236         for vpp_process in vpp_processes:
 237             for cpu_usage_set in cpu_usage_list:
 238                 try:
 239                     cpu_num = vpp_process.cpu_num()
 240                     if cpu_num in cpu_usage_set:
 241                         cpu_usage_set_index = cpu_usage_list.index(
 242                             cpu_usage_set)
 243                         if cpu_usage_set_index == len(cpu_usage_list) - 1:
 244                             cpu_usage_list.append({cpu_num})
 245                         else:
 246                             cpu_usage_list[cpu_usage_set_index + 1].add(
 247                                 cpu_num)
 248                         cpu_usage_set.remove(cpu_num)
 249                         break
 250                 except psutil.NoSuchProcess:
 251                     pass
 252
 253         for cpu_usage_set in cpu_usage_list:
 254             if len(cpu_usage_set) > 0:
 255                 min_usage_set = cpu_usage_set
 256                 break
 257
 258         return random.choice(tuple(min_usage_set))
 259
 260     @classmethod
 261     def print_header(cls):
 262         if not hasattr(cls, '_header_printed'):
 263             print(double_line_delim)
 264             print(colorize(getdoc(cls).splitlines()[0], GREEN))
 265             print(double_line_delim)
 266             cls._header_printed = True
 267
 268     @classmethod
 269     def setUpConstants(cls):
 270         """ Set-up the test case class based on environment variables """
 271         s = os.getenv("STEP", "n")
 272         cls.step = True if s.lower() in ("y", "yes", "1") else False
 273         d = os.getenv("DEBUG", None)
 274         c = os.getenv("CACHE_OUTPUT", "1")
 275         cls.cache_vpp_output = False if c.lower() in ("n", "no", "0") else True
 276         cls.set_debug_flags(d)
 277         cls.vpp_bin = os.getenv('VPP_BIN', "vpp")
 278         cls.plugin_path = os.getenv('VPP_PLUGIN_PATH')
 279         cls.extern_plugin_path = os.getenv('EXTERN_PLUGINS')
 280         plugin_path = None
 281         if cls.plugin_path is not None:
 282             if cls.extern_plugin_path is not None:
 283                 plugin_path = "%s:%s" % (
 284                     cls.plugin_path, cls.extern_plugin_path)
 285             else:
 286                 plugin_path = cls.plugin_path
 287         elif cls.extern_plugin_path is not None:
 288             plugin_path = cls.extern_plugin_path
 289         debug_cli = ""
 290         if cls.step or cls.debug_gdb or cls.debug_gdbserver:
 291             debug_cli = "cli-listen localhost:5002"
 292         coredump_size = None
 293         size = os.getenv("COREDUMP_SIZE")
 294         if size is not None:
 295             coredump_size = "coredump-size %s" % size
 296         if coredump_size is None:
 297             coredump_size = "coredump-size unlimited"
 298
 299         cpu_core_number = cls.get_least_used_cpu()
 300
 301         cls.vpp_cmdline = [cls.vpp_bin, "unix",
 302                            "{", "nodaemon", debug_cli, "full-coredump",
 303                            coredump_size, "runtime-dir", cls.tempdir, "}",
 304                            "api-trace", "{", "on", "}", "api-segment", "{",
 305                            "prefix", cls.shm_prefix, "}", "cpu", "{",
 306                            "main-core", str(cpu_core_number), "}", "statseg",
 307                            "{", "socket-name", cls.stats_sock, "}", "plugins",
 308                            "{", "plugin", "dpdk_plugin.so", "{", "disable",
 309                            "}", "plugin", "unittest_plugin.so", "{", "enable",
 310                            "}", "}", ]
 311         if cls.extra_vpp_punt_config is not None:
 312             cls.vpp_cmdline.extend(cls.extra_vpp_punt_config)
 313         if plugin_path is not None:
 314             cls.vpp_cmdline.extend(["plugin_path", plugin_path])
 315         cls.logger.info("vpp_cmdline args: %s" % cls.vpp_cmdline)
 316         cls.logger.info("vpp_cmdline: %s" % " ".join(cls.vpp_cmdline))
 317
 318     @classmethod
 319     def wait_for_enter(cls):
 320         if cls.debug_gdbserver:
 321             print(double_line_delim)
 322             print("Spawned GDB server with PID: %d" % cls.vpp.pid)
 323         elif cls.debug_gdb:
 324             print(double_line_delim)
 325             print("Spawned VPP with PID: %d" % cls.vpp.pid)
 326         else:
 327             cls.logger.debug("Spawned VPP with PID: %d" % cls.vpp.pid)
 328             return
 329         print(single_line_delim)
 330         print("You can debug the VPP using e.g.:")
 331         if cls.debug_gdbserver:
 332             print("gdb " + cls.vpp_bin + " -ex 'target remote localhost:7777'")
 333             print("Now is the time to attach a gdb by running the above "
 334                   "command, set up breakpoints etc. and then resume VPP from "
 335                   "within gdb by issuing the 'continue' command")
 336         elif cls.debug_gdb:
 337             print("gdb " + cls.vpp_bin + " -ex 'attach %s'" % cls.vpp.pid)
 338             print("Now is the time to attach a gdb by running the above "
 339                   "command and set up breakpoints etc.")
 340         print(single_line_delim)
 341         raw_input("Press ENTER to continue running the testcase...")
 342
 343     @classmethod
 344     def run_vpp(cls):
 345         cmdline = cls.vpp_cmdline
 346
 347         if cls.debug_gdbserver:
 348             gdbserver = '/usr/bin/gdbserver'
 349             if not os.path.isfile(gdbserver) or \
 350                     not os.access(gdbserver, os.X_OK):
 351                 raise Exception("gdbserver binary '%s' does not exist or is "
 352                                 "not executable" % gdbserver)
 353
 354             cmdline = [gdbserver, 'localhost:7777'] + cls.vpp_cmdline
 355             cls.logger.info("Gdbserver cmdline is %s", " ".join(cmdline))
 356
 357         try:
 358             cls.vpp = subprocess.Popen(cmdline,
 359                                        stdout=subprocess.PIPE,
 360                                        stderr=subprocess.PIPE,
 361                                        bufsize=1)
 362         except subprocess.CalledProcessError as e:
 363             cls.logger.critical("Couldn't start vpp: %s" % e)
 364             raise
 365
 366         cls.wait_for_enter()
 367
 368     @classmethod
 369     def wait_for_stats_socket(cls):
 370         deadline = time.time() + 3
 371         ok = False
 372         while time.time() < deadline or \
 373                 cls.debug_gdb or cls.debug_gdbserver:
 374             if os.path.exists(cls.stats_sock):
 375                 ok = True
 376                 break
 377             time.sleep(0.8)
 378         if not ok:
 379             cls.logger.critical("Couldn't stat : {}".format(cls.stats_sock))
 380
 381     @classmethod
 382     def setUpClass(cls):
 383         """
 384         Perform class setup before running the testcase
 385         Remove shared memory files, start vpp and connect the vpp-api
 386         """
 387         gc.collect()  # run garbage collection first
 388         random.seed()
 389         cls.print_header()
 390         cls.logger = get_logger(cls.__name__)
 391         if hasattr(cls, 'parallel_handler'):
 392             cls.logger.addHandler(cls.parallel_handler)
 393             cls.logger.propagate = False
 394         cls.tempdir = tempfile.mkdtemp(
 395             prefix='vpp-unittest-%s-' % cls.__name__)
 396         cls.stats_sock = "%s/stats.sock" % cls.tempdir
 397         cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir)
 398         cls.file_handler.setFormatter(
 399             Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s',
 400                       datefmt="%H:%M:%S"))
 401         cls.file_handler.setLevel(DEBUG)
 402         cls.logger.addHandler(cls.file_handler)
 403         cls.shm_prefix = os.path.basename(cls.tempdir)
 404         os.chdir(cls.tempdir)
 405         cls.logger.info("Temporary dir is %s, shm prefix is %s",
 406                         cls.tempdir, cls.shm_prefix)
 407         cls.setUpConstants()
 408         cls.reset_packet_infos()
 409         cls._captures = []
 410         cls._zombie_captures = []
 411         cls.verbose = 0
 412         cls.vpp_dead = False
 413         cls.registry = VppObjectRegistry()
 414         cls.vpp_startup_failed = False
 415         cls.reporter = KeepAliveReporter()
 416         # need to catch exceptions here because if we raise, then the cleanup
 417         # doesn't get called and we might end with a zombie vpp
 418         try:
 419             cls.run_vpp()
 420             cls.reporter.send_keep_alive(cls, 'setUpClass')
 421             VppTestResult.current_test_case_info = TestCaseInfo(
 422                 cls.logger, cls.tempdir, cls.vpp.pid, cls.vpp_bin)
 423             cls.vpp_stdout_deque = deque()
 424             cls.vpp_stderr_deque = deque()
 425             cls.pump_thread_stop_flag = Event()
 426             cls.pump_thread_wakeup_pipe = os.pipe()
 427             cls.pump_thread = Thread(target=pump_output, args=(cls,))
 428             cls.pump_thread.daemon = True
 429             cls.pump_thread.start()
 430             if cls.debug_gdb or cls.debug_gdbserver:
 431                 read_timeout = 0
 432             else:
 433                 read_timeout = 5
 434             cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls,
 435                                        read_timeout)
 436             if cls.step:
 437                 hook = StepHook(cls)
 438             else:
 439                 hook = PollHook(cls)
 440             cls.vapi.register_hook(hook)
 441             cls.wait_for_stats_socket()
 442             cls.statistics = VPPStats(socketname=cls.stats_sock)
 443             try:
 444                 hook.poll_vpp()
 445             except VppDiedError:
 446                 cls.vpp_startup_failed = True
 447                 cls.logger.critical(
 448                     "VPP died shortly after startup, check the"
 449                     " output to standard error for possible cause")
 450                 raise
 451             try:
 452                 cls.vapi.connect()
 453             except Exception:
 454                 try:
 455                     cls.vapi.disconnect()
 456                 except Exception:
 457                     pass
 458                 if cls.debug_gdbserver:
 459                     print(colorize("You're running VPP inside gdbserver but "
 460                                    "VPP-API connection failed, did you forget "
 461                                    "to 'continue' VPP from within gdb?", RED))
 462                 raise
 463         except Exception:
 464             try:
 465                 cls.quit()
 466             except Exception:
 467                 pass
 468             raise
 469
 470     @classmethod
 471     def quit(cls):
 472         """
 473         Disconnect vpp-api, kill vpp and cleanup shared memory files
 474         """
 475         if (cls.debug_gdbserver or cls.debug_gdb) and hasattr(cls, 'vpp'):
 476             cls.vpp.poll()
 477             if cls.vpp.returncode is None:
 478                 print(double_line_delim)
 479                 print("VPP or GDB server is still running")
 480                 print(single_line_delim)
 481                 raw_input("When done debugging, press ENTER to kill the "
 482                           "process and finish running the testcase...")
 483
 484         # first signal that we want to stop the pump thread, then wake it up
 485         if hasattr(cls, 'pump_thread_stop_flag'):
 486             cls.pump_thread_stop_flag.set()
 487         if hasattr(cls, 'pump_thread_wakeup_pipe'):
 488             os.write(cls.pump_thread_wakeup_pipe[1], b'ding dong wake up')
 489         if hasattr(cls, 'pump_thread'):
 490             cls.logger.debug("Waiting for pump thread to stop")
 491             cls.pump_thread.join()
 492         if hasattr(cls, 'vpp_stderr_reader_thread'):
 493             cls.logger.debug("Waiting for stdderr pump to stop")
 494             cls.vpp_stderr_reader_thread.join()
 495
 496         if hasattr(cls, 'vpp'):
 497             if hasattr(cls, 'vapi'):
 498                 cls.vapi.disconnect()
 499                 del cls.vapi
 500             cls.vpp.poll()
 501             if cls.vpp.returncode is None:
 502                 cls.logger.debug("Sending TERM to vpp")
 503                 cls.vpp.kill()
 504                 cls.logger.debug("Waiting for vpp to die")
 505                 cls.vpp.communicate()
 506             del cls.vpp
 507
 508         if cls.vpp_startup_failed:
 509             stdout_log = cls.logger.info
 510             stderr_log = cls.logger.critical
 511         else:
 512             stdout_log = cls.logger.info
 513             stderr_log = cls.logger.info
 514
 515         if hasattr(cls, 'vpp_stdout_deque'):
 516             stdout_log(single_line_delim)
 517             stdout_log('VPP output to stdout while running %s:', cls.__name__)
 518             stdout_log(single_line_delim)
 519             vpp_output = "".join(cls.vpp_stdout_deque)
 520             with open(cls.tempdir + '/vpp_stdout.txt', 'w') as f:
 521                 f.write(vpp_output)
 522             stdout_log('\n%s', vpp_output)
 523             stdout_log(single_line_delim)
 524
 525         if hasattr(cls, 'vpp_stderr_deque'):
 526             stderr_log(single_line_delim)
 527             stderr_log('VPP output to stderr while running %s:', cls.__name__)
 528             stderr_log(single_line_delim)
 529             vpp_output = "".join(str(cls.vpp_stderr_deque))
 530             with open(cls.tempdir + '/vpp_stderr.txt', 'w') as f:
 531                 f.write(vpp_output)
 532             stderr_log('\n%s', vpp_output)
 533             stderr_log(single_line_delim)
 534
 535     @classmethod
 536     def tearDownClass(cls):
 537         """ Perform final cleanup after running all tests in this test-case """
 538         cls.reporter.send_keep_alive(cls, 'tearDownClass')
 539         cls.quit()
 540         cls.file_handler.close()
 541         cls.reset_packet_infos()
 542         if debug_framework:
 543             debug_internal.on_tear_down_class(cls)
 544
 545     def tearDown(self):
 546         """ Show various debug prints after each test """
 547         self.logger.debug("--- tearDown() for %s.%s(%s) called ---" %
 548                           (self.__class__.__name__, self._testMethodName,
 549                            self._testMethodDoc))
 550         if not self.vpp_dead:
 551             self.logger.debug(self.vapi.cli("show trace"))
 552             self.logger.info(self.vapi.ppcli("show interface"))
 553             self.logger.info(self.vapi.ppcli("show hardware"))
 554             self.logger.info(self.statistics.set_errors_str())
 555             self.logger.info(self.vapi.ppcli("show run"))
 556             self.logger.info(self.vapi.ppcli("show log"))
 557             self.registry.remove_vpp_config(self.logger)
 558             # Save/Dump VPP api trace log
 559             api_trace = "vpp_api_trace.%s.log" % self._testMethodName
 560             tmp_api_trace = "/tmp/%s" % api_trace
 561             vpp_api_trace_log = "%s/%s" % (self.tempdir, api_trace)
 562             self.logger.info(self.vapi.ppcli("api trace save %s" % api_trace))
 563             self.logger.info("Moving %s to %s\n" % (tmp_api_trace,
 564                                                     vpp_api_trace_log))
 565             os.rename(tmp_api_trace, vpp_api_trace_log)
 566             self.logger.info(self.vapi.ppcli("api trace custom-dump %s" %
 567                                              vpp_api_trace_log))
 568         else:
 569             self.registry.unregister_all(self.logger)
 570
 571     def setUp(self):
 572         """ Clear trace before running each test"""
 573         self.reporter.send_keep_alive(self)
 574         self.logger.debug("--- setUp() for %s.%s(%s) called ---" %
 575                           (self.__class__.__name__, self._testMethodName,
 576                            self._testMethodDoc))
 577         if self.vpp_dead:
 578             raise Exception("VPP is dead when setting up the test")
 579         self.sleep(.1, "during setUp")
 580         self.vpp_stdout_deque.append(
 581             "--- test setUp() for %s.%s(%s) starts here ---\n" %
 582             (self.__class__.__name__, self._testMethodName,
 583              self._testMethodDoc))
 584         self.vpp_stderr_deque.append(
 585             "--- test setUp() for %s.%s(%s) starts here ---\n" %
 586             (self.__class__.__name__, self._testMethodName,
 587              self._testMethodDoc))
 588         self.vapi.cli("clear trace")
 589         # store the test instance inside the test class - so that objects
 590         # holding the class can access instance methods (like assertEqual)
 591         type(self).test_instance = self
 592
 593     @classmethod
 594     def pg_enable_capture(cls, interfaces=None):
 595         """
 596         Enable capture on packet-generator interfaces
 597
 598         :param interfaces: iterable interface indexes (if None,
 599                            use self.pg_interfaces)
 600
 601         """
 602         if interfaces is None:
 603             interfaces = cls.pg_interfaces
 604         for i in interfaces:
 605             i.enable_capture()
 606
 607     @classmethod
 608     def register_capture(cls, cap_name):
 609         """ Register a capture in the testclass """
 610         # add to the list of captures with current timestamp
 611         cls._captures.append((time.time(), cap_name))
 612         # filter out from zombies
 613         cls._zombie_captures = [(stamp, name)
 614                                 for (stamp, name) in cls._zombie_captures
 615                                 if name != cap_name]
 616
 617     @classmethod
 618     def pg_start(cls):
 619         """ Remove any zombie captures and enable the packet generator """
 620         # how long before capture is allowed to be deleted - otherwise vpp
 621         # crashes - 100ms seems enough (this shouldn't be needed at all)
 622         capture_ttl = 0.1
 623         now = time.time()
 624         for stamp, cap_name in cls._zombie_captures:
 625             wait = stamp + capture_ttl - now
 626             if wait > 0:
 627                 cls.sleep(wait, "before deleting capture %s" % cap_name)
 628                 now = time.time()
 629             cls.logger.debug("Removing zombie capture %s" % cap_name)
 630             cls.vapi.cli('packet-generator delete %s' % cap_name)
 631
 632         cls.vapi.cli("trace add pg-input 50")  # 50 is maximum
 633         cls.vapi.cli('packet-generator enable')
 634         cls._zombie_captures = cls._captures
 635         cls._captures = []
 636
 637     @classmethod
 638     def create_pg_interfaces(cls, interfaces):
 639         """
 640         Create packet-generator interfaces.
 641
 642         :param interfaces: iterable indexes of the interfaces.
 643         :returns: List of created interfaces.
 644
 645         """
 646         result = []
 647         for i in interfaces:
 648             intf = VppPGInterface(cls, i)
 649             setattr(cls, intf.name, intf)
 650             result.append(intf)
 651         cls.pg_interfaces = result
 652         return result
 653
 654     @classmethod
 655     def create_loopback_interfaces(cls, count):
 656         """
 657         Create loopback interfaces.
 658
 659         :param count: number of interfaces created.
 660         :returns: List of created interfaces.
 661         """
 662         result = [VppLoInterface(cls) for i in range(count)]
 663         for intf in result:
 664             setattr(cls, intf.name, intf)
 665         cls.lo_interfaces = result
 666         return result
 667
 668     @staticmethod
 669     def extend_packet(packet, size, padding=' '):
 670         """
 671         Extend packet to given size by padding with spaces or custom padding
 672         NOTE: Currently works only when Raw layer is present.
 673
 674         :param packet: packet
 675         :param size: target size
 676         :param padding: padding used to extend the payload
 677
 678         """
 679         packet_len = len(packet) + 4
 680         extend = size - packet_len
 681         if extend > 0:
 682             num = (extend / len(padding)) + 1
 683             packet[Raw].load += (padding * num)[:extend]
 684
 685     @classmethod
 686     def reset_packet_infos(cls):
 687         """ Reset the list of packet info objects and packet counts to zero """
 688         cls._packet_infos = {}
 689         cls._packet_count_for_dst_if_idx = {}
 690
 691     @classmethod
 692     def create_packet_info(cls, src_if, dst_if):
 693         """
 694         Create packet info object containing the source and destination indexes
 695         and add it to the testcase's packet info list
 696
 697         :param VppInterface src_if: source interface
 698         :param VppInterface dst_if: destination interface
 699
 700         :returns: _PacketInfo object
 701
 702         """
 703         info = _PacketInfo()
 704         info.index = len(cls._packet_infos)
 705         info.src = src_if.sw_if_index
 706         info.dst = dst_if.sw_if_index
 707         if isinstance(dst_if, VppSubInterface):
 708             dst_idx = dst_if.parent.sw_if_index
 709         else:
 710             dst_idx = dst_if.sw_if_index
 711         if dst_idx in cls._packet_count_for_dst_if_idx:
 712             cls._packet_count_for_dst_if_idx[dst_idx] += 1
 713         else:
 714             cls._packet_count_for_dst_if_idx[dst_idx] = 1
 715         cls._packet_infos[info.index] = info
 716         return info
 717
 718     @staticmethod
 719     def info_to_payload(info):
 720         """
 721         Convert _PacketInfo object to packet payload
 722
 723         :param info: _PacketInfo object
 724
 725         :returns: string containing serialized data from packet info
 726         """
 727         return "%d %d %d %d %d" % (info.index, info.src, info.dst,
 728                                    info.ip, info.proto)
 729
 730     @staticmethod
 731     def payload_to_info(payload):
 732         """
 733         Convert packet payload to _PacketInfo object
 734
 735         :param payload: packet payload
 736
 737         :returns: _PacketInfo object containing de-serialized data from payload
 738
 739         """
 740         numbers = payload.split()
 741         info = _PacketInfo()
 742         info.index = int(numbers[0])
 743         info.src = int(numbers[1])
 744         info.dst = int(numbers[2])
 745         info.ip = int(numbers[3])
 746         info.proto = int(numbers[4])
 747         return info
 748
 749     def get_next_packet_info(self, info):
 750         """
 751         Iterate over the packet info list stored in the testcase
 752         Start iteration with first element if info is None
 753         Continue based on index in info if info is specified
 754
 755         :param info: info or None
 756         :returns: next info in list or None if no more infos
 757         """
 758         if info is None:
 759             next_index = 0
 760         else:
 761             next_index = info.index + 1
 762         if next_index == len(self._packet_infos):
 763             return None
 764         else:
 765             return self._packet_infos[next_index]
 766
 767     def get_next_packet_info_for_interface(self, src_index, info):
 768         """
 769         Search the packet info list for the next packet info with same source
 770         interface index
 771
 772         :param src_index: source interface index to search for
 773         :param info: packet info - where to start the search
 774         :returns: packet info or None
 775
 776         """
 777         while True:
 778             info = self.get_next_packet_info(info)
 779             if info is None:
 780                 return None
 781             if info.src == src_index:
 782                 return info
 783
 784     def get_next_packet_info_for_interface2(self, src_index, dst_index, info):
 785         """
 786         Search the packet info list for the next packet info with same source
 787         and destination interface indexes
 788
 789         :param src_index: source interface index to search for
 790         :param dst_index: destination interface index to search for
 791         :param info: packet info - where to start the search
 792         :returns: packet info or None
 793
 794         """
 795         while True:
 796             info = self.get_next_packet_info_for_interface(src_index, info)
 797             if info is None:
 798                 return None
 799             if info.dst == dst_index:
 800                 return info
 801
 802     def assert_equal(self, real_value, expected_value, name_or_class=None):
 803         if name_or_class is None:
 804             self.assertEqual(real_value, expected_value)
 805             return
 806         try:
 807             msg = "Invalid %s: %d('%s') does not match expected value %d('%s')"
 808             msg = msg % (getdoc(name_or_class).strip(),
 809                          real_value, str(name_or_class(real_value)),
 810                          expected_value, str(name_or_class(expected_value)))
 811         except Exception:
 812             msg = "Invalid %s: %s does not match expected value %s" % (
 813                 name_or_class, real_value, expected_value)
 814
 815         self.assertEqual(real_value, expected_value, msg)
 816
 817     def assert_in_range(self,
 818                         real_value,
 819                         expected_min,
 820                         expected_max,
 821                         name=None):
 822         if name is None:
 823             msg = None
 824         else:
 825             msg = "Invalid %s: %s out of range <%s,%s>" % (
 826                 name, real_value, expected_min, expected_max)
 827         self.assertTrue(expected_min <= real_value <= expected_max, msg)
 828
 829     def assert_packet_checksums_valid(self, packet,
 830                                       ignore_zero_udp_checksums=True):
 831         received = packet.__class__(str(packet))
 832         self.logger.debug(
 833             ppp("Verifying packet checksums for packet:", received))
 834         udp_layers = ['UDP', 'UDPerror']
 835         checksum_fields = ['cksum', 'chksum']
 836         checksums = []
 837         counter = 0
 838         temp = received.__class__(str(received))
 839         while True:
 840             layer = temp.getlayer(counter)
 841             if layer:
 842                 for cf in checksum_fields:
 843                     if hasattr(layer, cf):
 844                         if ignore_zero_udp_checksums and \
 845                                         0 == getattr(layer, cf) and \
 846                                         layer.name in udp_layers:
 847                             continue
 848                         delattr(layer, cf)
 849                         checksums.append((counter, cf))
 850             else:
 851                 break
 852             counter = counter + 1
 853         if 0 == len(checksums):
 854             return
 855         temp = temp.__class__(str(temp))
 856         for layer, cf in checksums:
 857             calc_sum = getattr(temp[layer], cf)
 858             self.assert_equal(
 859                 getattr(received[layer], cf), calc_sum,
 860                 "packet checksum on layer #%d: %s" % (layer, temp[layer].name))
 861             self.logger.debug(
 862                 "Checksum field `%s` on `%s` layer has correct value `%s`" %
 863                 (cf, temp[layer].name, calc_sum))
 864
 865     def assert_checksum_valid(self, received_packet, layer,
 866                               field_name='chksum',
 867                               ignore_zero_checksum=False):
 868         """ Check checksum of received packet on given layer """
 869         received_packet_checksum = getattr(received_packet[layer], field_name)
 870         if ignore_zero_checksum and 0 == received_packet_checksum:
 871             return
 872         recalculated = received_packet.__class__(str(received_packet))
 873         delattr(recalculated[layer], field_name)
 874         recalculated = recalculated.__class__(str(recalculated))
 875         self.assert_equal(received_packet_checksum,
 876                           getattr(recalculated[layer], field_name),
 877                           "packet checksum on layer: %s" % layer)
 878
 879     def assert_ip_checksum_valid(self, received_packet,
 880                                  ignore_zero_checksum=False):
 881         self.assert_checksum_valid(received_packet, 'IP',
 882                                    ignore_zero_checksum=ignore_zero_checksum)
 883
 884     def assert_tcp_checksum_valid(self, received_packet,
 885                                   ignore_zero_checksum=False):
 886         self.assert_checksum_valid(received_packet, 'TCP',
 887                                    ignore_zero_checksum=ignore_zero_checksum)
 888
 889     def assert_udp_checksum_valid(self, received_packet,
 890                                   ignore_zero_checksum=True):
 891         self.assert_checksum_valid(received_packet, 'UDP',
 892                                    ignore_zero_checksum=ignore_zero_checksum)
 893
 894     def assert_embedded_icmp_checksum_valid(self, received_packet):
 895         if received_packet.haslayer(IPerror):
 896             self.assert_checksum_valid(received_packet, 'IPerror')
 897         if received_packet.haslayer(TCPerror):
 898             self.assert_checksum_valid(received_packet, 'TCPerror')
 899         if received_packet.haslayer(UDPerror):
 900             self.assert_checksum_valid(received_packet, 'UDPerror',
 901                                        ignore_zero_checksum=True)
 902         if received_packet.haslayer(ICMPerror):
 903             self.assert_checksum_valid(received_packet, 'ICMPerror')
 904
 905     def assert_icmp_checksum_valid(self, received_packet):
 906         self.assert_checksum_valid(received_packet, 'ICMP')
 907         self.assert_embedded_icmp_checksum_valid(received_packet)
 908
 909     def assert_icmpv6_checksum_valid(self, pkt):
 910         if pkt.haslayer(ICMPv6DestUnreach):
 911             self.assert_checksum_valid(pkt, 'ICMPv6DestUnreach', 'cksum')
 912             self.assert_embedded_icmp_checksum_valid(pkt)
 913         if pkt.haslayer(ICMPv6EchoRequest):
 914             self.assert_checksum_valid(pkt, 'ICMPv6EchoRequest', 'cksum')
 915         if pkt.haslayer(ICMPv6EchoReply):
 916             self.assert_checksum_valid(pkt, 'ICMPv6EchoReply', 'cksum')
 917
 918     def assert_packet_counter_equal(self, counter, expected_value):
 919         if counter.startswith("/"):
 920             counter_value = self.statistics.get_counter(counter)
 921             self.assert_equal(counter_value, expected_value,
 922                               "packet counter `%s'" % counter)
 923         else:
 924             counters = self.vapi.cli("sh errors").split('\n')
 925             counter_value = -1
 926             for i in range(1, len(counters) - 1):
 927                 results = counters[i].split()
 928                 if results[1] == counter:
 929                     counter_value = int(results[0])
 930                     break
 931
 932     @classmethod
 933     def sleep(cls, timeout, remark=None):
 934         if hasattr(cls, 'logger'):
 935             cls.logger.debug("Starting sleep for %es (%s)", timeout, remark)
 936         before = time.time()
 937         time.sleep(timeout)
 938         after = time.time()
 939         if hasattr(cls, 'logger') and after - before > 2 * timeout:
 940             cls.logger.error("unexpected time.sleep() result - "
 941                              "slept for %es instead of ~%es!",
 942                              after - before, timeout)
 943         if hasattr(cls, 'logger'):
 944             cls.logger.debug(
 945                 "Finished sleep (%s) - slept %es (wanted %es)",
 946                 remark, after - before, timeout)
 947
 948     def send_and_assert_no_replies(self, intf, pkts, remark="", timeout=None):
 949         self.vapi.cli("clear trace")
 950         intf.add_stream(pkts)
 951         self.pg_enable_capture(self.pg_interfaces)
 952         self.pg_start()
 953         if not timeout:
 954             timeout = 1
 955         for i in self.pg_interfaces:
 956             i.get_capture(0, timeout=timeout)
 957             i.assert_nothing_captured(remark=remark)
 958             timeout = 0.1
 959
 960     def send_and_expect(self, input, pkts, output):
 961         self.vapi.cli("clear trace")
 962         input.add_stream(pkts)
 963         self.pg_enable_capture(self.pg_interfaces)
 964         self.pg_start()
 965         if isinstance(object, (list,)):
 966             rx = []
 967             for o in output:
 968                 rx.append(output.get_capture(len(pkts)))
 969         else:
 970             rx = output.get_capture(len(pkts))
 971         return rx
 972
 973     def send_and_expect_only(self, input, pkts, output, timeout=None):
 974         self.vapi.cli("clear trace")
 975         input.add_stream(pkts)
 976         self.pg_enable_capture(self.pg_interfaces)
 977         self.pg_start()
 978         if isinstance(object, (list,)):
 979             outputs = output
 980             rx = []
 981             for o in outputs:
 982                 rx.append(output.get_capture(len(pkts)))
 983         else:
 984             rx = output.get_capture(len(pkts))
 985             outputs = [output]
 986         if not timeout:
 987             timeout = 1
 988         for i in self.pg_interfaces:
 989             if i not in outputs:
 990                 i.get_capture(0, timeout=timeout)
 991                 i.assert_nothing_captured()
 992                 timeout = 0.1
 993
 994         return rx
 995
 996
 997 def get_testcase_doc_name(test):
 998     return getdoc(test.__class__).splitlines()[0]
 999
1000
1001 def get_test_description(descriptions, test):
1002     short_description = test.shortDescription()
1003     if descriptions and short_description:
1004         return short_description
1005     else:
1006         return str(test)
1007
1008
1009 class TestCaseInfo(object):
1010     def __init__(self, logger, tempdir, vpp_pid, vpp_bin_path):
1011         self.logger = logger
1012         self.tempdir = tempdir
1013         self.vpp_pid = vpp_pid
1014         self.vpp_bin_path = vpp_bin_path
1015         self.core_crash_test = None
1016
1017
1018 class VppTestResult(unittest.TestResult):
1019     """
1020     @property result_string
1021      String variable to store the test case result string.
1022     @property errors
1023      List variable containing 2-tuples of TestCase instances and strings
1024      holding formatted tracebacks. Each tuple represents a test which
1025      raised an unexpected exception.
1026     @property failures
1027      List variable containing 2-tuples of TestCase instances and strings
1028      holding formatted tracebacks. Each tuple represents a test where
1029      a failure was explicitly signalled using the TestCase.assert*()
1030      methods.
1031     """
1032
1033     failed_test_cases_info = set()
1034     core_crash_test_cases_info = set()
1035     current_test_case_info = None
1036
1037     def __init__(self, stream, descriptions, verbosity, runner):
1038         """
1039         :param stream File descriptor to store where to report test results.
1040             Set to the standard error stream by default.
1041         :param descriptions Boolean variable to store information if to use
1042             test case descriptions.
1043         :param verbosity Integer variable to store required verbosity level.
1044         """
1045         unittest.TestResult.__init__(self, stream, descriptions, verbosity)
1046         self.stream = stream
1047         self.descriptions = descriptions
1048         self.verbosity = verbosity
1049         self.result_string = None
1050         self.runner = runner
1051
1052     def addSuccess(self, test):
1053         """
1054         Record a test succeeded result
1055
1056         :param test:
1057
1058         """
1059         if self.current_test_case_info:
1060             self.current_test_case_info.logger.debug(
1061                 "--- addSuccess() %s.%s(%s) called" % (test.__class__.__name__,
1062                                                        test._testMethodName,
1063                                                        test._testMethodDoc))
1064         unittest.TestResult.addSuccess(self, test)
1065         self.result_string = colorize("OK", GREEN)
1066
1067         self.send_result_through_pipe(test, PASS)
1068
1069     def addSkip(self, test, reason):
1070         """
1071         Record a test skipped.
1072
1073         :param test:
1074         :param reason:
1075
1076         """
1077         if self.current_test_case_info:
1078             self.current_test_case_info.logger.debug(
1079                 "--- addSkip() %s.%s(%s) called, reason is %s" %
1080                 (test.__class__.__name__, test._testMethodName,
1081                  test._testMethodDoc, reason))
1082         unittest.TestResult.addSkip(self, test, reason)
1083         self.result_string = colorize("SKIP", YELLOW)
1084
1085         self.send_result_through_pipe(test, SKIP)
1086
1087     def symlink_failed(self):
1088         if self.current_test_case_info:
1089             try:
1090                 failed_dir = os.getenv('FAILED_DIR')
1091                 link_path = os.path.join(
1092                     failed_dir,
1093                     '%s-FAILED' %
1094                     os.path.basename(self.current_test_case_info.tempdir))
1095                 if self.current_test_case_info.logger:
1096                     self.current_test_case_info.logger.debug(
1097                         "creating a link to the failed test")
1098                     self.current_test_case_info.logger.debug(
1099                         "os.symlink(%s, %s)" %
1100                         (self.current_test_case_info.tempdir, link_path))
1101                 if os.path.exists(link_path):
1102                     if self.current_test_case_info.logger:
1103                         self.current_test_case_info.logger.debug(
1104                             'symlink already exists')
1105                 else:
1106                     os.symlink(self.current_test_case_info.tempdir, link_path)
1107
1108             except Exception as e:
1109                 if self.current_test_case_info.logger:
1110                     self.current_test_case_info.logger.error(e)
1111
1112     def send_result_through_pipe(self, test, result):
1113         if hasattr(self, 'test_framework_result_pipe'):
1114             pipe = self.test_framework_result_pipe
1115             if pipe:
1116                 pipe.send((test.id(), result))
1117
1118     def log_error(self, test, err, fn_name):
1119         if self.current_test_case_info:
1120             if isinstance(test, unittest.suite._ErrorHolder):
1121                 test_name = test.description
1122             else:
1123                 test_name = '%s.%s(%s)' % (test.__class__.__name__,
1124                                            test._testMethodName,
1125                                            test._testMethodDoc)
1126             self.current_test_case_info.logger.debug(
1127                 "--- %s() %s called, err is %s" %
1128                 (fn_name, test_name, err))
1129             self.current_test_case_info.logger.debug(
1130                 "formatted exception is:\n%s" %
1131                 "".join(format_exception(*err)))
1132
1133     def add_error(self, test, err, unittest_fn, error_type):
1134         if error_type == FAIL:
1135             self.log_error(test, err, 'addFailure')
1136             error_type_str = colorize("FAIL", RED)
1137         elif error_type == ERROR:
1138             self.log_error(test, err, 'addError')
1139             error_type_str = colorize("ERROR", RED)
1140         else:
1141             raise Exception('Error type %s cannot be used to record an '
1142                             'error or a failure' % error_type)
1143
1144         unittest_fn(self, test, err)
1145         if self.current_test_case_info:
1146             self.result_string = "%s [ temp dir used by test case: %s ]" % \
1147                                  (error_type_str,
1148                                   self.current_test_case_info.tempdir)
1149             self.symlink_failed()
1150             self.failed_test_cases_info.add(self.current_test_case_info)
1151             if is_core_present(self.current_test_case_info.tempdir):
1152                 if not self.current_test_case_info.core_crash_test:
1153                     if isinstance(test, unittest.suite._ErrorHolder):
1154                         test_name = str(test)
1155                     else:
1156                         test_name = "'{}' ({})".format(
1157                             get_testcase_doc_name(test), test.id())
1158                     self.current_test_case_info.core_crash_test = test_name
1159                 self.core_crash_test_cases_info.add(
1160                     self.current_test_case_info)
1161         else:
1162             self.result_string = '%s [no temp dir]' % error_type_str
1163
1164         self.send_result_through_pipe(test, error_type)
1165
1166     def addFailure(self, test, err):
1167         """
1168         Record a test failed result
1169
1170         :param test:
1171         :param err: error message
1172
1173         """
1174         self.add_error(test, err, unittest.TestResult.addFailure, FAIL)
1175
1176     def addError(self, test, err):
1177         """
1178         Record a test error result
1179
1180         :param test:
1181         :param err: error message
1182
1183         """
1184         self.add_error(test, err, unittest.TestResult.addError, ERROR)
1185
1186     def getDescription(self, test):
1187         """
1188         Get test description
1189
1190         :param test:
1191         :returns: test description
1192
1193         """
1194         return get_test_description(self.descriptions, test)
1195
1196     def startTest(self, test):
1197         """
1198         Start a test
1199
1200         :param test:
1201
1202         """
1203         test.print_header()
1204
1205         unittest.TestResult.startTest(self, test)
1206         if self.verbosity > 0:
1207             self.stream.writeln(
1208                 "Starting " + self.getDescription(test) + " ...")
1209             self.stream.writeln(single_line_delim)
1210
1211     def stopTest(self, test):
1212         """
1213         Called when the given test has been run
1214
1215         :param test:
1216
1217         """
1218         unittest.TestResult.stopTest(self, test)
1219         if self.verbosity > 0:
1220             self.stream.writeln(single_line_delim)
1221             self.stream.writeln("%-73s%s" % (self.getDescription(test),
1222                                              self.result_string))
1223             self.stream.writeln(single_line_delim)
1224         else:
1225             self.stream.writeln("%-73s%s" % (self.getDescription(test),
1226                                              self.result_string))
1227
1228         self.send_result_through_pipe(test, TEST_RUN)
1229
1230     def printErrors(self):
1231         """
1232         Print errors from running the test case
1233         """
1234         if len(self.errors) > 0 or len(self.failures) > 0:
1235             self.stream.writeln()
1236             self.printErrorList('ERROR', self.errors)
1237             self.printErrorList('FAIL', self.failures)
1238
1239         # ^^ that is the last output from unittest before summary
1240         if not self.runner.print_summary:
1241             devnull = unittest.runner._WritelnDecorator(open(os.devnull, 'w'))
1242             self.stream = devnull
1243             self.runner.stream = devnull
1244
1245     def printErrorList(self, flavour, errors):
1246         """
1247         Print error list to the output stream together with error type
1248         and test case description.
1249
1250         :param flavour: error type
1251         :param errors: iterable errors
1252
1253         """
1254         for test, err in errors:
1255             self.stream.writeln(double_line_delim)
1256             self.stream.writeln("%s: %s" %
1257                                 (flavour, self.getDescription(test)))
1258             self.stream.writeln(single_line_delim)
1259             self.stream.writeln("%s" % err)
1260
1261
1262 class VppTestRunner(unittest.TextTestRunner):
1263     """
1264     A basic test runner implementation which prints results to standard error.
1265     """
1266
1267     @property
1268     def resultclass(self):
1269         """Class maintaining the results of the tests"""
1270         return VppTestResult
1271
1272     def __init__(self, keep_alive_pipe=None, descriptions=True, verbosity=1,
1273                  result_pipe=None, failfast=False, buffer=False,
1274                  resultclass=None, print_summary=True):
1275         # ignore stream setting here, use hard-coded stdout to be in sync
1276         # with prints from VppTestCase methods ...
1277         super(VppTestRunner, self).__init__(sys.stdout, descriptions,
1278                                             verbosity, failfast, buffer,
1279                                             resultclass)
1280         KeepAliveReporter.pipe = keep_alive_pipe
1281
1282         self.orig_stream = self.stream
1283         self.resultclass.test_framework_result_pipe = result_pipe
1284
1285         self.print_summary = print_summary
1286
1287     def _makeResult(self):
1288         return self.resultclass(self.stream,
1289                                 self.descriptions,
1290                                 self.verbosity,
1291                                 self)
1292
1293     def run(self, test):
1294         """
1295         Run the tests
1296
1297         :param test:
1298
1299         """
1300         faulthandler.enable()  # emit stack trace to stderr if killed by signal
1301
1302         result = super(VppTestRunner, self).run(test)
1303         if not self.print_summary:
1304             self.stream = self.orig_stream
1305             result.stream = self.orig_stream
1306         return result
1307
1308
1309 class Worker(Thread):
1310     def __init__(self, args, logger, env={}):
1311         self.logger = logger
1312         self.args = args
1313         self.result = None
1314         self.env = copy.deepcopy(env)
1315         super(Worker, self).__init__()
1316
1317     def run(self):
1318         executable = self.args[0]
1319         self.logger.debug("Running executable w/args `%s'" % self.args)
1320         env = os.environ.copy()
1321         env.update(self.env)
1322         env["CK_LOG_FILE_NAME"] = "-"
1323         self.process = subprocess.Popen(
1324             self.args, shell=False, env=env, preexec_fn=os.setpgrp,
1325             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1326         out, err = self.process.communicate()
1327         self.logger.debug("Finished running `%s'" % executable)
1328         self.logger.info("Return code is `%s'" % self.process.returncode)
1329         self.logger.info(single_line_delim)
1330         self.logger.info("Executable `%s' wrote to stdout:" % executable)
1331         self.logger.info(single_line_delim)
1332         self.logger.info(out)
1333         self.logger.info(single_line_delim)
1334         self.logger.info("Executable `%s' wrote to stderr:" % executable)
1335         self.logger.info(single_line_delim)
1336         self.logger.info(err)
1337         self.logger.info(single_line_delim)
1338         self.result = self.process.returncode