test/framework.py

   1 #!/usr/bin/env python
   2
   3 import subprocess
   4 import unittest
   5 import tempfile
   6 import time
   7 import resource
   8 from collections import deque
   9 from threading import Thread
  10 from inspect import getdoc
  11 from hook import StepHook, PollHook
  12 from vpp_pg_interface import VppPGInterface
  13 from vpp_sub_interface import VppSubInterface
  14 from vpp_lo_interface import VppLoInterface
  15 from vpp_papi_provider import VppPapiProvider
  16 from scapy.packet import Raw
  17 from logging import FileHandler, DEBUG
  18 from log import *
  19 from vpp_object import VppObjectRegistry
  20
  21 """
  22   Test framework module.
  23
  24   The module provides a set of tools for constructing and running tests and
  25   representing the results.
  26 """
  27
  28
  29 class _PacketInfo(object):
  30     """Private class to create packet info object.
  31
  32     Help process information about the next packet.
  33     Set variables to default values.
  34     """
  35     #: Store the index of the packet.
  36     index = -1
  37     #: Store the index of the source packet generator interface of the packet.
  38     src = -1
  39     #: Store the index of the destination packet generator interface
  40     #: of the packet.
  41     dst = -1
  42     #: Store the copy of the former packet.
  43     data = None
  44
  45     def __eq__(self, other):
  46         index = self.index == other.index
  47         src = self.src == other.src
  48         dst = self.dst == other.dst
  49         data = self.data == other.data
  50         return index and src and dst and data
  51
  52
  53 def pump_output(out, deque):
  54     for line in iter(out.readline, b''):
  55         deque.append(line)
  56
  57
  58 class VppTestCase(unittest.TestCase):
  59     """This subclass is a base class for VPP test cases that are implemented as
  60     classes. It provides methods to create and run test case.
  61     """
  62
  63     @property
  64     def packet_infos(self):
  65         """List of packet infos"""
  66         return self._packet_infos
  67
  68     @classmethod
  69     def get_packet_count_for_if_idx(cls, dst_if_index):
  70         """Get the number of packet info for specified destination if index"""
  71         if dst_if_index in cls._packet_count_for_dst_if_idx:
  72             return cls._packet_count_for_dst_if_idx[dst_if_index]
  73         else:
  74             return 0
  75
  76     @classmethod
  77     def instance(cls):
  78         """Return the instance of this testcase"""
  79         return cls.test_instance
  80
  81     @classmethod
  82     def set_debug_flags(cls, d):
  83         cls.debug_core = False
  84         cls.debug_gdb = False
  85         cls.debug_gdbserver = False
  86         if d is None:
  87             return
  88         dl = d.lower()
  89         if dl == "core":
  90             if resource.getrlimit(resource.RLIMIT_CORE)[0] <= 0:
  91                 # give a heads up if this is actually useless
  92                 cls.logger.critical("WARNING: core size limit is set 0, core "
  93                                     "files will NOT be created")
  94             cls.debug_core = True
  95         elif dl == "gdb":
  96             cls.debug_gdb = True
  97         elif dl == "gdbserver":
  98             cls.debug_gdbserver = True
  99         else:
 100             raise Exception("Unrecognized DEBUG option: '%s'" % d)
 101
 102     @classmethod
 103     def setUpConstants(cls):
 104         """ Set-up the test case class based on environment variables """
 105         try:
 106             s = os.getenv("STEP")
 107             cls.step = True if s.lower() in ("y", "yes", "1") else False
 108         except:
 109             cls.step = False
 110         try:
 111             d = os.getenv("DEBUG")
 112         except:
 113             d = None
 114         cls.set_debug_flags(d)
 115         cls.vpp_bin = os.getenv('VPP_TEST_BIN', "vpp")
 116         cls.plugin_path = os.getenv('VPP_TEST_PLUGIN_PATH')
 117         debug_cli = ""
 118         if cls.step or cls.debug_gdb or cls.debug_gdbserver:
 119             debug_cli = "cli-listen localhost:5002"
 120         cls.vpp_cmdline = [cls.vpp_bin, "unix", "{", "nodaemon", debug_cli, "}",
 121                            "api-segment", "{", "prefix", cls.shm_prefix, "}"]
 122         if cls.plugin_path is not None:
 123             cls.vpp_cmdline.extend(["plugin_path", cls.plugin_path])
 124         cls.logger.info("vpp_cmdline: %s" % cls.vpp_cmdline)
 125
 126     @classmethod
 127     def wait_for_enter(cls):
 128         if cls.debug_gdbserver:
 129             print(double_line_delim)
 130             print("Spawned GDB server with PID: %d" % cls.vpp.pid)
 131         elif cls.debug_gdb:
 132             print(double_line_delim)
 133             print("Spawned VPP with PID: %d" % cls.vpp.pid)
 134         else:
 135             cls.logger.debug("Spawned VPP with PID: %d" % cls.vpp.pid)
 136             return
 137         print(single_line_delim)
 138         print("You can debug the VPP using e.g.:")
 139         if cls.debug_gdbserver:
 140             print("gdb " + cls.vpp_bin + " -ex 'target remote localhost:7777'")
 141             print("Now is the time to attach a gdb by running the above "
 142                   "command, set up breakpoints etc. and then resume VPP from "
 143                   "within gdb by issuing the 'continue' command")
 144         elif cls.debug_gdb:
 145             print("gdb " + cls.vpp_bin + " -ex 'attach %s'" % cls.vpp.pid)
 146             print("Now is the time to attach a gdb by running the above "
 147                   "command and set up breakpoints etc.")
 148         print(single_line_delim)
 149         raw_input("Press ENTER to continue running the testcase...")
 150
 151     @classmethod
 152     def run_vpp(cls):
 153         cmdline = cls.vpp_cmdline
 154
 155         if cls.debug_gdbserver:
 156             gdbserver = '/usr/bin/gdbserver'
 157             if not os.path.isfile(gdbserver) or \
 158                     not os.access(gdbserver, os.X_OK):
 159                 raise Exception("gdbserver binary '%s' does not exist or is "
 160                                 "not executable" % gdbserver)
 161
 162             cmdline = [gdbserver, 'localhost:7777'] + cls.vpp_cmdline
 163             cls.logger.info("Gdbserver cmdline is %s", " ".join(cmdline))
 164
 165         try:
 166             cls.vpp = subprocess.Popen(cmdline,
 167                                        stdout=subprocess.PIPE,
 168                                        stderr=subprocess.PIPE,
 169                                        bufsize=1)
 170         except Exception as e:
 171             cls.logger.critical("Couldn't start vpp: %s" % e)
 172             raise
 173
 174         cls.wait_for_enter()
 175
 176     @classmethod
 177     def setUpClass(cls):
 178         """
 179         Perform class setup before running the testcase
 180         Remove shared memory files, start vpp and connect the vpp-api
 181         """
 182         cls.logger = getLogger(cls.__name__)
 183         cls.tempdir = tempfile.mkdtemp(
 184             prefix='vpp-unittest-' + cls.__name__ + '-')
 185         file_handler = FileHandler("%s/log.txt" % cls.tempdir)
 186         file_handler.setLevel(DEBUG)
 187         cls.logger.addHandler(file_handler)
 188         cls.shm_prefix = cls.tempdir.split("/")[-1]
 189         os.chdir(cls.tempdir)
 190         cls.logger.info("Temporary dir is %s, shm prefix is %s",
 191                         cls.tempdir, cls.shm_prefix)
 192         cls.setUpConstants()
 193         cls.reset_packet_infos()
 194         cls._captures = []
 195         cls._zombie_captures = []
 196         cls.verbose = 0
 197         cls.vpp_dead = False
 198         cls.registry = VppObjectRegistry()
 199         print(double_line_delim)
 200         print(colorize(getdoc(cls).splitlines()[0], YELLOW))
 201         print(double_line_delim)
 202         # need to catch exceptions here because if we raise, then the cleanup
 203         # doesn't get called and we might end with a zombie vpp
 204         try:
 205             cls.run_vpp()
 206             cls.vpp_stdout_deque = deque()
 207             cls.vpp_stdout_reader_thread = Thread(target=pump_output, args=(
 208                 cls.vpp.stdout, cls.vpp_stdout_deque))
 209             cls.vpp_stdout_reader_thread.start()
 210             cls.vpp_stderr_deque = deque()
 211             cls.vpp_stderr_reader_thread = Thread(target=pump_output, args=(
 212                 cls.vpp.stderr, cls.vpp_stderr_deque))
 213             cls.vpp_stderr_reader_thread.start()
 214             cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls)
 215             if cls.step:
 216                 hook = StepHook(cls)
 217             else:
 218                 hook = PollHook(cls)
 219             cls.vapi.register_hook(hook)
 220             time.sleep(0.1)
 221             hook.poll_vpp()
 222             try:
 223                 cls.vapi.connect()
 224             except:
 225                 if cls.debug_gdbserver:
 226                     print(colorize("You're running VPP inside gdbserver but "
 227                                    "VPP-API connection failed, did you forget "
 228                                    "to 'continue' VPP from within gdb?", RED))
 229                 raise
 230         except:
 231             t, v, tb = sys.exc_info()
 232             try:
 233                 cls.quit()
 234             except:
 235                 pass
 236             raise t, v, tb
 237
 238     @classmethod
 239     def quit(cls):
 240         """
 241         Disconnect vpp-api, kill vpp and cleanup shared memory files
 242         """
 243         if (cls.debug_gdbserver or cls.debug_gdb) and hasattr(cls, 'vpp'):
 244             cls.vpp.poll()
 245             if cls.vpp.returncode is None:
 246                 print(double_line_delim)
 247                 print("VPP or GDB server is still running")
 248                 print(single_line_delim)
 249                 raw_input("When done debugging, press ENTER to kill the process"
 250                           " and finish running the testcase...")
 251
 252         if hasattr(cls, 'vpp'):
 253             if hasattr(cls, 'vapi'):
 254                 cls.vapi.disconnect()
 255             cls.vpp.poll()
 256             if cls.vpp.returncode is None:
 257                 cls.vpp.terminate()
 258             del cls.vpp
 259
 260         if hasattr(cls, 'vpp_stdout_deque'):
 261             cls.logger.info(single_line_delim)
 262             cls.logger.info('VPP output to stdout while running %s:',
 263                             cls.__name__)
 264             cls.logger.info(single_line_delim)
 265             f = open(cls.tempdir + '/vpp_stdout.txt', 'w')
 266             vpp_output = "".join(cls.vpp_stdout_deque)
 267             f.write(vpp_output)
 268             cls.logger.info('\n%s', vpp_output)
 269             cls.logger.info(single_line_delim)
 270
 271         if hasattr(cls, 'vpp_stderr_deque'):
 272             cls.logger.info(single_line_delim)
 273             cls.logger.info('VPP output to stderr while running %s:',
 274                             cls.__name__)
 275             cls.logger.info(single_line_delim)
 276             f = open(cls.tempdir + '/vpp_stderr.txt', 'w')
 277             vpp_output = "".join(cls.vpp_stderr_deque)
 278             f.write(vpp_output)
 279             cls.logger.info('\n%s', vpp_output)
 280             cls.logger.info(single_line_delim)
 281
 282     @classmethod
 283     def tearDownClass(cls):
 284         """ Perform final cleanup after running all tests in this test-case """
 285         cls.quit()
 286
 287     def tearDown(self):
 288         """ Show various debug prints after each test """
 289         if not self.vpp_dead:
 290             self.logger.debug(self.vapi.cli("show trace"))
 291             self.logger.info(self.vapi.ppcli("show int"))
 292             self.logger.info(self.vapi.ppcli("show hardware"))
 293             self.logger.info(self.vapi.ppcli("show error"))
 294             self.logger.info(self.vapi.ppcli("show run"))
 295             self.registry.remove_vpp_config(self.logger)
 296
 297     def setUp(self):
 298         """ Clear trace before running each test"""
 299         if self.vpp_dead:
 300             raise Exception("VPP is dead when setting up the test")
 301         time.sleep(.1)
 302         self.vpp_stdout_deque.append(
 303             "--- test setUp() for %s.%s(%s) starts here ---\n" %
 304             (self.__class__.__name__, self._testMethodName,
 305              self._testMethodDoc))
 306         self.vpp_stderr_deque.append(
 307             "--- test setUp() for %s.%s(%s) starts here ---\n" %
 308             (self.__class__.__name__, self._testMethodName,
 309              self._testMethodDoc))
 310         self.vapi.cli("clear trace")
 311         # store the test instance inside the test class - so that objects
 312         # holding the class can access instance methods (like assertEqual)
 313         type(self).test_instance = self
 314
 315     @classmethod
 316     def pg_enable_capture(cls, interfaces):
 317         """
 318         Enable capture on packet-generator interfaces
 319
 320         :param interfaces: iterable interface indexes
 321
 322         """
 323         for i in interfaces:
 324             i.enable_capture()
 325
 326     @classmethod
 327     def register_capture(cls, cap_name):
 328         """ Register a capture in the testclass """
 329         # add to the list of captures with current timestamp
 330         cls._captures.append((time.time(), cap_name))
 331         # filter out from zombies
 332         cls._zombie_captures = [(stamp, name)
 333                                 for (stamp, name) in cls._zombie_captures
 334                                 if name != cap_name]
 335
 336     @classmethod
 337     def pg_start(cls):
 338         """ Remove any zombie captures and enable the packet generator """
 339         # how long before capture is allowed to be deleted - otherwise vpp
 340         # crashes - 100ms seems enough (this shouldn't be needed at all)
 341         capture_ttl = 0.1
 342         now = time.time()
 343         for stamp, cap_name in cls._zombie_captures:
 344             wait = stamp + capture_ttl - now
 345             if wait > 0:
 346                 cls.logger.debug("Waiting for %ss before deleting capture %s",
 347                                  wait, cap_name)
 348                 time.sleep(wait)
 349                 now = time.time()
 350             cls.logger.debug("Removing zombie capture %s" % cap_name)
 351             cls.vapi.cli('packet-generator delete %s' % cap_name)
 352
 353         cls.vapi.cli("trace add pg-input 50")  # 50 is maximum
 354         cls.vapi.cli('packet-generator enable')
 355         cls._zombie_captures = cls._captures
 356         cls._captures = []
 357
 358     @classmethod
 359     def create_pg_interfaces(cls, interfaces):
 360         """
 361         Create packet-generator interfaces.
 362
 363         :param interfaces: iterable indexes of the interfaces.
 364         :returns: List of created interfaces.
 365
 366         """
 367         result = []
 368         for i in interfaces:
 369             intf = VppPGInterface(cls, i)
 370             setattr(cls, intf.name, intf)
 371             result.append(intf)
 372         cls.pg_interfaces = result
 373         return result
 374
 375     @classmethod
 376     def create_loopback_interfaces(cls, interfaces):
 377         """
 378         Create loopback interfaces.
 379
 380         :param interfaces: iterable indexes of the interfaces.
 381         :returns: List of created interfaces.
 382         """
 383         result = []
 384         for i in interfaces:
 385             intf = VppLoInterface(cls, i)
 386             setattr(cls, intf.name, intf)
 387             result.append(intf)
 388         cls.lo_interfaces = result
 389         return result
 390
 391     @staticmethod
 392     def extend_packet(packet, size):
 393         """
 394         Extend packet to given size by padding with spaces
 395         NOTE: Currently works only when Raw layer is present.
 396
 397         :param packet: packet
 398         :param size: target size
 399
 400         """
 401         packet_len = len(packet) + 4
 402         extend = size - packet_len
 403         if extend > 0:
 404             packet[Raw].load += ' ' * extend
 405
 406     @classmethod
 407     def reset_packet_infos(cls):
 408         """ Reset the list of packet info objects and packet counts to zero """
 409         cls._packet_infos = {}
 410         cls._packet_count_for_dst_if_idx = {}
 411
 412     @classmethod
 413     def create_packet_info(cls, src_if, dst_if):
 414         """
 415         Create packet info object containing the source and destination indexes
 416         and add it to the testcase's packet info list
 417
 418         :param VppInterface src_if: source interface
 419         :param VppInterface dst_if: destination interface
 420
 421         :returns: _PacketInfo object
 422
 423         """
 424         info = _PacketInfo()
 425         info.index = len(cls._packet_infos)
 426         info.src = src_if.sw_if_index
 427         info.dst = dst_if.sw_if_index
 428         if isinstance(dst_if, VppSubInterface):
 429             dst_idx = dst_if.parent.sw_if_index
 430         else:
 431             dst_idx = dst_if.sw_if_index
 432         if dst_idx in cls._packet_count_for_dst_if_idx:
 433             cls._packet_count_for_dst_if_idx[dst_idx] += 1
 434         else:
 435             cls._packet_count_for_dst_if_idx[dst_idx] = 1
 436         cls._packet_infos[info.index] = info
 437         return info
 438
 439     @staticmethod
 440     def info_to_payload(info):
 441         """
 442         Convert _PacketInfo object to packet payload
 443
 444         :param info: _PacketInfo object
 445
 446         :returns: string containing serialized data from packet info
 447         """
 448         return "%d %d %d" % (info.index, info.src, info.dst)
 449
 450     @staticmethod
 451     def payload_to_info(payload):
 452         """
 453         Convert packet payload to _PacketInfo object
 454
 455         :param payload: packet payload
 456
 457         :returns: _PacketInfo object containing de-serialized data from payload
 458
 459         """
 460         numbers = payload.split()
 461         info = _PacketInfo()
 462         info.index = int(numbers[0])
 463         info.src = int(numbers[1])
 464         info.dst = int(numbers[2])
 465         return info
 466
 467     def get_next_packet_info(self, info):
 468         """
 469         Iterate over the packet info list stored in the testcase
 470         Start iteration with first element if info is None
 471         Continue based on index in info if info is specified
 472
 473         :param info: info or None
 474         :returns: next info in list or None if no more infos
 475         """
 476         if info is None:
 477             next_index = 0
 478         else:
 479             next_index = info.index + 1
 480         if next_index == len(self._packet_infos):
 481             return None
 482         else:
 483             return self._packet_infos[next_index]
 484
 485     def get_next_packet_info_for_interface(self, src_index, info):
 486         """
 487         Search the packet info list for the next packet info with same source
 488         interface index
 489
 490         :param src_index: source interface index to search for
 491         :param info: packet info - where to start the search
 492         :returns: packet info or None
 493
 494         """
 495         while True:
 496             info = self.get_next_packet_info(info)
 497             if info is None:
 498                 return None
 499             if info.src == src_index:
 500                 return info
 501
 502     def get_next_packet_info_for_interface2(self, src_index, dst_index, info):
 503         """
 504         Search the packet info list for the next packet info with same source
 505         and destination interface indexes
 506
 507         :param src_index: source interface index to search for
 508         :param dst_index: destination interface index to search for
 509         :param info: packet info - where to start the search
 510         :returns: packet info or None
 511
 512         """
 513         while True:
 514             info = self.get_next_packet_info_for_interface(src_index, info)
 515             if info is None:
 516                 return None
 517             if info.dst == dst_index:
 518                 return info
 519
 520     def assert_equal(self, real_value, expected_value, name_or_class=None):
 521         if name_or_class is None:
 522             self.assertEqual(real_value, expected_value, msg)
 523             return
 524         try:
 525             msg = "Invalid %s: %d('%s') does not match expected value %d('%s')"
 526             msg = msg % (getdoc(name_or_class).strip(),
 527                          real_value, str(name_or_class(real_value)),
 528                          expected_value, str(name_or_class(expected_value)))
 529         except:
 530             msg = "Invalid %s: %s does not match expected value %s" % (
 531                 name_or_class, real_value, expected_value)
 532
 533         self.assertEqual(real_value, expected_value, msg)
 534
 535     def assert_in_range(
 536             self,
 537             real_value,
 538             expected_min,
 539             expected_max,
 540             name=None):
 541         if name is None:
 542             msg = None
 543         else:
 544             msg = "Invalid %s: %s out of range <%s,%s>" % (
 545                 name, real_value, expected_min, expected_max)
 546         self.assertTrue(expected_min <= real_value <= expected_max, msg)
 547
 548
 549 class VppTestResult(unittest.TestResult):
 550     """
 551     @property result_string
 552      String variable to store the test case result string.
 553     @property errors
 554      List variable containing 2-tuples of TestCase instances and strings
 555      holding formatted tracebacks. Each tuple represents a test which
 556      raised an unexpected exception.
 557     @property failures
 558      List variable containing 2-tuples of TestCase instances and strings
 559      holding formatted tracebacks. Each tuple represents a test where
 560      a failure was explicitly signalled using the TestCase.assert*()
 561      methods.
 562     """
 563
 564     def __init__(self, stream, descriptions, verbosity):
 565         """
 566         :param stream File descriptor to store where to report test results. Set
 567             to the standard error stream by default.
 568         :param descriptions Boolean variable to store information if to use test
 569             case descriptions.
 570         :param verbosity Integer variable to store required verbosity level.
 571         """
 572         unittest.TestResult.__init__(self, stream, descriptions, verbosity)
 573         self.stream = stream
 574         self.descriptions = descriptions
 575         self.verbosity = verbosity
 576         self.result_string = None
 577
 578     def addSuccess(self, test):
 579         """
 580         Record a test succeeded result
 581
 582         :param test:
 583
 584         """
 585         unittest.TestResult.addSuccess(self, test)
 586         self.result_string = colorize("OK", GREEN)
 587
 588     def addSkip(self, test, reason):
 589         """
 590         Record a test skipped.
 591
 592         :param test:
 593         :param reason:
 594
 595         """
 596         unittest.TestResult.addSkip(self, test, reason)
 597         self.result_string = colorize("SKIP", YELLOW)
 598
 599     def addFailure(self, test, err):
 600         """
 601         Record a test failed result
 602
 603         :param test:
 604         :param err: error message
 605
 606         """
 607         unittest.TestResult.addFailure(self, test, err)
 608         if hasattr(test, 'tempdir'):
 609             self.result_string = colorize("FAIL", RED) + \
 610                 ' [ temp dir used by test case: ' + test.tempdir + ' ]'
 611         else:
 612             self.result_string = colorize("FAIL", RED) + ' [no temp dir]'
 613
 614     def addError(self, test, err):
 615         """
 616         Record a test error result
 617
 618         :param test:
 619         :param err: error message
 620
 621         """
 622         unittest.TestResult.addError(self, test, err)
 623         if hasattr(test, 'tempdir'):
 624             self.result_string = colorize("ERROR", RED) + \
 625                 ' [ temp dir used by test case: ' + test.tempdir + ' ]'
 626         else:
 627             self.result_string = colorize("ERROR", RED) + ' [no temp dir]'
 628
 629     def getDescription(self, test):
 630         """
 631         Get test description
 632
 633         :param test:
 634         :returns: test description
 635
 636         """
 637         # TODO: if none print warning not raise exception
 638         short_description = test.shortDescription()
 639         if self.descriptions and short_description:
 640             return short_description
 641         else:
 642             return str(test)
 643
 644     def startTest(self, test):
 645         """
 646         Start a test
 647
 648         :param test:
 649
 650         """
 651         unittest.TestResult.startTest(self, test)
 652         if self.verbosity > 0:
 653             self.stream.writeln(
 654                 "Starting " + self.getDescription(test) + " ...")
 655             self.stream.writeln(single_line_delim)
 656
 657     def stopTest(self, test):
 658         """
 659         Stop a test
 660
 661         :param test:
 662
 663         """
 664         unittest.TestResult.stopTest(self, test)
 665         if self.verbosity > 0:
 666             self.stream.writeln(single_line_delim)
 667             self.stream.writeln("%-60s%s" %
 668                                 (self.getDescription(test), self.result_string))
 669             self.stream.writeln(single_line_delim)
 670         else:
 671             self.stream.writeln("%-60s%s" %
 672                                 (self.getDescription(test), self.result_string))
 673
 674     def printErrors(self):
 675         """
 676         Print errors from running the test case
 677         """
 678         self.stream.writeln()
 679         self.printErrorList('ERROR', self.errors)
 680         self.printErrorList('FAIL', self.failures)
 681
 682     def printErrorList(self, flavour, errors):
 683         """
 684         Print error list to the output stream together with error type
 685         and test case description.
 686
 687         :param flavour: error type
 688         :param errors: iterable errors
 689
 690         """
 691         for test, err in errors:
 692             self.stream.writeln(double_line_delim)
 693             self.stream.writeln("%s: %s" %
 694                                 (flavour, self.getDescription(test)))
 695             self.stream.writeln(single_line_delim)
 696             self.stream.writeln("%s" % err)
 697
 698
 699 class VppTestRunner(unittest.TextTestRunner):
 700     """
 701     A basic test runner implementation which prints results on standard error.
 702     """
 703     @property
 704     def resultclass(self):
 705         """Class maintaining the results of the tests"""
 706         return VppTestResult
 707
 708     def run(self, test):
 709         """
 710         Run the tests
 711
 712         :param test:
 713
 714         """
 715         print("Running tests using custom test runner")  # debug message
 716         return super(VppTestRunner, self).run(test)