L2-learn; cache prefetch fix

[vpp.git] / test / framework.py

diff --git a/test/framework.py b/test/framework.py
index f90197b..9ad9640 100644 (file)
--- a/test/framework.py
+++ b/test/framework.py
@@ -11,6 +11,7 @@ import time
 import faulthandler
 import random
 import copy
 import faulthandler
 import random
 import copy

+import psutil

 from collections import deque
 from threading import Thread, Event
 from inspect import getdoc, isclass
 from collections import deque
 from threading import Thread, Event
 from inspect import getdoc, isclass


@@ -22,9 +23,11 @@ from vpp_pg_interface import VppPGInterface
 from vpp_sub_interface import VppSubInterface
 from vpp_lo_interface import VppLoInterface
 from vpp_papi_provider import VppPapiProvider
 from vpp_sub_interface import VppSubInterface
 from vpp_lo_interface import VppLoInterface
 from vpp_papi_provider import VppPapiProvider

+from vpp_papi.vpp_stats import VPPStats

 from log import RED, GREEN, YELLOW, double_line_delim, single_line_delim, \
     getLogger, colorize
 from vpp_object import VppObjectRegistry
 from log import RED, GREEN, YELLOW, double_line_delim, single_line_delim, \
     getLogger, colorize
 from vpp_object import VppObjectRegistry

+from util import ppp

 from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror
 from scapy.layers.inet6 import ICMPv6DestUnreach, ICMPv6EchoRequest
 from scapy.layers.inet6 import ICMPv6EchoReply
 from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror
 from scapy.layers.inet6 import ICMPv6DestUnreach, ICMPv6EchoRequest
 from scapy.layers.inet6 import ICMPv6EchoReply


@@ -82,7 +85,7 @@ def pump_output(testclass):
     """ pump output from vpp stdout/stderr to proper queues """
     stdout_fragment = ""
     stderr_fragment = ""
     """ pump output from vpp stdout/stderr to proper queues """
     stdout_fragment = ""
     stderr_fragment = ""

-    while not testclass.pump_thread_stop_flag.wait(0):
+    while not testclass.pump_thread_stop_flag.is_set():

         readable = select.select([testclass.vpp.stdout.fileno(),
                                   testclass.vpp.stderr.fileno(),
                                   testclass.pump_thread_wakeup_pipe[0]],
         readable = select.select([testclass.vpp.stdout.fileno(),
                                   testclass.vpp.stderr.fileno(),
                                   testclass.pump_thread_wakeup_pipe[0]],


@@ -210,6 +213,35 @@ class VppTestCase(unittest.TestCase):
         else:
             raise Exception("Unrecognized DEBUG option: '%s'" % d)
 
         else:
             raise Exception("Unrecognized DEBUG option: '%s'" % d)
 

+    @classmethod
+    def get_least_used_cpu(self):
+        cpu_usage_list = [set(range(psutil.cpu_count()))]
+        vpp_processes = [p for p in psutil.process_iter(attrs=['pid', 'name'])
+                         if 'vpp_main' == p.info['name']]
+        for vpp_process in vpp_processes:
+            for cpu_usage_set in cpu_usage_list:
+                try:
+                    cpu_num = vpp_process.cpu_num()
+                    if cpu_num in cpu_usage_set:
+                        cpu_usage_set_index = cpu_usage_list.index(
+                            cpu_usage_set)
+                        if cpu_usage_set_index == len(cpu_usage_list) - 1:
+                            cpu_usage_list.append({cpu_num})
+                        else:
+                            cpu_usage_list[cpu_usage_set_index + 1].add(
+                                cpu_num)
+                        cpu_usage_set.remove(cpu_num)
+                        break
+                except psutil.NoSuchProcess:
+                    pass
+
+        for cpu_usage_set in cpu_usage_list:
+            if len(cpu_usage_set) > 0:
+                min_usage_set = cpu_usage_set
+                break
+
+        return random.choice(tuple(min_usage_set))
+

     @classmethod
     def setUpConstants(cls):
         """ Set-up the test case class based on environment variables """
     @classmethod
     def setUpConstants(cls):
         """ Set-up the test case class based on environment variables """


@@ -240,12 +272,19 @@ class VppTestCase(unittest.TestCase):
             coredump_size = "coredump-size %s" % size
         if coredump_size is None:
             coredump_size = "coredump-size unlimited"
             coredump_size = "coredump-size %s" % size
         if coredump_size is None:
             coredump_size = "coredump-size unlimited"

+
+        cpu_core_number = cls.get_least_used_cpu()
+

         cls.vpp_cmdline = [cls.vpp_bin, "unix",
                            "{", "nodaemon", debug_cli, "full-coredump",
                            coredump_size, "}", "api-trace", "{", "on", "}",
                            "api-segment", "{", "prefix", cls.shm_prefix, "}",
         cls.vpp_cmdline = [cls.vpp_bin, "unix",
                            "{", "nodaemon", debug_cli, "full-coredump",
                            coredump_size, "}", "api-trace", "{", "on", "}",
                            "api-segment", "{", "prefix", cls.shm_prefix, "}",

+                           "cpu", "{", "main-core", str(cpu_core_number), "}",
+                           "stats", "{", "socket-name",
+                           cls.tempdir + "/stats.sock", "}",

                            "plugins", "{", "plugin", "dpdk_plugin.so", "{",
                            "plugins", "{", "plugin", "dpdk_plugin.so", "{",

-                           "disable", "}", "}", ]
+                           "disable", "}", "plugin", "unittest_plugin.so",
+                           "{", "enable", "}", "}", ]

         if plugin_path is not None:
             cls.vpp_cmdline.extend(["plugin_path", plugin_path])
         cls.logger.info("vpp_cmdline: %s" % cls.vpp_cmdline)
         if plugin_path is not None:
             cls.vpp_cmdline.extend(["plugin_path", plugin_path])
         cls.logger.info("vpp_cmdline: %s" % cls.vpp_cmdline)


@@ -308,7 +347,10 @@ class VppTestCase(unittest.TestCase):
         """
         gc.collect()  # run garbage collection first
         random.seed()
         """
         gc.collect()  # run garbage collection first
         random.seed()

-        cls.logger = getLogger(cls.__name__)
+        if not hasattr(cls, 'logger'):
+            cls.logger = getLogger(cls.__name__)
+        else:
+            cls.logger.name = cls.__name__

         cls.tempdir = tempfile.mkdtemp(
             prefix='vpp-unittest-%s-' % cls.__name__)
         cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir)
         cls.tempdir = tempfile.mkdtemp(
             prefix='vpp-unittest-%s-' % cls.__name__)
         cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir)


@@ -317,7 +359,7 @@ class VppTestCase(unittest.TestCase):
                       datefmt="%H:%M:%S"))
         cls.file_handler.setLevel(DEBUG)
         cls.logger.addHandler(cls.file_handler)
                       datefmt="%H:%M:%S"))
         cls.file_handler.setLevel(DEBUG)
         cls.logger.addHandler(cls.file_handler)

-        cls.shm_prefix = cls.tempdir.split("/")[-1]
+        cls.shm_prefix = os.path.basename(cls.tempdir)

         os.chdir(cls.tempdir)
         cls.logger.info("Temporary dir is %s, shm prefix is %s",
                         cls.tempdir, cls.shm_prefix)
         os.chdir(cls.tempdir)
         cls.logger.info("Temporary dir is %s, shm prefix is %s",
                         cls.tempdir, cls.shm_prefix)


@@ -349,6 +391,7 @@ class VppTestCase(unittest.TestCase):
                 hook = PollHook(cls)
             cls.vapi.register_hook(hook)
             cls.sleep(0.1, "after vpp startup, before initial poll")
                 hook = PollHook(cls)
             cls.vapi.register_hook(hook)
             cls.sleep(0.1, "after vpp startup, before initial poll")

+            cls.statistics = VPPStats(socketname=cls.tempdir+'/stats.sock')

             try:
                 hook.poll_vpp()
             except VppDiedError:
             try:
                 hook.poll_vpp()
             except VppDiedError:


@@ -390,8 +433,11 @@ class VppTestCase(unittest.TestCase):
                 raw_input("When done debugging, press ENTER to kill the "
                           "process and finish running the testcase...")
 
                 raw_input("When done debugging, press ENTER to kill the "
                           "process and finish running the testcase...")
 

-        os.write(cls.pump_thread_wakeup_pipe[1], 'ding dong wake up')
-        cls.pump_thread_stop_flag.set()
+        # first signal that we want to stop the pump thread, then wake it up
+        if hasattr(cls, 'pump_thread_stop_flag'):
+            cls.pump_thread_stop_flag.set()
+        if hasattr(cls, 'pump_thread_wakeup_pipe'):
+            os.write(cls.pump_thread_wakeup_pipe[1], 'ding dong wake up')

         if hasattr(cls, 'pump_thread'):
             cls.logger.debug("Waiting for pump thread to stop")
             cls.pump_thread.join()
         if hasattr(cls, 'pump_thread'):
             cls.logger.debug("Waiting for pump thread to stop")
             cls.pump_thread.join()


@@ -456,7 +502,7 @@ class VppTestCase(unittest.TestCase):
             self.logger.debug(self.vapi.cli("show trace"))
             self.logger.info(self.vapi.ppcli("show interface"))
             self.logger.info(self.vapi.ppcli("show hardware"))
             self.logger.debug(self.vapi.cli("show trace"))
             self.logger.info(self.vapi.ppcli("show interface"))
             self.logger.info(self.vapi.ppcli("show hardware"))

-            self.logger.info(self.vapi.ppcli("show error"))
+            self.logger.info(self.statistics.set_errors_str())

             self.logger.info(self.vapi.ppcli("show run"))
             self.logger.info(self.vapi.ppcli("show log"))
             self.registry.remove_vpp_config(self.logger)
             self.logger.info(self.vapi.ppcli("show run"))
             self.logger.info(self.vapi.ppcli("show log"))
             self.registry.remove_vpp_config(self.logger)


@@ -557,18 +603,16 @@ class VppTestCase(unittest.TestCase):
         return result
 
     @classmethod
         return result
 
     @classmethod

-    def create_loopback_interfaces(cls, interfaces):
+    def create_loopback_interfaces(cls, count):

         """
         Create loopback interfaces.
 
         """
         Create loopback interfaces.
 

-        :param interfaces: iterable indexes of the interfaces.
+        :param count: number of interfaces created.

         :returns: List of created interfaces.
         """
         :returns: List of created interfaces.
         """

-        result = []
-        for i in interfaces:
-            intf = VppLoInterface(cls, i)
+        result = [VppLoInterface(cls) for i in range(count)]
+        for intf in result:

             setattr(cls, intf.name, intf)
             setattr(cls, intf.name, intf)

-            result.append(intf)

         cls.lo_interfaces = result
         return result
 
         cls.lo_interfaces = result
         return result
 


@@ -735,11 +779,14 @@ class VppTestCase(unittest.TestCase):
 
     def assert_packet_checksums_valid(self, packet,
                                       ignore_zero_udp_checksums=True):
 
     def assert_packet_checksums_valid(self, packet,
                                       ignore_zero_udp_checksums=True):

+        received = packet.__class__(str(packet))
+        self.logger.debug(
+            ppp("Verifying packet checksums for packet:", received))

         udp_layers = ['UDP', 'UDPerror']
         checksum_fields = ['cksum', 'chksum']
         checksums = []
         counter = 0
         udp_layers = ['UDP', 'UDPerror']
         checksum_fields = ['cksum', 'chksum']
         checksums = []
         counter = 0

-        temp = packet.__class__(str(packet))
+        temp = received.__class__(str(received))

         while True:
             layer = temp.getlayer(counter)
             if layer:
         while True:
             layer = temp.getlayer(counter)
             if layer:


@@ -754,12 +801,17 @@ class VppTestCase(unittest.TestCase):
             else:
                 break
             counter = counter + 1
             else:
                 break
             counter = counter + 1

+        if 0 == len(checksums):
+            return

         temp = temp.__class__(str(temp))
         for layer, cf in checksums:
         temp = temp.__class__(str(temp))
         for layer, cf in checksums:

-            self.assert_equal(getattr(packet[layer], cf),
-                              getattr(temp[layer], cf),
-                              "packet checksum on layer #%d: %s" % (
-                                  layer, temp[layer].name))
+            calc_sum = getattr(temp[layer], cf)
+            self.assert_equal(
+                getattr(received[layer], cf), calc_sum,
+                "packet checksum on layer #%d: %s" % (layer, temp[layer].name))
+            self.logger.debug(
+                "Checksum field `%s` on `%s` layer has correct value `%s`" %
+                (cf, temp[layer].name, calc_sum))

 
     def assert_checksum_valid(self, received_packet, layer,
                               field_name='chksum',
 
     def assert_checksum_valid(self, received_packet, layer,
                               field_name='chksum',


@@ -830,12 +882,13 @@ class VppTestCase(unittest.TestCase):
                 "Finished sleep (%s) - slept %ss (wanted %ss)" % (
                     remark, after - before, timeout))
 
                 "Finished sleep (%s) - slept %ss (wanted %ss)" % (
                     remark, after - before, timeout))
 

-    def send_and_assert_no_replies(self, intf, pkts, remark=""):
+    def send_and_assert_no_replies(self, intf, pkts, remark="", timeout=None):

         self.vapi.cli("clear trace")
         intf.add_stream(pkts)
         self.pg_enable_capture(self.pg_interfaces)
         self.pg_start()
         self.vapi.cli("clear trace")
         intf.add_stream(pkts)
         self.pg_enable_capture(self.pg_interfaces)
         self.pg_start()

-        timeout = 1
+        if not timeout:
+            timeout = 1

         for i in self.pg_interfaces:
             i.get_capture(0, timeout=timeout)
             i.assert_nothing_captured(remark=remark)
         for i in self.pg_interfaces:
             i.get_capture(0, timeout=timeout)
             i.assert_nothing_captured(remark=remark)


@@ -850,6 +903,19 @@ class VppTestCase(unittest.TestCase):
         return rx
 
 
         return rx
 
 

+def get_testcase_doc_name(test):
+    return getdoc(test.__class__).splitlines()[0]
+
+
+def get_test_description(descriptions, test):
+    # TODO: if none print warning not raise exception
+    short_description = test.shortDescription()
+    if descriptions and short_description:
+        return short_description
+    else:
+        return str(test)
+
+

 class TestCasePrinter(object):
     _shared_state = {}
 
 class TestCasePrinter(object):
     _shared_state = {}
 


@@ -861,7 +927,7 @@ class TestCasePrinter(object):
     def print_test_case_heading_if_first_time(self, case):
         if case.__class__ not in self._test_case_set:
             print(double_line_delim)
     def print_test_case_heading_if_first_time(self, case):
         if case.__class__ not in self._test_case_set:
             print(double_line_delim)

-            print(colorize(getdoc(case.__class__).splitlines()[0], YELLOW))
+            print(colorize(get_testcase_doc_name(case), GREEN))

             print(double_line_delim)
             self._test_case_set.add(case.__class__)
 
             print(double_line_delim)
             self._test_case_set.add(case.__class__)
 


@@ -895,6 +961,7 @@ class VppTestResult(unittest.TestResult):
         self.verbosity = verbosity
         self.result_string = None
         self.printer = TestCasePrinter()
         self.verbosity = verbosity
         self.result_string = None
         self.printer = TestCasePrinter()

+        self.passed = []

 
     def addSuccess(self, test):
         """
 
     def addSuccess(self, test):
         """


@@ -908,6 +975,7 @@ class VppTestResult(unittest.TestResult):
                               % (test.__class__.__name__,
                                  test._testMethodName,
                                  test._testMethodDoc))
                               % (test.__class__.__name__,
                                  test._testMethodName,
                                  test._testMethodDoc))

+        self.passed.append(test.id())

         unittest.TestResult.addSuccess(self, test)
         self.result_string = colorize("OK", GREEN)
 
         unittest.TestResult.addSuccess(self, test)
         self.result_string = colorize("OK", GREEN)
 


@@ -935,22 +1003,27 @@ class VppTestResult(unittest.TestResult):
         if hasattr(test, 'tempdir'):
             try:
                 failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
         if hasattr(test, 'tempdir'):
             try:
                 failed_dir = os.getenv('VPP_TEST_FAILED_DIR')

-                link_path = '%s/%s-FAILED' % (failed_dir,
-                                              test.tempdir.split("/")[-1])
+                link_path = os.path.join(failed_dir, '%s-FAILED' %
+                                         os.path.basename(test.tempdir))

                 if logger:
                     logger.debug("creating a link to the failed test")
                     logger.debug("os.symlink(%s, %s)" %
                                  (test.tempdir, link_path))
                 if logger:
                     logger.debug("creating a link to the failed test")
                     logger.debug("os.symlink(%s, %s)" %
                                  (test.tempdir, link_path))

-                os.symlink(test.tempdir, link_path)
+                if os.path.exists(link_path):
+                    if logger:
+                        logger.debug('symlink already exists')
+                else:
+                    os.symlink(test.tempdir, link_path)
+

             except Exception as e:
                 if logger:
                     logger.error(e)
 
             except Exception as e:
                 if logger:
                     logger.error(e)
 

-    def send_failure_through_pipe(self, test):
-        if hasattr(self, 'test_framework_failed_pipe'):
-            pipe = self.test_framework_failed_pipe
+    def send_results_through_pipe(self):
+        if hasattr(self, 'test_framework_results_pipe'):
+            pipe = self.test_framework_results_pipe

             if pipe:
             if pipe:

-                pipe.send(test.__class__)
+                pipe.send(self)

 
     def addFailure(self, test, err):
         """
 
     def addFailure(self, test, err):
         """


@@ -975,8 +1048,6 @@ class VppTestResult(unittest.TestResult):
         else:
             self.result_string = colorize("FAIL", RED) + ' [no temp dir]'
 
         else:
             self.result_string = colorize("FAIL", RED) + ' [no temp dir]'
 

-        self.send_failure_through_pipe(test)
-

     def addError(self, test, err):
         """
         Record a test error result
     def addError(self, test, err):
         """
         Record a test error result


@@ -1000,8 +1071,6 @@ class VppTestResult(unittest.TestResult):
         else:
             self.result_string = colorize("ERROR", RED) + ' [no temp dir]'
 
         else:
             self.result_string = colorize("ERROR", RED) + ' [no temp dir]'
 

-        self.send_failure_through_pipe(test)
-

     def getDescription(self, test):
         """
         Get test description
     def getDescription(self, test):
         """
         Get test description


@@ -1010,12 +1079,7 @@ class VppTestResult(unittest.TestResult):
         :returns: test description
 
         """
         :returns: test description
 
         """

-        # TODO: if none print warning not raise exception
-        short_description = test.shortDescription()
-        if self.descriptions and short_description:
-            return short_description
-        else:
-            return str(test)
+        return get_test_description(self.descriptions, test)

 
     def startTest(self, test):
         """
 
     def startTest(self, test):
         """


@@ -1033,7 +1097,7 @@ class VppTestResult(unittest.TestResult):
 
     def stopTest(self, test):
         """
 
     def stopTest(self, test):
         """

-        Stop a test
+        Called when the given test has been run

 
         :param test:
 
 
         :param test:
 


@@ -1047,6 +1111,7 @@ class VppTestResult(unittest.TestResult):
         else:
             self.stream.writeln("%-73s%s" % (self.getDescription(test),
                                              self.result_string))
         else:
             self.stream.writeln("%-73s%s" % (self.getDescription(test),
                                              self.result_string))

+        self.send_results_through_pipe()

 
     def printErrors(self):
         """
 
     def printErrors(self):
         """


@@ -1073,22 +1138,6 @@ class VppTestResult(unittest.TestResult):
             self.stream.writeln("%s" % err)
 
 
             self.stream.writeln("%s" % err)
 
 

-class Filter_by_test_option:
-    def __init__(self, filter_file_name, filter_class_name, filter_func_name):
-        self.filter_file_name = filter_file_name
-        self.filter_class_name = filter_class_name
-        self.filter_func_name = filter_func_name
-
-    def __call__(self, file_name, class_name, func_name):
-        if self.filter_file_name and file_name != self.filter_file_name:
-            return False
-        if self.filter_class_name and class_name != self.filter_class_name:
-            return False
-        if self.filter_func_name and func_name != self.filter_func_name:
-            return False
-        return True
-
-

 class VppTestRunner(unittest.TextTestRunner):
     """
     A basic test runner implementation which prints results to standard error.
 class VppTestRunner(unittest.TextTestRunner):
     """
     A basic test runner implementation which prints results to standard error.


@@ -1098,9 +1147,9 @@ class VppTestRunner(unittest.TextTestRunner):
         """Class maintaining the results of the tests"""
         return VppTestResult
 
         """Class maintaining the results of the tests"""
         return VppTestResult
 

-    def __init__(self, keep_alive_pipe=None, failed_pipe=None,
-                 stream=sys.stderr, descriptions=True,
-                 verbosity=1, failfast=False, buffer=False, resultclass=None):
+    def __init__(self, keep_alive_pipe=None, descriptions=True, verbosity=1,
+                 results_pipe=None, failfast=False, buffer=False,
+                 resultclass=None):

         # ignore stream setting here, use hard-coded stdout to be in sync
         # with prints from VppTestCase methods ...
         super(VppTestRunner, self).__init__(sys.stdout, descriptions,
         # ignore stream setting here, use hard-coded stdout to be in sync
         # with prints from VppTestCase methods ...
         super(VppTestRunner, self).__init__(sys.stdout, descriptions,


@@ -1108,63 +1157,8 @@ class VppTestRunner(unittest.TextTestRunner):
                                             resultclass)
         reporter = KeepAliveReporter()
         reporter.pipe = keep_alive_pipe
                                             resultclass)
         reporter = KeepAliveReporter()
         reporter.pipe = keep_alive_pipe

-        # this is super-ugly, but very simple to implement and works as long
-        # as we run only one test at the same time
-        VppTestResult.test_framework_failed_pipe = failed_pipe
-
-    test_option = "TEST"
-
-    def parse_test_option(self):
-        f = os.getenv(self.test_option, None)
-        filter_file_name = None
-        filter_class_name = None
-        filter_func_name = None
-        if f:
-            if '.' in f:
-                parts = f.split('.')
-                if len(parts) > 3:
-                    raise Exception("Unrecognized %s option: %s" %
-                                    (self.test_option, f))
-                if len(parts) > 2:
-                    if parts[2] not in ('*', ''):
-                        filter_func_name = parts[2]
-                if parts[1] not in ('*', ''):
-                    filter_class_name = parts[1]
-                if parts[0] not in ('*', ''):
-                    if parts[0].startswith('test_'):
-                        filter_file_name = parts[0]
-                    else:
-                        filter_file_name = 'test_%s' % parts[0]
-            else:
-                if f.startswith('test_'):
-                    filter_file_name = f
-                else:
-                    filter_file_name = 'test_%s' % f
-        return filter_file_name, filter_class_name, filter_func_name

 
 

-    @staticmethod
-    def filter_tests(tests, filter_cb):
-        result = unittest.suite.TestSuite()
-        for t in tests:
-            if isinstance(t, unittest.suite.TestSuite):
-                # this is a bunch of tests, recursively filter...
-                x = VppTestRunner.filter_tests(t, filter_cb)
-                if x.countTestCases() > 0:
-                    result.addTest(x)
-            elif isinstance(t, unittest.TestCase):
-                # this is a single test
-                parts = t.id().split('.')
-                # t.id() for common cases like this:
-                # test_classifier.TestClassifier.test_acl_ip
-                # apply filtering only if it is so
-                if len(parts) == 3:
-                    if not filter_cb(parts[0], parts[1], parts[2]):
-                        continue
-                result.addTest(t)
-            else:
-                # unexpected object, don't touch it
-                result.addTest(t)
-        return result
+        VppTestResult.test_framework_results_pipe = results_pipe

 
     def run(self, test):
         """
 
     def run(self, test):
         """


@@ -1174,18 +1168,9 @@ class VppTestRunner(unittest.TextTestRunner):
 
         """
         faulthandler.enable()  # emit stack trace to stderr if killed by signal
 
         """
         faulthandler.enable()  # emit stack trace to stderr if killed by signal

-        print("Running tests using custom test runner")  # debug message
-        filter_file, filter_class, filter_func = self.parse_test_option()
-        print("Active filters: file=%s, class=%s, function=%s" % (
-            filter_file, filter_class, filter_func))
-        filter_cb = Filter_by_test_option(
-            filter_file, filter_class, filter_func)
-        filtered = self.filter_tests(test, filter_cb)
-        print("%s out of %s tests match specified filters" % (
-            filtered.countTestCases(), test.countTestCases()))
-        if not running_extended_tests():
-            print("Not running extended tests (some tests will be skipped)")
-        return super(VppTestRunner, self).run(filtered)
+
+        result = super(VppTestRunner, self).run(test)
+        return result

 
 
 class Worker(Thread):
 
 
 class Worker(Thread):