make test: improve console output messages
[vpp.git] / test / framework.py
index 6b1799a..fd493db 100644 (file)
@@ -9,6 +9,7 @@ import unittest
 import tempfile
 import time
 import resource
+import faulthandler
 from collections import deque
 from threading import Thread, Event
 from inspect import getdoc
@@ -50,6 +51,10 @@ class _PacketInfo(object):
     #: Store the index of the destination packet generator interface
     #: of the packet.
     dst = -1
+    #: Store expected ip version
+    ip = -1
+    #: Store expected upper protocol
+    proto = -1
     #: Store the copy of the former packet.
     data = None
 
@@ -142,6 +147,16 @@ class VppTestCase(unittest.TestCase):
         cls.set_debug_flags(d)
         cls.vpp_bin = os.getenv('VPP_TEST_BIN', "vpp")
         cls.plugin_path = os.getenv('VPP_TEST_PLUGIN_PATH')
+        cls.extern_plugin_path = os.getenv('EXTERN_PLUGINS')
+        plugin_path = None
+        if cls.plugin_path is not None:
+            if cls.extern_plugin_path is not None:
+                plugin_path = "%s:%s" % (
+                    cls.plugin_path, cls.extern_plugin_path)
+            else:
+                plugin_path = cls.plugin_path
+        elif cls.extern_plugin_path is not None:
+            plugin_path = cls.extern_plugin_path
         debug_cli = ""
         if cls.step or cls.debug_gdb or cls.debug_gdbserver:
             debug_cli = "cli-listen localhost:5002"
@@ -157,9 +172,11 @@ class VppTestCase(unittest.TestCase):
         cls.vpp_cmdline = [cls.vpp_bin, "unix",
                            "{", "nodaemon", debug_cli, coredump_size, "}",
                            "api-trace", "{", "on", "}",
-                           "api-segment", "{", "prefix", cls.shm_prefix, "}"]
-        if cls.plugin_path is not None:
-            cls.vpp_cmdline.extend(["plugin_path", cls.plugin_path])
+                           "api-segment", "{", "prefix", cls.shm_prefix, "}",
+                           "plugins", "{", "plugin", "dpdk_plugin.so", "{",
+                           "disable", "}", "}"]
+        if plugin_path is not None:
+            cls.vpp_cmdline.extend(["plugin_path", plugin_path])
         cls.logger.info("vpp_cmdline: %s" % cls.vpp_cmdline)
 
     @classmethod
@@ -222,12 +239,12 @@ class VppTestCase(unittest.TestCase):
         cls.logger = getLogger(cls.__name__)
         cls.tempdir = tempfile.mkdtemp(
             prefix='vpp-unittest-' + cls.__name__ + '-')
-        file_handler = FileHandler("%s/log.txt" % cls.tempdir)
-        file_handler.setFormatter(
+        cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir)
+        cls.file_handler.setFormatter(
             Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s',
                       datefmt="%H:%M:%S"))
-        file_handler.setLevel(DEBUG)
-        cls.logger.addHandler(file_handler)
+        cls.file_handler.setLevel(DEBUG)
+        cls.logger.addHandler(cls.file_handler)
         cls.shm_prefix = cls.tempdir.split("/")[-1]
         os.chdir(cls.tempdir)
         cls.logger.info("Temporary dir is %s, shm prefix is %s",
@@ -239,6 +256,7 @@ class VppTestCase(unittest.TestCase):
         cls.verbose = 0
         cls.vpp_dead = False
         cls.registry = VppObjectRegistry()
+        cls.vpp_startup_failed = False
         # need to catch exceptions here because if we raise, then the cleanup
         # doesn't get called and we might end with a zombie vpp
         try:
@@ -257,7 +275,14 @@ class VppTestCase(unittest.TestCase):
                 hook = PollHook(cls)
             cls.vapi.register_hook(hook)
             cls.sleep(0.1, "after vpp startup, before initial poll")
-            hook.poll_vpp()
+            try:
+                hook.poll_vpp()
+            except:
+                cls.vpp_startup_failed = True
+                cls.logger.critical(
+                    "VPP died shortly after startup, check the"
+                    " output to standard error for possible cause")
+                raise
             try:
                 cls.vapi.connect()
             except:
@@ -309,32 +334,38 @@ class VppTestCase(unittest.TestCase):
                 cls.vpp.communicate()
             del cls.vpp
 
+        if cls.vpp_startup_failed:
+            stdout_log = cls.logger.info
+            stderr_log = cls.logger.critical
+        else:
+            stdout_log = cls.logger.info
+            stderr_log = cls.logger.info
+
         if hasattr(cls, 'vpp_stdout_deque'):
-            cls.logger.info(single_line_delim)
-            cls.logger.info('VPP output to stdout while running %s:',
-                            cls.__name__)
-            cls.logger.info(single_line_delim)
-            f = open(cls.tempdir + '/vpp_stdout.txt', 'w')
+            stdout_log(single_line_delim)
+            stdout_log('VPP output to stdout while running %s:', cls.__name__)
+            stdout_log(single_line_delim)
             vpp_output = "".join(cls.vpp_stdout_deque)
-            f.write(vpp_output)
-            cls.logger.info('\n%s', vpp_output)
-            cls.logger.info(single_line_delim)
+            with open(cls.tempdir + '/vpp_stdout.txt', 'w') as f:
+                f.write(vpp_output)
+            stdout_log('\n%s', vpp_output)
+            stdout_log(single_line_delim)
 
         if hasattr(cls, 'vpp_stderr_deque'):
-            cls.logger.info(single_line_delim)
-            cls.logger.info('VPP output to stderr while running %s:',
-                            cls.__name__)
-            cls.logger.info(single_line_delim)
-            f = open(cls.tempdir + '/vpp_stderr.txt', 'w')
+            stderr_log(single_line_delim)
+            stderr_log('VPP output to stderr while running %s:', cls.__name__)
+            stderr_log(single_line_delim)
             vpp_output = "".join(cls.vpp_stderr_deque)
-            f.write(vpp_output)
-            cls.logger.info('\n%s', vpp_output)
-            cls.logger.info(single_line_delim)
+            with open(cls.tempdir + '/vpp_stderr.txt', 'w') as f:
+                f.write(vpp_output)
+            stderr_log('\n%s', vpp_output)
+            stderr_log(single_line_delim)
 
     @classmethod
     def tearDownClass(cls):
         """ Perform final cleanup after running all tests in this test-case """
         cls.quit()
+        cls.file_handler.close()
 
     def tearDown(self):
         """ Show various debug prints after each test """
@@ -343,7 +374,7 @@ class VppTestCase(unittest.TestCase):
                            self._testMethodDoc))
         if not self.vpp_dead:
             self.logger.debug(self.vapi.cli("show trace"))
-            self.logger.info(self.vapi.ppcli("show int"))
+            self.logger.info(self.vapi.ppcli("show interface"))
             self.logger.info(self.vapi.ppcli("show hardware"))
             self.logger.info(self.vapi.ppcli("show error"))
             self.logger.info(self.vapi.ppcli("show run"))
@@ -513,7 +544,8 @@ class VppTestCase(unittest.TestCase):
 
         :returns: string containing serialized data from packet info
         """
-        return "%d %d %d" % (info.index, info.src, info.dst)
+        return "%d %d %d %d %d" % (info.index, info.src, info.dst,
+                                   info.ip, info.proto)
 
     @staticmethod
     def payload_to_info(payload):
@@ -530,6 +562,8 @@ class VppTestCase(unittest.TestCase):
         info.index = int(numbers[0])
         info.src = int(numbers[1])
         info.dst = int(numbers[2])
+        info.ip = int(numbers[3])
+        info.proto = int(numbers[4])
         return info
 
     def get_next_packet_info(self, info):
@@ -615,8 +649,18 @@ class VppTestCase(unittest.TestCase):
     @classmethod
     def sleep(cls, timeout, remark=None):
         if hasattr(cls, 'logger'):
-            cls.logger.debug("Sleeping for %ss (%s)" % (timeout, remark))
+            cls.logger.debug("Starting sleep for %ss (%s)" % (timeout, remark))
+        before = time.time()
         time.sleep(timeout)
+        after = time.time()
+        if after - before > 2 * timeout:
+            cls.logger.error("unexpected time.sleep() result - "
+                             "slept for %ss instead of ~%ss!" % (
+                                 after - before, timeout))
+        if hasattr(cls, 'logger'):
+            cls.logger.debug(
+                "Finished sleep (%s) - slept %ss (wanted %ss)" % (
+                    remark, after - before, timeout))
 
 
 class TestCasePrinter(object):
@@ -897,7 +941,7 @@ class VppTestRunner(unittest.TextTestRunner):
         :param test:
 
         """
-        gc.disable()  # disable garbage collection, we'll do that manually
+        faulthandler.enable()  # emit stack trace to stderr if killed by signal
         print("Running tests using custom test runner")  # debug message
         filter_file, filter_class, filter_func = self.parse_test_option()
         print("Active filters: file=%s, class=%s, function=%s" % (
@@ -906,4 +950,6 @@ class VppTestRunner(unittest.TextTestRunner):
                                      filter_func)
         print("%s out of %s tests match specified filters" % (
             filtered.countTestCases(), test.countTestCases()))
+        if not running_extended_tests():
+            print("Not running extended tests (some tests will be skipped)")
         return super(VppTestRunner, self).run(filtered)