ipsec: IPSec protection for multi-point tunnel interfaces
[vpp.git] / test / framework.py
index e3c605b..872dd45 100644 (file)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 import gc
@@ -28,6 +28,7 @@ from vpp_sub_interface import VppSubInterface
 from vpp_lo_interface import VppLoInterface
 from vpp_bvi_interface import VppBviInterface
 from vpp_papi_provider import VppPapiProvider
+import vpp_papi
 from vpp_papi.vpp_stats import VPPStats
 from vpp_papi.vpp_transport_shmem import VppTransportShmemIOError
 from log import RED, GREEN, YELLOW, double_line_delim, single_line_delim, \
@@ -171,7 +172,7 @@ def pump_output(testclass):
                 testclass.vpp_stdout_deque.extend(split[:limit])
                 if not testclass.cache_vpp_output:
                     for line in split[:limit]:
-                        testclass.logger.debug(
+                        testclass.logger.info(
                             "VPP STDOUT: %s" % line.rstrip("\n"))
         if testclass.vpp.stderr.fileno() in readable:
             read = os.read(testclass.vpp.stderr.fileno(), 102400)
@@ -189,7 +190,7 @@ def pump_output(testclass):
                 testclass.vpp_stderr_deque.extend(split[:limit])
                 if not testclass.cache_vpp_output:
                     for line in split[:limit]:
-                        testclass.logger.debug(
+                        testclass.logger.error(
                             "VPP STDERR: %s" % line.rstrip("\n"))
                         # ignoring the dummy pipe here intentionally - the
                         # flag will take care of properly terminating the loop
@@ -267,6 +268,7 @@ class VppTestCase(unittest.TestCase):
 
     extra_vpp_punt_config = []
     extra_vpp_plugin_config = []
+    vapi_response_timeout = 5
 
     @property
     def packet_infos(self):
@@ -379,6 +381,7 @@ class VppTestCase(unittest.TestCase):
                            "prefix", cls.shm_prefix, "}", "cpu", "{",
                            "main-core", str(cpu_core_number),
                            cls.worker_config, "}",
+                           "physmem", "{", "max-size", "32m", "}",
                            "statseg", "{", "socket-name", cls.stats_sock, "}",
                            "socksvr", "{", "socket-name", cls.api_sock, "}",
                            "plugins",
@@ -443,8 +446,7 @@ class VppTestCase(unittest.TestCase):
         try:
             cls.vpp = subprocess.Popen(cmdline,
                                        stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE,
-                                       bufsize=1)
+                                       stderr=subprocess.PIPE)
         except subprocess.CalledProcessError as e:
             cls.logger.critical("Subprocess returned with non-0 return code: ("
                                 "%s)", e.returncode)
@@ -460,19 +462,6 @@ class VppTestCase(unittest.TestCase):
 
         cls.wait_for_enter()
 
-    @classmethod
-    def wait_for_stats_socket(cls):
-        deadline = time.time() + 300
-        ok = False
-        while time.time() < deadline or \
-                cls.debug_gdb or cls.debug_gdbserver:
-            if os.path.exists(cls.stats_sock):
-                ok = True
-                break
-            cls.sleep(0.8)
-        if not ok:
-            cls.logger.critical("Couldn't stat : {}".format(cls.stats_sock))
-
     @classmethod
     def wait_for_coredump(cls):
         corefile = cls.tempdir + "/core"
@@ -503,8 +492,9 @@ class VppTestCase(unittest.TestCase):
         """
         super(VppTestCase, cls).setUpClass()
         gc.collect()  # run garbage collection first
-        random.seed()
         cls.logger = get_logger(cls.__name__)
+        seed = os.environ["RND_SEED"]
+        random.seed(seed)
         if hasattr(cls, 'parallel_handler'):
             cls.logger.addHandler(cls.parallel_handler)
             cls.logger.propagate = False
@@ -525,6 +515,7 @@ class VppTestCase(unittest.TestCase):
         os.chdir(cls.tempdir)
         cls.logger.info("Temporary dir is %s, shm prefix is %s",
                         cls.tempdir, cls.shm_prefix)
+        cls.logger.debug("Random seed is %s" % seed)
         cls.setUpConstants()
         cls.reset_packet_infos()
         cls._captures = []
@@ -548,17 +539,14 @@ class VppTestCase(unittest.TestCase):
             cls.pump_thread.daemon = True
             cls.pump_thread.start()
             if cls.debug_gdb or cls.debug_gdbserver:
-                read_timeout = 0
-            else:
-                read_timeout = 5
+                cls.vapi_response_timeout = 0
             cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls,
-                                       read_timeout)
+                                       cls.vapi_response_timeout)
             if cls.step:
                 hook = hookmodule.StepHook(cls)
             else:
                 hook = hookmodule.PollHook(cls)
             cls.vapi.register_hook(hook)
-            cls.wait_for_stats_socket()
             cls.statistics = VPPStats(socketname=cls.stats_sock)
             try:
                 hook.poll_vpp()
@@ -570,11 +558,10 @@ class VppTestCase(unittest.TestCase):
                 raise
             try:
                 cls.vapi.connect()
-            except Exception:
-                try:
-                    cls.vapi.disconnect()
-                except Exception:
-                    pass
+            except vpp_papi.VPPIOError as e:
+                cls.logger.debug("Exception connecting to vapi: %s" % e)
+                cls.vapi.disconnect()
+
                 if cls.debug_gdbserver:
                     print(colorize("You're running VPP inside gdbserver but "
                                    "VPP-API connection failed, did you forget "
@@ -586,20 +573,28 @@ class VppTestCase(unittest.TestCase):
             cls.quit()
             raise
 
+    @classmethod
+    def _debug_quit(cls):
+        if (cls.debug_gdbserver or cls.debug_gdb):
+            try:
+                cls.vpp.poll()
+
+                if cls.vpp.returncode is None:
+                    print()
+                    print(double_line_delim)
+                    print("VPP or GDB server is still running")
+                    print(single_line_delim)
+                    input("When done debugging, press ENTER to kill the "
+                          "process and finish running the testcase...")
+            except AttributeError:
+                pass
+
     @classmethod
     def quit(cls):
         """
         Disconnect vpp-api, kill vpp and cleanup shared memory files
         """
-        if (cls.debug_gdbserver or cls.debug_gdb) and hasattr(cls, 'vpp'):
-            cls.vpp.poll()
-            if cls.vpp.returncode is None:
-                print()
-                print(double_line_delim)
-                print("VPP or GDB server is still running")
-                print(single_line_delim)
-                input("When done debugging, press ENTER to kill the "
-                      "process and finish running the testcase...")
+        cls._debug_quit()
 
         # first signal that we want to stop the pump thread, then wake it up
         if hasattr(cls, 'pump_thread_stop_flag'):
@@ -610,11 +605,12 @@ class VppTestCase(unittest.TestCase):
             cls.logger.debug("Waiting for pump thread to stop")
             cls.pump_thread.join()
         if hasattr(cls, 'vpp_stderr_reader_thread'):
-            cls.logger.debug("Waiting for stdderr pump to stop")
+            cls.logger.debug("Waiting for stderr pump to stop")
             cls.vpp_stderr_reader_thread.join()
 
         if hasattr(cls, 'vpp'):
             if hasattr(cls, 'vapi'):
+                cls.logger.debug(cls.vapi.vpp.get_stats())
                 cls.logger.debug("Disconnecting class vapi client on %s",
                                  cls.__name__)
                 cls.vapi.disconnect()
@@ -1006,13 +1002,15 @@ class VppTestCase(unittest.TestCase):
         while True:
             layer = temp.getlayer(counter)
             if layer:
+                layer = layer.copy()
+                layer.remove_payload()
                 for cf in checksum_fields:
                     if hasattr(layer, cf):
                         if ignore_zero_udp_checksums and \
                                 0 == getattr(layer, cf) and \
                                 layer.name in udp_layers:
                             continue
-                        delattr(layer, cf)
+                        delattr(temp.getlayer(counter), cf)
                         checksums.append((counter, cf))
             else:
                 break
@@ -1135,9 +1133,9 @@ class VppTestCase(unittest.TestCase):
                 "Finished sleep (%s) - slept %es (wanted %es)",
                 remark, after - before, timeout)
 
-    def pg_send(self, intf, pkts):
+    def pg_send(self, intf, pkts, worker=None):
         self.vapi.cli("clear trace")
-        intf.add_stream(pkts)
+        intf.add_stream(pkts, worker=worker)
         self.pg_enable_capture(self.pg_interfaces)
         self.pg_start()
 
@@ -1150,10 +1148,10 @@ class VppTestCase(unittest.TestCase):
             i.assert_nothing_captured(remark=remark)
             timeout = 0.1
 
-    def send_and_expect(self, intf, pkts, output, n_rx=None):
+    def send_and_expect(self, intf, pkts, output, n_rx=None, worker=None):
         if not n_rx:
             n_rx = len(pkts)
-        self.pg_send(intf, pkts)
+        self.pg_send(intf, pkts, worker=worker)
         rx = output.get_capture(n_rx)
         return rx
 
@@ -1171,11 +1169,6 @@ class VppTestCase(unittest.TestCase):
 
         return rx
 
-    def runTest(self):
-        """ unittest calls runTest when TestCase is instantiated without a
-        test case.  Use case: Writing unittests against VppTestCase"""
-        pass
-
 
 def get_testcase_doc_name(test):
     return getdoc(test.__class__).splitlines()[0]
@@ -1393,7 +1386,7 @@ class VppTestResult(unittest.TestResult):
             test.__class__._header_printed = True
 
         print_header(test)
-
+        self.start_test = time.time()
         unittest.TestResult.startTest(self, test)
         if self.verbosity > 0:
             self.stream.writeln(
@@ -1408,14 +1401,17 @@ class VppTestResult(unittest.TestResult):
 
         """
         unittest.TestResult.stopTest(self, test)
+
         if self.verbosity > 0:
             self.stream.writeln(single_line_delim)
             self.stream.writeln("%-73s%s" % (self.getDescription(test),
                                              self.result_string))
             self.stream.writeln(single_line_delim)
         else:
-            self.stream.writeln("%-73s%s" % (self.getDescription(test),
-                                             self.result_string))
+            self.stream.writeln("%-68s %4.2f %s" %
+                                (self.getDescription(test),
+                                 time.time() - self.start_test,
+                                 self.result_string))
 
         self.send_result_through_pipe(test, TEST_RUN)
 
@@ -1499,16 +1495,17 @@ class VppTestRunner(unittest.TextTestRunner):
 
 
 class Worker(Thread):
-    def __init__(self, args, logger, env=None):
+    def __init__(self, executable_args, logger, env=None, *args, **kwargs):
+        super(Worker, self).__init__(*args, **kwargs)
         self.logger = logger
-        self.args = args
+        self.args = executable_args
         if hasattr(self, 'testcase') and self.testcase.debug_all:
             if self.testcase.debug_gdbserver:
                 self.args = ['/usr/bin/gdbserver', 'localhost:{port}'
                              .format(port=self.testcase.gdbserver_port)] + args
             elif self.testcase.debug_gdb and hasattr(self, 'wait_for_gdb'):
                 self.args.append(self.wait_for_gdb)
-        self.app_bin = args[0]
+        self.app_bin = executable_args[0]
         self.app_name = os.path.basename(self.app_bin)
         if hasattr(self, 'role'):
             self.app_name += ' {role}'.format(role=self.role)
@@ -1516,7 +1513,6 @@ class Worker(Thread):
         self.result = None
         env = {} if env is None else env
         self.env = copy.deepcopy(env)
-        super(Worker, self).__init__()
 
     def wait_for_enter(self):
         if not hasattr(self, 'testcase'):