tests: "force solo" testcase support

Some of the tests are time-sensitive, and at present require a non-trivial
modification in order to run at high concurrency.

Without these modifications, they intermittently fail, and require
the test retries.

Rather than setting them to the extended tests and forgetting
about them, put them into a "solo" set, which gets run in a
single-threaded mode after the rest of the tests are done.

Mark a few of the tests that showed errors during TEST_JOBS=48
as forced-solo.

Also, give a better diagnostic if the testcase misses a docstring
needed to represent it in the diagnostic outputs.

Type: fix

Change-Id: I33fe62eb17edc1885bd2c3523892051d52da6546
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>

diff --git a/src/plugins/dhcp/test/test_dhcp.py b/src/plugins/dhcp/test/test_dhcp.py
index 065683f..266932c 100644 (file)
--- a/src/plugins/dhcp/test/test_dhcp.py
+++ b/src/plugins/dhcp/test/test_dhcp.py
@@ -35,6 +35,10 @@ DHCP6_SERVER_PORT = 546
 class TestDHCP(VppTestCase):
     """ DHCP Test Case """
 
+    @classmethod
+    def force_solo(cls):
+        return True
+
     @classmethod
     def setUpClass(cls):
         super(TestDHCP, cls).setUpClass()

diff --git a/src/plugins/dhcp/test/test_dhcp6.py b/src/plugins/dhcp/test/test_dhcp6.py
index 7254496..ebd4372 100644 (file)
--- a/src/plugins/dhcp/test/test_dhcp6.py
+++ b/src/plugins/dhcp/test/test_dhcp6.py
@@ -223,6 +223,10 @@ class TestDHCPv6DataPlane(VppTestCase):
 class TestDHCPv6IANAControlPlane(VppTestCase):
     """ DHCPv6 IA NA Control Plane Test Case """
 
+    @classmethod
+    def force_solo(cls):
+        return True
+
     @classmethod
     def setUpClass(cls):
         super(TestDHCPv6IANAControlPlane, cls).setUpClass()

diff --git a/src/plugins/flowprobe/test/test_flowprobe.py b/src/plugins/flowprobe/test/test_flowprobe.py
index 092e8d3..4cf019a 100644 (file)
--- a/src/plugins/flowprobe/test/test_flowprobe.py
+++ b/src/plugins/flowprobe/test/test_flowprobe.py
@@ -346,6 +346,10 @@ class MethodHolder(VppTestCase):
 class Flowprobe(MethodHolder):
     """Template verification, timer tests"""
 
+    @classmethod
+    def force_solo(cls):
+        return True
+
     @classmethod
     def setUpClass(cls):
         super(Flowprobe, cls).setUpClass()

diff --git a/src/plugins/memif/test/test_memif.py b/src/plugins/memif/test/test_memif.py
index f11dd89..244818c 100644 (file)
--- a/src/plugins/memif/test/test_memif.py
+++ b/src/plugins/memif/test/test_memif.py
@@ -16,6 +16,10 @@ from vpp_papi import VppEnum
 class TestMemif(VppTestCase):
     """ Memif Test Case """
 
+    @classmethod
+    def force_solo(cls):
+        return True
+
     @classmethod
     def setUpClass(cls):
         # fork new process before client connects to VPP

diff --git a/test/framework.py b/test/framework.py
index c9d72a7..ba5f401 100644 (file)
--- a/test/framework.py
+++ b/test/framework.py
@@ -298,6 +298,11 @@ class VppTestCase(unittest.TestCase):
         else:
             return 0
 
+    @classmethod
+    def force_solo(cls):
+        """ if the test case class is timing-sensitive - return true """
+        return False
+
     @classmethod
     def instance(cls):
         """Return the instance of this testcase"""
@@ -1412,9 +1417,19 @@ class VppTestResult(unittest.TestResult):
         """
 
         def print_header(test):
+            test_doc = getdoc(test)
+            if not test_doc:
+                raise Exception("No doc string for test '%s'" % test.id())
+            test_title = test_doc.splitlines()[0]
+            test_title_colored = colorize(test_title, GREEN)
+            if test.force_solo():
+                # long live PEP-8 and 80 char width limitation...
+                c = YELLOW
+                test_title_colored = colorize("SOLO RUN: " + test_title, c)
+
             if not hasattr(test.__class__, '_header_printed'):
                 print(double_line_delim)
-                print(colorize(getdoc(test).splitlines()[0], GREEN))
+                print(test_title_colored)
                 print(double_line_delim)
             test.__class__._header_printed = True
 

diff --git a/test/run_tests.py b/test/run_tests.py
index 499d6df..66118ca 100644 (file)
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -324,6 +324,8 @@ def process_finished_testsuite(wrapped_testcase_suite,
 
 def run_forked(testcase_suites):
     wrapped_testcase_suites = set()
+    solo_testcase_suites = []
+    total_test_runners = 0
 
     # suites are unhashable, need to use list
     results = []
@@ -331,12 +333,29 @@ def run_forked(testcase_suites):
     finished_unread_testcases = set()
     manager = StreamQueueManager()
     manager.start()
-    for i in range(concurrent_tests):
+    total_test_runners = 0
+    while total_test_runners < concurrent_tests:
         if testcase_suites:
-            wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0),
+            a_suite = testcase_suites.pop(0)
+            if a_suite.force_solo:
+                solo_testcase_suites.append(a_suite)
+                continue
+            wrapped_testcase_suite = TestCaseWrapper(a_suite,
                                                      manager)
             wrapped_testcase_suites.add(wrapped_testcase_suite)
             unread_testcases.add(wrapped_testcase_suite)
+            total_test_runners = total_test_runners + 1
+        else:
+            break
+
+    while total_test_runners < 1 and solo_testcase_suites:
+        if solo_testcase_suites:
+            a_suite = solo_testcase_suites.pop(0)
+            wrapped_testcase_suite = TestCaseWrapper(a_suite,
+                                                     manager)
+            wrapped_testcase_suites.add(wrapped_testcase_suite)
+            unread_testcases.add(wrapped_testcase_suite)
+            total_test_runners = total_test_runners + 1
         else:
             break
 
@@ -448,14 +467,32 @@ def run_forked(testcase_suites):
                 wrapped_testcase_suites.remove(finished_testcase)
                 finished_unread_testcases.add(finished_testcase)
                 finished_testcase.stdouterr_queue.put(None)
+                total_test_runners = total_test_runners - 1
                 if stop_run:
                     while testcase_suites:
                         results.append(TestResult(testcase_suites.pop(0)))
                 elif testcase_suites:
-                    new_testcase = TestCaseWrapper(testcase_suites.pop(0),
-                                                   manager)
-                    wrapped_testcase_suites.add(new_testcase)
-                    unread_testcases.add(new_testcase)
+                    a_testcase = testcase_suites.pop(0)
+                    while a_testcase and a_testcase.force_solo:
+                        solo_testcase_suites.append(a_testcase)
+                        if testcase_suites:
+                            a_testcase = testcase_suites.pop(0)
+                        else:
+                            a_testcase = None
+                    if a_testcase:
+                        new_testcase = TestCaseWrapper(a_testcase,
+                                                       manager)
+                        wrapped_testcase_suites.add(new_testcase)
+                        total_test_runners = total_test_runners + 1
+                        unread_testcases.add(new_testcase)
+                else:
+                    if solo_testcase_suites and total_test_runners == 0:
+                        a_testcase = solo_testcase_suites.pop(0)
+                        new_testcase = TestCaseWrapper(a_testcase,
+                                                       manager)
+                        wrapped_testcase_suites.add(new_testcase)
+                        total_test_runners = total_test_runners + 1
+                        unread_testcases.add(new_testcase)
             time.sleep(0.1)
     except Exception:
         for wrapped_testcase_suite in wrapped_testcase_suites:
@@ -484,7 +521,10 @@ class SplitToSuitesCallback:
             self.suite_name = file_name + cls.__name__
             if self.suite_name not in self.suites:
                 self.suites[self.suite_name] = unittest.TestSuite()
+                self.suites[self.suite_name].force_solo = False
             self.suites[self.suite_name].addTest(test_method)
+            if test_method.force_solo():
+                self.suites[self.suite_name].force_solo = True
 
         else:
             self.filtered.addTest(test_method)

diff --git a/test/test_bfd.py b/test/test_bfd.py
index 67b6276..f66f75a 100644 (file)
--- a/test/test_bfd.py
+++ b/test/test_bfd.py
@@ -685,6 +685,10 @@ class BFD4TestCase(VppTestCase):
     vpp_session = None
     test_session = None
 
+    @classmethod
+    def force_solo(cls):
+        return True
+
     @classmethod
     def setUpClass(cls):
         super(BFD4TestCase, cls).setUpClass()
@@ -1493,6 +1497,10 @@ class BFD6TestCase(VppTestCase):
     vpp_session = None
     test_session = None
 
+    @classmethod
+    def force_solo(cls):
+        return True
+
     @classmethod
     def setUpClass(cls):
         super(BFD6TestCase, cls).setUpClass()
@@ -1704,6 +1712,10 @@ class BFDFIBTestCase(VppTestCase):
     vpp_session = None
     test_session = None
 
+    @classmethod
+    def force_solo(cls):
+        return True
+
     @classmethod
     def setUpClass(cls):
         super(BFDFIBTestCase, cls).setUpClass()
@@ -1892,6 +1904,10 @@ class BFDSHA1TestCase(VppTestCase):
     vpp_session = None
     test_session = None
 
+    @classmethod
+    def force_solo(cls):
+        return True
+
     @classmethod
     def setUpClass(cls):
         super(BFDSHA1TestCase, cls).setUpClass()
@@ -2122,6 +2138,10 @@ class BFDAuthOnOffTestCase(VppTestCase):
     vpp_session = None
     test_session = None
 
+    @classmethod
+    def force_solo(cls):
+        return True
+
     @classmethod
     def setUpClass(cls):
         super(BFDAuthOnOffTestCase, cls).setUpClass()
@@ -2331,6 +2351,10 @@ class BFDCLITestCase(VppTestCase):
     """Bidirectional Forwarding Detection (BFD) (CLI) """
     pg0 = None
 
+    @classmethod
+    def force_solo(cls):
+        return True
+
     @classmethod
     def setUpClass(cls):
         super(BFDCLITestCase, cls).setUpClass()

diff --git a/test/test_ip6.py b/test/test_ip6.py
index f8a2dc2..36ef577 100644 (file)
--- a/test/test_ip6.py
+++ b/test/test_ip6.py
@@ -164,6 +164,10 @@ class TestIPv6ND(VppTestCase):
 class TestIPv6(TestIPv6ND):
     """ IPv6 Test Case """
 
+    @classmethod
+    def force_solo(cls):
+        return True
+
     @classmethod
     def setUpClass(cls):
         super(TestIPv6, cls).setUpClass()

diff --git a/test/test_session.py b/test/test_session.py
index 94218d0..309bf6e 100644 (file)
--- a/test/test_session.py
+++ b/test/test_session.py
@@ -120,6 +120,10 @@ class TestSessionUnitTests(VppTestCase):
 class TestSvmFifoUnitTests(VppTestCase):
     """ SVM Fifo Unit Tests Case """
 
+    @classmethod
+    def force_solo(cls):
+        return True
+
     @classmethod
     def setUpClass(cls):
         super(TestSvmFifoUnitTests, cls).setUpClass()

diff --git a/test/test_util.py b/test/test_util.py
index 51cc739..8501881 100755 (executable)
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -7,8 +7,15 @@ from vpp_papi import mac_pton, mac_ntop
 
 
 class TestUtil (unittest.TestCase):
-    """ MAC to binary and back """
+    """ Test framework utility tests """
+
+    @classmethod
+    def force_solo(cls):
+        """ if the test case class is timing-sensitive - return true """
+        return False
+
     def test_mac_to_binary(self):
+        """ MAC to binary and back """
         mac = 'aa:bb:cc:dd:ee:ff'
         b = mac_pton(mac)
         mac2 = mac_ntop(b)