tests: mark the test suites broken when ASan is enabled

Mark broken test suites using @tag_fixme_asan.The main issue
is that some tests do not pass with ASan. These will need to
be fixed, but it takes time and in the meantime, new issues are
introduced that trip up ASan. When this tag is set and Asan
is enabled, failing ASan tests are skipped, so incremental
fixes can pass the CI

Type: improvement

Change-Id: I02602eb74234c25a4c701279e14704b81d4c5b71
Signed-off-by: Naveen Joy <najoy@cisco.com>

diff --git a/test/framework.py b/test/framework.py
index 8c0df28..2c74a03 100644 (file)
--- a/test/framework.py
+++ b/test/framework.py
@@ -286,6 +286,8 @@ class TestCaseTag(Enum):
     RUN_SOLO = 1
     # marks the suites broken on VPP multi-worker
     FIXME_VPP_WORKERS = 2
+    # marks the suites broken when ASan is enabled
+    FIXME_ASAN = 3
 
 
 def create_tag_decorator(e):
@@ -300,6 +302,7 @@ def create_tag_decorator(e):
 
 tag_run_solo = create_tag_decorator(TestCaseTag.RUN_SOLO)
 tag_fixme_vpp_workers = create_tag_decorator(TestCaseTag.FIXME_VPP_WORKERS)
+tag_fixme_asan = create_tag_decorator(TestCaseTag.FIXME_ASAN)
 
 
 class DummyVpp:
@@ -365,6 +368,14 @@ class VppTestCase(CPUInterface, unittest.TestCase):
         """ if the test case class is timing-sensitive - return true """
         return cls.has_tag(TestCaseTag.RUN_SOLO)
 
+    @classmethod
+    def skip_fixme_asan(cls):
+        """ if @tag_fixme_asan & ASan is enabled - mark for skip """
+        if cls.has_tag(TestCaseTag.FIXME_ASAN):
+            vpp_extra_cmake_args = os.environ.get('VPP_EXTRA_CMAKE_ARGS', '')
+            if 'DVPP_ENABLE_SANITIZE_ADDR=ON' in vpp_extra_cmake_args:
+                cls = unittest.skip("Skipping @tag_fixme_asan tests")(cls)
+
     @classmethod
     def instance(cls):
         """Return the instance of this testcase"""
@@ -1559,6 +1570,11 @@ class VppTestResult(unittest.TestResult):
                 test_title = colorize(
                     f"FIXME with VPP workers: {test_title}", RED)
 
+            if test.has_tag(TestCaseTag.FIXME_ASAN):
+                test_title = colorize(
+                    f"FIXME with ASAN: {test_title}", RED)
+                test.skip_fixme_asan()
+
             if hasattr(test, 'vpp_worker_count'):
                 if test.vpp_worker_count == 0:
                     test_title += " [main thread only]"