tests: allow to define SKIP_TESTS in a similar fashion to TEST

This change adds the possibility to specify the tests to skip
from the selected set of tests. This allows to construct test
sets "everything except foo and bar" that are tedious to make
otherwise.

Type: improvement
Change-Id: I0862031baf22fef926554873a88a068dfc8f0623
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>

diff --git a/test/Makefile b/test/Makefile
index 0a1b770..e26bf7f 100644 (file)
--- a/test/Makefile
+++ b/test/Makefile
@@ -276,7 +276,7 @@ endif
 
 EXTRA_ARGS=$(ARG0) $(ARG1) $(ARG2) $(ARG3) $(ARG4) $(ARG5) $(ARG6) $(ARG7) $(ARG8) $(ARG9) $(ARG10) $(ARG11) $(ARG12) $(ARG13) $(ARG14) $(ARG15) $(ARG16) $(ARG17) $(ARG18) $(ARG19)
 
-RUN_TESTS_ARGS=--failed-dir=$(FAILED_DIR) --verbose=$(V) --jobs=$(TEST_JOBS) --filter=$(TEST) --retries=$(RETRIES) --venv-dir=$(VENV_PATH) --vpp-ws-dir=$(WS_ROOT) --vpp-tag=$(TAG) --rnd-seed=$(RND_SEED) --vpp-worker-count="$(VPP_WORKER_COUNT)" --keep-pcaps $(PLUGIN_PATH_ARGS) $(EXC_PLUGINS_ARG) $(TEST_PLUGIN_PATH_ARGS) $(EXTRA_ARGS)
+RUN_TESTS_ARGS=--failed-dir=$(FAILED_DIR) --verbose=$(V) --jobs=$(TEST_JOBS) --filter=$(TEST) --skip-filter=$(SKIP_TESTS) --retries=$(RETRIES) --venv-dir=$(VENV_PATH) --vpp-ws-dir=$(WS_ROOT) --vpp-tag=$(TAG) --rnd-seed=$(RND_SEED) --vpp-worker-count="$(VPP_WORKER_COUNT)" --keep-pcaps $(PLUGIN_PATH_ARGS) $(EXC_PLUGINS_ARG) $(TEST_PLUGIN_PATH_ARGS) $(EXTRA_ARGS)
 RUN_SCRIPT_ARGS=--python-opts=$(PYTHON_OPTS)
 
 define retest-func
@@ -606,6 +606,12 @@ help:
        @echo "                    'test_icmp_error' in all files"
        @echo "       (default: '')"
        @echo ""
+       @echo "   SKIP_TESTS=<filter>,[<filter>],..."
+       @echo "       Skip tests matching one or more comma-delimited"
+       @echo "       filter expressions, even if they were selected by TEST"
+       @echo ""
+       @echo "       (default: '')"
+       @echo ""
        @echo "   VARIANT=<variant>"
        @echo "       specify which march node variant to unit test"
        @echo "           e.g. VARIANT=skx test the skx march variants"

diff --git a/test/config.py b/test/config.py
index e939f18..2870c55 100644 (file)
--- a/test/config.py
+++ b/test/config.py
@@ -151,6 +151,21 @@ parser.add_argument(
     "--filter", action="store", metavar="FILTER_EXPRESSION", help=filter_help_string
 )
 
+skip_filter_help_string = """\
+expression consists of one or more filters separated by commas (',')
+filter consists of 3 string selectors separated by dots ('.')
+
+The syntax is identical to the expression used to select the tests,
+except this one one has the effect to skip the tests that match it.
+"""
+
+parser.add_argument(
+    "--skip-filter",
+    action="store",
+    metavar="SKIP_FILTER_EXPR",
+    help=skip_filter_help_string,
+)
+
 default_retries = 0
 
 parser.add_argument(

diff --git a/test/run_tests.py b/test/run_tests.py
index 66e0ee4..166f5d3 100644 (file)
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -685,8 +685,9 @@ def filter_tests(tests, filter_cb):
 
 
 class FilterByTestOption:
-    def __init__(self, filters):
+    def __init__(self, filters, skip_filters):
         self.filters = filters
+        self.skip_filters = skip_filters
 
     def __call__(self, file_name, class_name, func_name):
         def test_one(
@@ -716,6 +717,21 @@ class FilterByTestOption:
                 class_name,
                 func_name,
             ):
+                for (
+                    x_filter_file_name,
+                    x_filter_class_name,
+                    x_filter_func_name,
+                ) in self.skip_filters:
+                    if test_one(
+                        x_filter_file_name,
+                        x_filter_class_name,
+                        x_filter_func_name,
+                        file_name,
+                        class_name,
+                        func_name,
+                    ):
+                        # If the test matches the excluded tests, do not run it
+                        return False
                 return True
 
         return False
@@ -976,6 +992,11 @@ if __name__ == "__main__":
 
     print("Running tests using custom test runner.")
     filters = [(parse_test_filter(f)) for f in config.filter.split(",")]
+    skip_filters = [(parse_test_filter(f)) for f in config.skip_filter.split(",")]
+    # Remove the (None, None, None) triplet that gets pushed by default
+    skip_filters = [
+        triplet for triplet in skip_filters if triplet != (None, None, None)
+    ]
 
     print(
         "Selected filters: ",
@@ -984,8 +1005,15 @@ if __name__ == "__main__":
             for filter_file, filter_class, filter_func in filters
         ),
     )
+    print(
+        "Selected skip filters: ",
+        "|".join(
+            f"file={filter_file}, class={filter_class}, function={filter_func}"
+            for filter_file, filter_class, filter_func in skip_filters
+        ),
+    )
 
-    filter_cb = FilterByTestOption(filters)
+    filter_cb = FilterByTestOption(filters, skip_filters)
 
     cb = SplitToSuitesCallback(filter_cb)
     for d in config.test_src_dir: