X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=test%2Fframework.py;h=c9ecafd2bcabc1719f66eac9fd636eea2955a98f;hb=c8eae8d28aa6c9fb946a8be5b1731e0e79d7cdff;hp=19834026ef9b67a3acbf3ab2ee83a73f3d33f56c;hpb=4830e4f78fb8e46b23a1a0711cd06969a77d8d95;p=vpp.git diff --git a/test/framework.py b/test/framework.py index 19834026ef9..c9ecafd2bca 100644 --- a/test/framework.py +++ b/test/framework.py @@ -2,10 +2,12 @@ from __future__ import print_function import gc +import logging import sys import os import select import signal +import subprocess import unittest import tempfile import time @@ -19,6 +21,7 @@ from threading import Thread, Event from inspect import getdoc, isclass from traceback import format_exception from logging import FileHandler, DEBUG, Formatter +from enum import Enum import scapy.compat from scapy.packet import Raw @@ -39,18 +42,11 @@ from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror from scapy.layers.inet6 import ICMPv6DestUnreach, ICMPv6EchoRequest from scapy.layers.inet6 import ICMPv6EchoReply -if os.name == 'posix' and sys.version_info[0] < 3: - # using subprocess32 is recommended by python official documentation - # @ https://docs.python.org/2/library/subprocess.html - import subprocess32 as subprocess -else: - import subprocess - -# Python2/3 compatible -try: - input = raw_input -except NameError: - pass +logger = logging.getLogger(__name__) + +# Set up an empty logger for the testcase that can be overridden as necessary +null_logger = logging.getLogger('VppTestCase') +null_logger.addHandler(logging.NullHandler()) PASS = 0 FAIL = 1 @@ -224,14 +220,6 @@ def _running_gcov_tests(): running_gcov_tests = _running_gcov_tests() -def _running_on_centos(): - os_id = os.getenv("OS_ID", "") - return True if "centos" in os_id.lower() else False - - -running_on_centos = _running_on_centos() - - class KeepAliveReporter(object): """ Singleton object which reports test start to parent process @@ -268,6 +256,28 @@ class KeepAliveReporter(object): self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid)) +class TestCaseTag(Enum): + # marks the suites that must run at the end + # using only a single test runner + RUN_SOLO = 1 + # marks the suites broken on VPP multi-worker + FIXME_VPP_WORKERS = 2 + + +def create_tag_decorator(e): + def decorator(cls): + try: + cls.test_tags.append(e) + except AttributeError: + cls.test_tags = [e] + return cls + return decorator + + +tag_run_solo = create_tag_decorator(TestCaseTag.RUN_SOLO) +tag_fixme_vpp_workers = create_tag_decorator(TestCaseTag.FIXME_VPP_WORKERS) + + class VppTestCase(unittest.TestCase): """This subclass is a base class for VPP test cases that are implemented as classes. It provides methods to create and run test case. @@ -275,6 +285,7 @@ class VppTestCase(unittest.TestCase): extra_vpp_punt_config = [] extra_vpp_plugin_config = [] + logger = null_logger vapi_response_timeout = 5 @property @@ -290,6 +301,20 @@ class VppTestCase(unittest.TestCase): else: return 0 + @classmethod + def has_tag(cls, tag): + """ if the test case has a given tag - return true """ + try: + return tag in cls.test_tags + except AttributeError: + pass + return False + + @classmethod + def is_tagged_run_solo(cls): + """ if the test case class is timing-sensitive - return true """ + return cls.has_tag(TestCaseTag.RUN_SOLO) + @classmethod def instance(cls): """Return the instance of this testcase""" @@ -379,7 +404,10 @@ class VppTestCase(unittest.TestCase): cpu_core_number = cls.get_least_used_cpu() if not hasattr(cls, "worker_config"): - cls.worker_config = "" + cls.worker_config = os.getenv("VPP_WORKER_CONFIG", "") + if cls.worker_config != "": + if cls.has_tag(TestCaseTag.FIXME_VPP_WORKERS): + cls.worker_config = "" default_variant = os.getenv("VARIANT") if default_variant is not None: @@ -387,6 +415,10 @@ class VppTestCase(unittest.TestCase): else: default_variant = "" + api_fuzzing = os.getenv("API_FUZZ") + if api_fuzzing is None: + api_fuzzing = 'off' + cls.vpp_cmdline = [cls.vpp_bin, "unix", "{", "nodaemon", debug_cli, "full-coredump", coredump_size, "runtime-dir", cls.tempdir, "}", @@ -398,9 +430,12 @@ class VppTestCase(unittest.TestCase): "statseg", "{", "socket-name", cls.stats_sock, "}", "socksvr", "{", "socket-name", cls.api_sock, "}", "node { ", default_variant, "}", + "api-fuzz {", api_fuzzing, "}", "plugins", "{", "plugin", "dpdk_plugin.so", "{", "disable", "}", "plugin", "rdma_plugin.so", "{", "disable", + "}", "plugin", "lisp_unittest_plugin.so", "{", + "enable", "}", "plugin", "unittest_plugin.so", "{", "enable", "}"] + cls.extra_vpp_plugin_config + ["}", ] @@ -582,9 +617,12 @@ class VppTestCase(unittest.TestCase): "VPP-API connection failed, did you forget " "to 'continue' VPP from within gdb?", RED)) raise + except vpp_papi.VPPRuntimeError as e: + cls.logger.debug("%s" % e) + cls.quit() + raise except Exception as e: cls.logger.debug("Exception connecting to VPP: %s" % e) - cls.quit() raise @@ -1139,17 +1177,16 @@ class VppTestCase(unittest.TestCase): time.sleep(0) return - if hasattr(cls, 'logger'): - cls.logger.debug("Starting sleep for %es (%s)", timeout, remark) + cls.logger.debug("Starting sleep for %es (%s)", timeout, remark) before = time.time() time.sleep(timeout) after = time.time() - if hasattr(cls, 'logger') and after - before > 2 * timeout: + if after - before > 2 * timeout: cls.logger.error("unexpected self.sleep() result - " "slept for %es instead of ~%es!", after - before, timeout) - if hasattr(cls, 'logger'): - cls.logger.debug( + + cls.logger.debug( "Finished sleep (%s) - slept %es (wanted %es)", remark, after - before, timeout) @@ -1289,22 +1326,20 @@ class VppTestResult(unittest.TestResult): failed_dir, '%s-FAILED' % os.path.basename(self.current_test_case_info.tempdir)) - if self.current_test_case_info.logger: - self.current_test_case_info.logger.debug( + + self.current_test_case_info.logger.debug( "creating a link to the failed test") - self.current_test_case_info.logger.debug( + self.current_test_case_info.logger.debug( "os.symlink(%s, %s)" % (self.current_test_case_info.tempdir, link_path)) if os.path.exists(link_path): - if self.current_test_case_info.logger: - self.current_test_case_info.logger.debug( + self.current_test_case_info.logger.debug( 'symlink already exists') else: os.symlink(self.current_test_case_info.tempdir, link_path) except Exception as e: - if self.current_test_case_info.logger: - self.current_test_case_info.logger.error(e) + self.current_test_case_info.logger.error(e) def send_result_through_pipe(self, test, result): if hasattr(self, 'test_framework_result_pipe'): @@ -1399,9 +1434,26 @@ class VppTestResult(unittest.TestResult): """ def print_header(test): + test_doc = getdoc(test) + if not test_doc: + raise Exception("No doc string for test '%s'" % test.id()) + test_title = test_doc.splitlines()[0] + test_title_colored = colorize(test_title, GREEN) + if test.is_tagged_run_solo(): + # long live PEP-8 and 80 char width limitation... + c = YELLOW + test_title_colored = colorize("SOLO RUN: " + test_title, c) + + # This block may overwrite the colorized title above, + # but we want this to stand out and be fixed + if test.has_tag(TestCaseTag.FIXME_VPP_WORKERS): + c = RED + w = "FIXME with VPP workers: " + test_title_colored = colorize(w + test_title, c) + if not hasattr(test.__class__, '_header_printed'): print(double_line_delim) - print(colorize(getdoc(test).splitlines()[0], GREEN)) + print(test_title_colored) print(double_line_delim) test.__class__._header_printed = True