X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;ds=sidebyside;f=test%2Fframework.py;h=f3c74ceec3ef09ad84383260f2361f5d6d01d8f6;hb=7554aef84a8a406c9dae73a7445e8b97920a83b3;hp=ba5f40147984ac5cceb3faf17e3f32ca115b7983;hpb=a3b7c554c669afc627f9a1e32666211bb6fb6b25;p=vpp.git diff --git a/test/framework.py b/test/framework.py index ba5f4014798..f3c74ceec3e 100644 --- a/test/framework.py +++ b/test/framework.py @@ -7,6 +7,7 @@ import sys import os import select import signal +import subprocess import unittest import tempfile import time @@ -20,6 +21,7 @@ from threading import Thread, Event from inspect import getdoc, isclass from traceback import format_exception from logging import FileHandler, DEBUG, Formatter +from enum import Enum import scapy.compat from scapy.packet import Raw @@ -31,7 +33,7 @@ from vpp_bvi_interface import VppBviInterface from vpp_papi_provider import VppPapiProvider import vpp_papi from vpp_papi.vpp_stats import VPPStats -from vpp_papi.vpp_transport_shmem import VppTransportShmemIOError +from vpp_papi.vpp_transport_socket import VppTransportSocketIOError from log import RED, GREEN, YELLOW, double_line_delim, single_line_delim, \ get_logger, colorize from vpp_object import VppObjectRegistry @@ -40,19 +42,6 @@ from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror from scapy.layers.inet6 import ICMPv6DestUnreach, ICMPv6EchoRequest from scapy.layers.inet6 import ICMPv6EchoReply -if os.name == 'posix' and sys.version_info[0] < 3: - # using subprocess32 is recommended by python official documentation - # @ https://docs.python.org/2/library/subprocess.html - import subprocess32 as subprocess -else: - import subprocess - -# Python2/3 compatible -try: - input = raw_input -except NameError: - pass - logger = logging.getLogger(__name__) # Set up an empty logger for the testcase that can be overridden as necessary @@ -117,13 +106,16 @@ class VppDiedError(Exception): if testcase is None and method_name is None: in_msg = '' else: - in_msg = 'running %s.%s ' % (testcase, method_name) + in_msg = ' while running %s.%s' % (testcase, method_name) + + if self.rv: + msg = "VPP subprocess died unexpectedly%s with return code: %d%s."\ + % (in_msg, self.rv, ' [%s]' % + (self.signal_name if + self.signal_name is not None else '')) + else: + msg = "VPP subprocess died unexpectedly%s." % in_msg - msg = "VPP subprocess died %sunexpectedly with return code: %d%s." % ( - in_msg, - self.rv, - ' [%s]' % (self.signal_name if - self.signal_name is not None else '')) super(VppDiedError, self).__init__(msg) @@ -231,14 +223,6 @@ def _running_gcov_tests(): running_gcov_tests = _running_gcov_tests() -def _running_on_centos(): - os_id = os.getenv("OS_ID", "") - return True if "centos" in os_id.lower() else False - - -running_on_centos = _running_on_centos() - - class KeepAliveReporter(object): """ Singleton object which reports test start to parent process @@ -275,6 +259,28 @@ class KeepAliveReporter(object): self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid)) +class TestCaseTag(Enum): + # marks the suites that must run at the end + # using only a single test runner + RUN_SOLO = 1 + # marks the suites broken on VPP multi-worker + FIXME_VPP_WORKERS = 2 + + +def create_tag_decorator(e): + def decorator(cls): + try: + cls.test_tags.append(e) + except AttributeError: + cls.test_tags = [e] + return cls + return decorator + + +tag_run_solo = create_tag_decorator(TestCaseTag.RUN_SOLO) +tag_fixme_vpp_workers = create_tag_decorator(TestCaseTag.FIXME_VPP_WORKERS) + + class VppTestCase(unittest.TestCase): """This subclass is a base class for VPP test cases that are implemented as classes. It provides methods to create and run test case. @@ -299,10 +305,19 @@ class VppTestCase(unittest.TestCase): return 0 @classmethod - def force_solo(cls): - """ if the test case class is timing-sensitive - return true """ + def has_tag(cls, tag): + """ if the test case has a given tag - return true """ + try: + return tag in cls.test_tags + except AttributeError: + pass return False + @classmethod + def is_tagged_run_solo(cls): + """ if the test case class is timing-sensitive - return true """ + return cls.has_tag(TestCaseTag.RUN_SOLO) + @classmethod def instance(cls): """Return the instance of this testcase""" @@ -391,8 +406,18 @@ class VppTestCase(unittest.TestCase): coredump_size = "coredump-size unlimited" cpu_core_number = cls.get_least_used_cpu() - if not hasattr(cls, "worker_config"): - cls.worker_config = "" + if not hasattr(cls, "vpp_worker_count"): + cls.vpp_worker_count = 0 + worker_config = os.getenv("VPP_WORKER_CONFIG", "") + if worker_config: + elems = worker_config.split(" ") + if elems[0] != "workers" or len(elems) != 2: + raise ValueError("Wrong VPP_WORKER_CONFIG == '%s' value." % + worker_config) + cls.vpp_worker_count = int(elems[1]) + if cls.vpp_worker_count > 0 and\ + cls.has_tag(TestCaseTag.FIXME_VPP_WORKERS): + cls.vpp_worker_count = 0 default_variant = os.getenv("VARIANT") if default_variant is not None: @@ -404,23 +429,27 @@ class VppTestCase(unittest.TestCase): if api_fuzzing is None: api_fuzzing = 'off' - cls.vpp_cmdline = [cls.vpp_bin, "unix", - "{", "nodaemon", debug_cli, "full-coredump", - coredump_size, "runtime-dir", cls.tempdir, "}", - "api-trace", "{", "on", "}", "api-segment", "{", - "prefix", cls.shm_prefix, "}", "cpu", "{", - "main-core", str(cpu_core_number), - cls.worker_config, "}", - "physmem", "{", "max-size", "32m", "}", - "statseg", "{", "socket-name", cls.stats_sock, "}", - "socksvr", "{", "socket-name", cls.api_sock, "}", - "node { ", default_variant, "}", - "api-fuzz {", api_fuzzing, "}", - "plugins", - "{", "plugin", "dpdk_plugin.so", "{", "disable", - "}", "plugin", "rdma_plugin.so", "{", "disable", - "}", "plugin", "unittest_plugin.so", "{", "enable", - "}"] + cls.extra_vpp_plugin_config + ["}", ] + cls.vpp_cmdline = [ + cls.vpp_bin, + "unix", "{", "nodaemon", debug_cli, "full-coredump", + coredump_size, "runtime-dir", cls.tempdir, "}", + "api-trace", "{", "on", "}", + "api-segment", "{", "prefix", cls.shm_prefix, "}", + "cpu", "{", "main-core", str(cpu_core_number), ] + if cls.vpp_worker_count: + cls.vpp_cmdline.extend(["workers", str(cls.vpp_worker_count)]) + cls.vpp_cmdline.extend([ + "}", + "physmem", "{", "max-size", "32m", "}", + "statseg", "{", "socket-name", cls.stats_sock, "}", + "socksvr", "{", "socket-name", cls.api_sock, "}", + "node { ", default_variant, "}", + "api-fuzz {", api_fuzzing, "}", + "plugins", "{", "plugin", "dpdk_plugin.so", "{", "disable", "}", + "plugin", "rdma_plugin.so", "{", "disable", "}", + "plugin", "lisp_unittest_plugin.so", "{", "enable", "}", + "plugin", "unittest_plugin.so", "{", "enable", "}" + ] + cls.extra_vpp_plugin_config + ["}", ]) if cls.extra_vpp_punt_config is not None: cls.vpp_cmdline.extend(cls.extra_vpp_punt_config) @@ -544,10 +573,10 @@ class VppTestCase(unittest.TestCase): cls.logger.addHandler(cls.file_handler) cls.logger.debug("--- setUpClass() for %s called ---" % cls.__name__) - cls.shm_prefix = os.path.basename(cls.tempdir) + cls.shm_prefix = os.path.basename(cls.tempdir) # Only used for VAPI os.chdir(cls.tempdir) - cls.logger.info("Temporary dir is %s, shm prefix is %s", - cls.tempdir, cls.shm_prefix) + cls.logger.info("Temporary dir is %s, api socket is %s", + cls.tempdir, cls.api_sock) cls.logger.debug("Random seed is %s" % seed) cls.setUpConstants() cls.reset_packet_infos() @@ -573,7 +602,7 @@ class VppTestCase(unittest.TestCase): cls.pump_thread.start() if cls.debug_gdb or cls.debug_gdbserver: cls.vapi_response_timeout = 0 - cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls, + cls.vapi = VppPapiProvider(cls.__name__, cls, cls.vapi_response_timeout) if cls.step: hook = hookmodule.StepHook(cls) @@ -591,7 +620,7 @@ class VppTestCase(unittest.TestCase): raise try: cls.vapi.connect() - except vpp_papi.VPPIOError as e: + except (vpp_papi.VPPIOError, Exception) as e: cls.logger.debug("Exception connecting to vapi: %s" % e) cls.vapi.disconnect() @@ -599,15 +628,15 @@ class VppTestCase(unittest.TestCase): print(colorize("You're running VPP inside gdbserver but " "VPP-API connection failed, did you forget " "to 'continue' VPP from within gdb?", RED)) - raise + raise e except vpp_papi.VPPRuntimeError as e: cls.logger.debug("%s" % e) cls.quit() - raise + raise e except Exception as e: cls.logger.debug("Exception connecting to VPP: %s" % e) cls.quit() - raise + raise e @classmethod def _debug_quit(cls): @@ -659,9 +688,15 @@ class VppTestCase(unittest.TestCase): cls.logger.debug("Sending TERM to vpp") cls.vpp.terminate() cls.logger.debug("Waiting for vpp to die") - cls.vpp.communicate() + try: + outs, errs = cls.vpp.communicate(timeout=5) + except subprocess.TimeoutExpired: + cls.vpp.kill() + outs, errs = cls.vpp.communicate() cls.logger.debug("Deleting class vpp attribute on %s", cls.__name__) + cls.vpp.stdout.close() + cls.vpp.stderr.close() del cls.vpp if cls.vpp_startup_failed: @@ -736,8 +771,8 @@ class VppTestCase(unittest.TestCase): os.rename(tmp_api_trace, vpp_api_trace_log) self.logger.info(self.vapi.ppcli("api trace custom-dump %s" % vpp_api_trace_log)) - except VppTransportShmemIOError: - self.logger.debug("VppTransportShmemIOError: Vpp dead. " + except VppTransportSocketIOError: + self.logger.debug("VppTransportSocketIOError: Vpp dead. " "Cannot log show commands.") self.vpp_dead = True else: @@ -748,7 +783,6 @@ class VppTestCase(unittest.TestCase): super(VppTestCase, self).setUp() self.reporter.send_keep_alive(self) if self.vpp_dead: - raise VppDiedError(rv=None, testcase=self.__class__.__name__, method_name=self._testMethodName) self.sleep(.1, "during setUp") @@ -805,9 +839,11 @@ class VppTestCase(unittest.TestCase): cls.sleep(0.1) @classmethod - def pg_start(cls): + def pg_start(cls, trace=True): """ Enable the PG, wait till it is done, then clean up """ - cls.vapi.cli("trace add pg-input 1000") + if trace: + cls.vapi.cli("clear trace") + cls.vapi.cli("trace add pg-input 1000") cls.vapi.cli('packet-generator enable') # PG, when starts, runs to completion - # so let's avoid a race condition, @@ -1141,7 +1177,7 @@ class VppTestCase(unittest.TestCase): "packet counter `%s'" % counter) def assert_error_counter_equal(self, counter, expected_value): - counter_value = self.statistics.get_err_counter(counter) + counter_value = self.statistics[counter].sum() self.assert_equal(counter_value, expected_value, "error counter `%s'" % counter) @@ -1170,14 +1206,13 @@ class VppTestCase(unittest.TestCase): after - before, timeout) cls.logger.debug( - "Finished sleep (%s) - slept %es (wanted %es)", - remark, after - before, timeout) + "Finished sleep (%s) - slept %es (wanted %es)", + remark, after - before, timeout) - def pg_send(self, intf, pkts, worker=None): - self.vapi.cli("clear trace") + def pg_send(self, intf, pkts, worker=None, trace=True): intf.add_stream(pkts, worker=worker) self.pg_enable_capture(self.pg_interfaces) - self.pg_start() + self.pg_start(trace=trace) def send_and_assert_no_replies(self, intf, pkts, remark="", timeout=None): self.pg_send(intf, pkts) @@ -1188,10 +1223,11 @@ class VppTestCase(unittest.TestCase): i.assert_nothing_captured(remark=remark) timeout = 0.1 - def send_and_expect(self, intf, pkts, output, n_rx=None, worker=None): + def send_and_expect(self, intf, pkts, output, n_rx=None, worker=None, + trace=True): if not n_rx: n_rx = len(pkts) - self.pg_send(intf, pkts, worker=worker) + self.pg_send(intf, pkts, worker=worker, trace=trace) rx = output.get_capture(n_rx) return rx @@ -1311,13 +1347,13 @@ class VppTestResult(unittest.TestResult): os.path.basename(self.current_test_case_info.tempdir)) self.current_test_case_info.logger.debug( - "creating a link to the failed test") + "creating a link to the failed test") self.current_test_case_info.logger.debug( - "os.symlink(%s, %s)" % - (self.current_test_case_info.tempdir, link_path)) + "os.symlink(%s, %s)" % + (self.current_test_case_info.tempdir, link_path)) if os.path.exists(link_path): self.current_test_case_info.logger.debug( - 'symlink already exists') + 'symlink already exists') else: os.symlink(self.current_test_case_info.tempdir, link_path) @@ -1422,11 +1458,18 @@ class VppTestResult(unittest.TestResult): raise Exception("No doc string for test '%s'" % test.id()) test_title = test_doc.splitlines()[0] test_title_colored = colorize(test_title, GREEN) - if test.force_solo(): + if test.is_tagged_run_solo(): # long live PEP-8 and 80 char width limitation... c = YELLOW test_title_colored = colorize("SOLO RUN: " + test_title, c) + # This block may overwrite the colorized title above, + # but we want this to stand out and be fixed + if test.has_tag(TestCaseTag.FIXME_VPP_WORKERS): + c = RED + w = "FIXME with VPP workers: " + test_title_colored = colorize(w + test_title, c) + if not hasattr(test.__class__, '_header_printed'): print(double_line_delim) print(test_title_colored)