X-Git-Url: https://gerrit.fd.io/r/gitweb?p=vpp.git;a=blobdiff_plain;f=test%2Fframework.py;h=dce477d16d2fc4a0567c41e35d09ead7cc8315eb;hp=5eeb5187041124a1981742d3862095965dabb16a;hb=192b13f96;hpb=919efad2671993d4c6d5a0dba8eeb99d5c60edf1 diff --git a/test/framework.py b/test/framework.py index 5eeb5187041..dce477d16d2 100644 --- a/test/framework.py +++ b/test/framework.py @@ -18,13 +18,14 @@ from threading import Thread, Event from inspect import getdoc, isclass from traceback import format_exception from logging import FileHandler, DEBUG, Formatter + +import scapy.compat from scapy.packet import Raw from hook import StepHook, PollHook, VppDiedError -from vpp_config import VppTestCaseVppConfig -from vpp_interface import VppInterface -from vpp_sub_interface import VppSubInterface from vpp_pg_interface import VppPGInterface +from vpp_sub_interface import VppSubInterface from vpp_lo_interface import VppLoInterface +from vpp_bvi_interface import VppBviInterface from vpp_papi_provider import VppPapiProvider from vpp_papi.vpp_stats import VPPStats from log import RED, GREEN, YELLOW, double_line_delim, single_line_delim, \ @@ -207,8 +208,8 @@ class VppTestCase(unittest.TestCase): classes. It provides methods to create and run test case. """ - CLI_LISTEN_DEFAULT = 'localhost:5002' - config = VppTestCaseVppConfig() + extra_vpp_punt_config = [] + extra_vpp_plugin_config = [] @property def packet_infos(self): @@ -274,14 +275,6 @@ class VppTestCase(unittest.TestCase): return random.choice(tuple(min_usage_set)) - @classmethod - def print_header(cls): - if not hasattr(cls, '_header_printed'): - print(double_line_delim) - print(colorize(getdoc(cls).splitlines()[0], GREEN)) - print(double_line_delim) - cls._header_printed = True - @classmethod def setUpConstants(cls): """ Set-up the test case class based on environment variables """ @@ -303,26 +296,32 @@ class VppTestCase(unittest.TestCase): plugin_path = cls.plugin_path elif cls.extern_plugin_path is not None: plugin_path = cls.extern_plugin_path - + debug_cli = "" if cls.step or cls.debug_gdb or cls.debug_gdbserver: - cls.config.add('unix', 'cli-listen', cls.CLI_LISTEN_DEFAULT) - + debug_cli = "cli-listen localhost:5002" coredump_size = None size = os.getenv("COREDUMP_SIZE") - cls.config.add('unix', 'coredump-size', - size if size is not None else 'unlimited') - - cls.config.add('unix', 'runtime-dir', cls.tempdir) - cls.config.add('api-segment', 'prefix', cls.shm_prefix) - cls.config.add('cpu', 'main-core', str(cls.get_least_used_cpu())) - cls.config.add('statseg', 'socket-name', cls.stats_sock) - + if size is not None: + coredump_size = "coredump-size %s" % size + if coredump_size is None: + coredump_size = "coredump-size unlimited" + + cpu_core_number = cls.get_least_used_cpu() + + cls.vpp_cmdline = [cls.vpp_bin, "unix", + "{", "nodaemon", debug_cli, "full-coredump", + coredump_size, "runtime-dir", cls.tempdir, "}", + "api-trace", "{", "on", "}", "api-segment", "{", + "prefix", cls.shm_prefix, "}", "cpu", "{", + "main-core", str(cpu_core_number), "}", "statseg", + "{", "socket-name", cls.stats_sock, "}", "plugins", + "{", "plugin", "dpdk_plugin.so", "{", "disable", + "}", "plugin", "unittest_plugin.so", "{", "enable", + "}"] + cls.extra_vpp_plugin_config + ["}", ] + if cls.extra_vpp_punt_config is not None: + cls.vpp_cmdline.extend(cls.extra_vpp_punt_config) if plugin_path is not None: - cls.config.add('plugins', 'path', plugin_path) - cls.config.add_plugin('dpdk_plugin.so', 'disable') - cls.config.add_plugin('unittest_plugin.so', 'enable') - - cls.vpp_cmdline = [cls.vpp_bin] + cls.config.shlex() + cls.vpp_cmdline.extend(["plugin_path", plugin_path]) cls.logger.info("vpp_cmdline args: %s" % cls.vpp_cmdline) cls.logger.info("vpp_cmdline: %s" % " ".join(cls.vpp_cmdline)) @@ -371,7 +370,16 @@ class VppTestCase(unittest.TestCase): stderr=subprocess.PIPE, bufsize=1) except subprocess.CalledProcessError as e: - cls.logger.critical("Couldn't start vpp: %s" % e) + cls.logger.critical("Subprocess returned with non-0 return code: (" + "%s)", e.returncode) + raise + except OSError as e: + cls.logger.critical("Subprocess returned with OS error: " + "(%s) %s", e.errno, e.strerror) + raise + except Exception as e: + cls.logger.exception("Subprocess returned unexpected from " + "%s:", cmdline) raise cls.wait_for_enter() @@ -385,7 +393,7 @@ class VppTestCase(unittest.TestCase): if os.path.exists(cls.stats_sock): ok = True break - time.sleep(0.8) + cls.sleep(0.8) if not ok: cls.logger.critical("Couldn't stat : {}".format(cls.stats_sock)) @@ -395,9 +403,9 @@ class VppTestCase(unittest.TestCase): Perform class setup before running the testcase Remove shared memory files, start vpp and connect the vpp-api """ + super(VppTestCase, cls).setUpClass() gc.collect() # run garbage collection first random.seed() - cls.print_header() cls.logger = get_logger(cls.__name__) if hasattr(cls, 'parallel_handler'): cls.logger.addHandler(cls.parallel_handler) @@ -506,7 +514,11 @@ class VppTestCase(unittest.TestCase): if hasattr(cls, 'vpp'): if hasattr(cls, 'vapi'): + cls.logger.debug("Disconnecting class vapi client on %s", + cls.__name__) cls.vapi.disconnect() + cls.logger.debug("Deleting class vapi attribute on %s", + cls.__name__) del cls.vapi cls.vpp.poll() if cls.vpp.returncode is None: @@ -514,6 +526,8 @@ class VppTestCase(unittest.TestCase): cls.vpp.kill() cls.logger.debug("Waiting for vpp to die") cls.vpp.communicate() + cls.logger.debug("Deleting class vpp attribute on %s", + cls.__name__) del cls.vpp if cls.vpp_startup_failed: @@ -537,7 +551,7 @@ class VppTestCase(unittest.TestCase): stderr_log(single_line_delim) stderr_log('VPP output to stderr while running %s:', cls.__name__) stderr_log(single_line_delim) - vpp_output = "".join(str(cls.vpp_stderr_deque)) + vpp_output = "".join(cls.vpp_stderr_deque) with open(cls.tempdir + '/vpp_stderr.txt', 'w') as f: f.write(vpp_output) stderr_log('\n%s', vpp_output) @@ -559,7 +573,7 @@ class VppTestCase(unittest.TestCase): (self.__class__.__name__, self._testMethodName, self._testMethodDoc)) if not self.vpp_dead: - self.logger.debug(self.vapi.cli("show trace")) + self.logger.debug(self.vapi.cli("show trace max 1000")) self.logger.info(self.vapi.ppcli("show interface")) self.logger.info(self.vapi.ppcli("show hardware")) self.logger.info(self.statistics.set_errors_str()) @@ -581,6 +595,7 @@ class VppTestCase(unittest.TestCase): def setUp(self): """ Clear trace before running each test""" + super(VppTestCase, self).setUp() self.reporter.send_keep_alive(self) self.logger.debug("--- setUp() for %s.%s(%s) called ---" % (self.__class__.__name__, self._testMethodName, @@ -640,7 +655,7 @@ class VppTestCase(unittest.TestCase): cls.logger.debug("Removing zombie capture %s" % cap_name) cls.vapi.cli('packet-generator delete %s' % cap_name) - cls.vapi.cli("trace add pg-input 50") # 50 is maximum + cls.vapi.cli("trace add pg-input 1000") cls.vapi.cli('packet-generator enable') cls._zombie_captures = cls._captures cls._captures = [] @@ -676,6 +691,20 @@ class VppTestCase(unittest.TestCase): cls.lo_interfaces = result return result + @classmethod + def create_bvi_interfaces(cls, count): + """ + Create BVI interfaces. + + :param count: number of interfaces created. + :returns: List of created interfaces. + """ + result = [VppBviInterface(cls) for i in range(count)] + for intf in result: + setattr(cls, intf.name, intf) + cls.bvi_interfaces = result + return result + @staticmethod def extend_packet(packet, size, padding=' '): """ @@ -739,16 +768,19 @@ class VppTestCase(unittest.TestCase): info.ip, info.proto) @staticmethod - def payload_to_info(payload): + def payload_to_info(payload, payload_field='load'): """ Convert packet payload to _PacketInfo object :param payload: packet payload - + :type payload: + :param payload_field: packet fieldname of payload "load" for + + :type payload_field: str :returns: _PacketInfo object containing de-serialized data from payload """ - numbers = payload.split() + numbers = getattr(payload, payload_field).split() info = _PacketInfo() info.index = int(numbers[0]) info.src = int(numbers[1]) @@ -839,22 +871,22 @@ class VppTestCase(unittest.TestCase): def assert_packet_checksums_valid(self, packet, ignore_zero_udp_checksums=True): - received = packet.__class__(str(packet)) + received = packet.__class__(scapy.compat.raw(packet)) self.logger.debug( ppp("Verifying packet checksums for packet:", received)) udp_layers = ['UDP', 'UDPerror'] checksum_fields = ['cksum', 'chksum'] checksums = [] counter = 0 - temp = received.__class__(str(received)) + temp = received.__class__(scapy.compat.raw(received)) while True: layer = temp.getlayer(counter) if layer: for cf in checksum_fields: if hasattr(layer, cf): if ignore_zero_udp_checksums and \ - 0 == getattr(layer, cf) and \ - layer.name in udp_layers: + 0 == getattr(layer, cf) and \ + layer.name in udp_layers: continue delattr(layer, cf) checksums.append((counter, cf)) @@ -863,7 +895,7 @@ class VppTestCase(unittest.TestCase): counter = counter + 1 if 0 == len(checksums): return - temp = temp.__class__(str(temp)) + temp = temp.__class__(scapy.compat.raw(temp)) for layer, cf in checksums: calc_sum = getattr(temp[layer], cf) self.assert_equal( @@ -880,9 +912,10 @@ class VppTestCase(unittest.TestCase): received_packet_checksum = getattr(received_packet[layer], field_name) if ignore_zero_checksum and 0 == received_packet_checksum: return - recalculated = received_packet.__class__(str(received_packet)) + recalculated = received_packet.__class__( + scapy.compat.raw(received_packet)) delattr(recalculated[layer], field_name) - recalculated = recalculated.__class__(str(recalculated)) + recalculated = recalculated.__class__(scapy.compat.raw(recalculated)) self.assert_equal(received_packet_checksum, getattr(recalculated[layer], field_name), "packet checksum on layer: %s" % layer) @@ -942,13 +975,26 @@ class VppTestCase(unittest.TestCase): @classmethod def sleep(cls, timeout, remark=None): + + # /* Allow sleep(0) to maintain win32 semantics, and as decreed + # * by Guido, only the main thread can be interrupted. + # */ + # https://github.com/python/cpython/blob/6673decfa0fb078f60587f5cb5e98460eea137c2/Modules/timemodule.c#L1892 # noqa + if timeout == 0: + # yield quantum + if hasattr(os, 'sched_yield'): + os.sched_yield() + else: + time.sleep(0) + return + if hasattr(cls, 'logger'): cls.logger.debug("Starting sleep for %es (%s)", timeout, remark) before = time.time() time.sleep(timeout) after = time.time() if hasattr(cls, 'logger') and after - before > 2 * timeout: - cls.logger.error("unexpected time.sleep() result - " + cls.logger.error("unexpected self.sleep() result - " "slept for %es instead of ~%es!", after - before, timeout) if hasattr(cls, 'logger'): @@ -956,11 +1002,14 @@ class VppTestCase(unittest.TestCase): "Finished sleep (%s) - slept %es (wanted %es)", remark, after - before, timeout) - def send_and_assert_no_replies(self, intf, pkts, remark="", timeout=None): + def pg_send(self, intf, pkts): self.vapi.cli("clear trace") intf.add_stream(pkts) self.pg_enable_capture(self.pg_interfaces) self.pg_start() + + def send_and_assert_no_replies(self, intf, pkts, remark="", timeout=None): + self.pg_send(intf, pkts) if not timeout: timeout = 1 for i in self.pg_interfaces: @@ -968,32 +1017,15 @@ class VppTestCase(unittest.TestCase): i.assert_nothing_captured(remark=remark) timeout = 0.1 - def send_and_expect(self, input, pkts, output): - self.vapi.cli("clear trace") - input.add_stream(pkts) - self.pg_enable_capture(self.pg_interfaces) - self.pg_start() - if isinstance(object, (list,)): - rx = [] - for o in output: - rx.append(output.get_capture(len(pkts))) - else: - rx = output.get_capture(len(pkts)) + def send_and_expect(self, intf, pkts, output): + self.pg_send(intf, pkts) + rx = output.get_capture(len(pkts)) return rx - def send_and_expect_only(self, input, pkts, output, timeout=None): - self.vapi.cli("clear trace") - input.add_stream(pkts) - self.pg_enable_capture(self.pg_interfaces) - self.pg_start() - if isinstance(object, (list,)): - outputs = output - rx = [] - for o in outputs: - rx.append(output.get_capture(len(pkts))) - else: - rx = output.get_capture(len(pkts)) - outputs = [output] + def send_and_expect_only(self, intf, pkts, output, timeout=None): + self.pg_send(intf, pkts) + rx = output.get_capture(len(pkts)) + outputs = [output] if not timeout: timeout = 1 for i in self.pg_interfaces: @@ -1004,6 +1036,11 @@ class VppTestCase(unittest.TestCase): return rx + def runTest(self): + """ unittest calls runTest when TestCase is instantiated without a + test case. Use case: Writing unittests against VppTestCase""" + pass + def get_testcase_doc_name(test): return getdoc(test.__class__).splitlines()[0] @@ -1045,7 +1082,8 @@ class VppTestResult(unittest.TestResult): core_crash_test_cases_info = set() current_test_case_info = None - def __init__(self, stream, descriptions, verbosity, runner): + def __init__(self, stream=None, descriptions=None, verbosity=None, + runner=None): """ :param stream File descriptor to store where to report test results. Set to the standard error stream by default. @@ -1053,7 +1091,7 @@ class VppTestResult(unittest.TestResult): test case descriptions. :param verbosity Integer variable to store required verbosity level. """ - unittest.TestResult.__init__(self, stream, descriptions, verbosity) + super(VppTestResult, self).__init__(stream, descriptions, verbosity) self.stream = stream self.descriptions = descriptions self.verbosity = verbosity @@ -1164,7 +1202,7 @@ class VppTestResult(unittest.TestResult): if isinstance(test, unittest.suite._ErrorHolder): test_name = str(test) else: - test_name = "'{}' ({})".format( + test_name = "'{!s}' ({!s})".format( get_testcase_doc_name(test), test.id()) self.current_test_case_info.core_crash_test = test_name self.core_crash_test_cases_info.add( @@ -1211,7 +1249,15 @@ class VppTestResult(unittest.TestResult): :param test: """ - test.print_header() + + def print_header(test): + if not hasattr(test.__class__, '_header_printed'): + print(double_line_delim) + print(colorize(getdoc(test).splitlines()[0], GREEN)) + print(double_line_delim) + test.__class__._header_printed = True + + print_header(test) unittest.TestResult.startTest(self, test) if self.verbosity > 0: @@ -1282,12 +1328,12 @@ class VppTestRunner(unittest.TextTestRunner): def __init__(self, keep_alive_pipe=None, descriptions=True, verbosity=1, result_pipe=None, failfast=False, buffer=False, - resultclass=None, print_summary=True): + resultclass=None, print_summary=True, **kwargs): # ignore stream setting here, use hard-coded stdout to be in sync # with prints from VppTestCase methods ... super(VppTestRunner, self).__init__(sys.stdout, descriptions, verbosity, failfast, buffer, - resultclass) + resultclass, **kwargs) KeepAliveReporter.pipe = keep_alive_pipe self.orig_stream = self.stream @@ -1347,3 +1393,6 @@ class Worker(Thread): self.logger.info(err) self.logger.info(single_line_delim) self.result = self.process.returncode + +if __name__ == '__main__': + pass