X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=test%2Fframework.py;h=9bb3e01c31e5e2721b003fd3c5bb87d9a3ef7b5e;hb=6809538e646bf86c000dc1faba60b0a4157ad898;hp=24ee8692976bb455b8a25dae0b484415a3c31a25;hpb=bfd7d294d06e22f7bf809e8ac26220ebf51db683;p=vpp.git diff --git a/test/framework.py b/test/framework.py index 24ee8692976..9bb3e01c31e 100644 --- a/test/framework.py +++ b/test/framework.py @@ -2,10 +2,12 @@ from __future__ import print_function import gc +import logging import sys import os import select import signal +import subprocess import unittest import tempfile import time @@ -19,6 +21,7 @@ from threading import Thread, Event from inspect import getdoc, isclass from traceback import format_exception from logging import FileHandler, DEBUG, Formatter +from enum import Enum import scapy.compat from scapy.packet import Raw @@ -28,6 +31,7 @@ from vpp_sub_interface import VppSubInterface from vpp_lo_interface import VppLoInterface from vpp_bvi_interface import VppBviInterface from vpp_papi_provider import VppPapiProvider +import vpp_papi from vpp_papi.vpp_stats import VPPStats from vpp_papi.vpp_transport_shmem import VppTransportShmemIOError from log import RED, GREEN, YELLOW, double_line_delim, single_line_delim, \ @@ -38,18 +42,11 @@ from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror from scapy.layers.inet6 import ICMPv6DestUnreach, ICMPv6EchoRequest from scapy.layers.inet6 import ICMPv6EchoReply -if os.name == 'posix' and sys.version_info[0] < 3: - # using subprocess32 is recommended by python official documentation - # @ https://docs.python.org/2/library/subprocess.html - import subprocess32 as subprocess -else: - import subprocess - -# Python2/3 compatible -try: - input = raw_input -except NameError: - pass +logger = logging.getLogger(__name__) + +# Set up an empty logger for the testcase that can be overridden as necessary +null_logger = logging.getLogger('VppTestCase') +null_logger.addHandler(logging.NullHandler()) PASS = 0 FAIL = 1 @@ -216,12 +213,11 @@ def _running_extended_tests(): running_extended_tests = _running_extended_tests() -def _running_on_centos(): - os_id = os.getenv("OS_ID", "") - return True if "centos" in os_id.lower() else False +def _running_gcov_tests(): + return BoolEnvironmentVariable("GCOV_TESTS") -running_on_centos = _running_on_centos() +running_gcov_tests = _running_gcov_tests() class KeepAliveReporter(object): @@ -260,6 +256,28 @@ class KeepAliveReporter(object): self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid)) +class TestCaseTag(Enum): + # marks the suites that must run at the end + # using only a single test runner + RUN_SOLO = 1 + # marks the suites broken on VPP multi-worker + FIXME_VPP_WORKERS = 2 + + +def create_tag_decorator(e): + def decorator(cls): + try: + cls.test_tags.append(e) + except AttributeError: + cls.test_tags = [e] + return cls + return decorator + + +tag_run_solo = create_tag_decorator(TestCaseTag.RUN_SOLO) +tag_fixme_vpp_workers = create_tag_decorator(TestCaseTag.FIXME_VPP_WORKERS) + + class VppTestCase(unittest.TestCase): """This subclass is a base class for VPP test cases that are implemented as classes. It provides methods to create and run test case. @@ -267,6 +285,7 @@ class VppTestCase(unittest.TestCase): extra_vpp_punt_config = [] extra_vpp_plugin_config = [] + logger = null_logger vapi_response_timeout = 5 @property @@ -282,6 +301,20 @@ class VppTestCase(unittest.TestCase): else: return 0 + @classmethod + def has_tag(cls, tag): + """ if the test case has a given tag - return true """ + try: + return tag in cls.test_tags + except AttributeError: + pass + return False + + @classmethod + def is_tagged_run_solo(cls): + """ if the test case class is timing-sensitive - return true """ + return cls.has_tag(TestCaseTag.RUN_SOLO) + @classmethod def instance(cls): """Return the instance of this testcase""" @@ -371,7 +404,20 @@ class VppTestCase(unittest.TestCase): cpu_core_number = cls.get_least_used_cpu() if not hasattr(cls, "worker_config"): - cls.worker_config = "" + cls.worker_config = os.getenv("VPP_WORKER_CONFIG", "") + if cls.worker_config != "": + if cls.has_tag(TestCaseTag.FIXME_VPP_WORKERS): + cls.worker_config = "" + + default_variant = os.getenv("VARIANT") + if default_variant is not None: + default_variant = "defaults { %s 100 }" % default_variant + else: + default_variant = "" + + api_fuzzing = os.getenv("API_FUZZ") + if api_fuzzing is None: + api_fuzzing = 'off' cls.vpp_cmdline = [cls.vpp_bin, "unix", "{", "nodaemon", debug_cli, "full-coredump", @@ -380,13 +426,19 @@ class VppTestCase(unittest.TestCase): "prefix", cls.shm_prefix, "}", "cpu", "{", "main-core", str(cpu_core_number), cls.worker_config, "}", + "physmem", "{", "max-size", "32m", "}", "statseg", "{", "socket-name", cls.stats_sock, "}", "socksvr", "{", "socket-name", cls.api_sock, "}", + "node { ", default_variant, "}", + "api-fuzz {", api_fuzzing, "}", "plugins", "{", "plugin", "dpdk_plugin.so", "{", "disable", "}", "plugin", "rdma_plugin.so", "{", "disable", + "}", "plugin", "lisp_unittest_plugin.so", "{", + "enable", "}", "plugin", "unittest_plugin.so", "{", "enable", "}"] + cls.extra_vpp_plugin_config + ["}", ] + if cls.extra_vpp_punt_config is not None: cls.vpp_cmdline.extend(cls.extra_vpp_punt_config) if plugin_path is not None: @@ -444,8 +496,7 @@ class VppTestCase(unittest.TestCase): try: cls.vpp = subprocess.Popen(cmdline, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=1) + stderr=subprocess.PIPE) except subprocess.CalledProcessError as e: cls.logger.critical("Subprocess returned with non-0 return code: (" "%s)", e.returncode) @@ -461,19 +512,6 @@ class VppTestCase(unittest.TestCase): cls.wait_for_enter() - @classmethod - def wait_for_stats_socket(cls): - deadline = time.time() + 300 - ok = False - while time.time() < deadline or \ - cls.debug_gdb or cls.debug_gdbserver: - if os.path.exists(cls.stats_sock): - ok = True - break - cls.sleep(0.8) - if not ok: - cls.logger.critical("Couldn't stat : {}".format(cls.stats_sock)) - @classmethod def wait_for_coredump(cls): corefile = cls.tempdir + "/core" @@ -559,7 +597,6 @@ class VppTestCase(unittest.TestCase): else: hook = hookmodule.PollHook(cls) cls.vapi.register_hook(hook) - cls.wait_for_stats_socket() cls.statistics = VPPStats(socketname=cls.stats_sock) try: hook.poll_vpp() @@ -571,36 +608,46 @@ class VppTestCase(unittest.TestCase): raise try: cls.vapi.connect() - except Exception: - try: - cls.vapi.disconnect() - except Exception: - pass + except vpp_papi.VPPIOError as e: + cls.logger.debug("Exception connecting to vapi: %s" % e) + cls.vapi.disconnect() + if cls.debug_gdbserver: print(colorize("You're running VPP inside gdbserver but " "VPP-API connection failed, did you forget " "to 'continue' VPP from within gdb?", RED)) raise + except vpp_papi.VPPRuntimeError as e: + cls.logger.debug("%s" % e) + cls.quit() + raise except Exception as e: cls.logger.debug("Exception connecting to VPP: %s" % e) - cls.quit() raise + @classmethod + def _debug_quit(cls): + if (cls.debug_gdbserver or cls.debug_gdb): + try: + cls.vpp.poll() + + if cls.vpp.returncode is None: + print() + print(double_line_delim) + print("VPP or GDB server is still running") + print(single_line_delim) + input("When done debugging, press ENTER to kill the " + "process and finish running the testcase...") + except AttributeError: + pass + @classmethod def quit(cls): """ Disconnect vpp-api, kill vpp and cleanup shared memory files """ - if (cls.debug_gdbserver or cls.debug_gdb) and hasattr(cls, 'vpp'): - cls.vpp.poll() - if cls.vpp.returncode is None: - print() - print(double_line_delim) - print("VPP or GDB server is still running") - print(single_line_delim) - input("When done debugging, press ENTER to kill the " - "process and finish running the testcase...") + cls._debug_quit() # first signal that we want to stop the pump thread, then wake it up if hasattr(cls, 'pump_thread_stop_flag'): @@ -611,11 +658,12 @@ class VppTestCase(unittest.TestCase): cls.logger.debug("Waiting for pump thread to stop") cls.pump_thread.join() if hasattr(cls, 'vpp_stderr_reader_thread'): - cls.logger.debug("Waiting for stdderr pump to stop") + cls.logger.debug("Waiting for stderr pump to stop") cls.vpp_stderr_reader_thread.join() if hasattr(cls, 'vpp'): if hasattr(cls, 'vapi'): + cls.logger.debug(cls.vapi.vpp.get_stats()) cls.logger.debug("Disconnecting class vapi client on %s", cls.__name__) cls.vapi.disconnect() @@ -631,6 +679,8 @@ class VppTestCase(unittest.TestCase): cls.vpp.communicate() cls.logger.debug("Deleting class vpp attribute on %s", cls.__name__) + cls.vpp.stdout.close() + cls.vpp.stderr.close() del cls.vpp if cls.vpp_startup_failed: @@ -756,7 +806,12 @@ class VppTestCase(unittest.TestCase): @classmethod def get_vpp_time(cls): - return float(cls.vapi.cli('show clock').replace("Time now ", "")) + # processes e.g. "Time now 2.190522, Wed, 11 Mar 2020 17:29:54 GMT" + # returns float("2.190522") + timestr = cls.vapi.cli('show clock') + head, sep, tail = timestr.partition(',') + head, sep, tail = head.partition('Time now') + return float(tail) @classmethod def sleep_on_vpp_time(cls, sec): @@ -1007,13 +1062,15 @@ class VppTestCase(unittest.TestCase): while True: layer = temp.getlayer(counter) if layer: + layer = layer.copy() + layer.remove_payload() for cf in checksum_fields: if hasattr(layer, cf): if ignore_zero_udp_checksums and \ 0 == getattr(layer, cf) and \ layer.name in udp_layers: continue - delattr(layer, cf) + delattr(temp.getlayer(counter), cf) checksums.append((counter, cf)) else: break @@ -1122,23 +1179,22 @@ class VppTestCase(unittest.TestCase): time.sleep(0) return - if hasattr(cls, 'logger'): - cls.logger.debug("Starting sleep for %es (%s)", timeout, remark) + cls.logger.debug("Starting sleep for %es (%s)", timeout, remark) before = time.time() time.sleep(timeout) after = time.time() - if hasattr(cls, 'logger') and after - before > 2 * timeout: + if after - before > 2 * timeout: cls.logger.error("unexpected self.sleep() result - " "slept for %es instead of ~%es!", after - before, timeout) - if hasattr(cls, 'logger'): - cls.logger.debug( + + cls.logger.debug( "Finished sleep (%s) - slept %es (wanted %es)", remark, after - before, timeout) - def pg_send(self, intf, pkts): + def pg_send(self, intf, pkts, worker=None): self.vapi.cli("clear trace") - intf.add_stream(pkts) + intf.add_stream(pkts, worker=worker) self.pg_enable_capture(self.pg_interfaces) self.pg_start() @@ -1151,10 +1207,10 @@ class VppTestCase(unittest.TestCase): i.assert_nothing_captured(remark=remark) timeout = 0.1 - def send_and_expect(self, intf, pkts, output, n_rx=None): + def send_and_expect(self, intf, pkts, output, n_rx=None, worker=None): if not n_rx: n_rx = len(pkts) - self.pg_send(intf, pkts) + self.pg_send(intf, pkts, worker=worker) rx = output.get_capture(n_rx) return rx @@ -1172,11 +1228,6 @@ class VppTestCase(unittest.TestCase): return rx - def runTest(self): - """ unittest calls runTest when TestCase is instantiated without a - test case. Use case: Writing unittests against VppTestCase""" - pass - def get_testcase_doc_name(test): return getdoc(test.__class__).splitlines()[0] @@ -1277,22 +1328,20 @@ class VppTestResult(unittest.TestResult): failed_dir, '%s-FAILED' % os.path.basename(self.current_test_case_info.tempdir)) - if self.current_test_case_info.logger: - self.current_test_case_info.logger.debug( + + self.current_test_case_info.logger.debug( "creating a link to the failed test") - self.current_test_case_info.logger.debug( + self.current_test_case_info.logger.debug( "os.symlink(%s, %s)" % (self.current_test_case_info.tempdir, link_path)) if os.path.exists(link_path): - if self.current_test_case_info.logger: - self.current_test_case_info.logger.debug( + self.current_test_case_info.logger.debug( 'symlink already exists') else: os.symlink(self.current_test_case_info.tempdir, link_path) except Exception as e: - if self.current_test_case_info.logger: - self.current_test_case_info.logger.error(e) + self.current_test_case_info.logger.error(e) def send_result_through_pipe(self, test, result): if hasattr(self, 'test_framework_result_pipe'): @@ -1387,14 +1436,31 @@ class VppTestResult(unittest.TestResult): """ def print_header(test): + test_doc = getdoc(test) + if not test_doc: + raise Exception("No doc string for test '%s'" % test.id()) + test_title = test_doc.splitlines()[0] + test_title_colored = colorize(test_title, GREEN) + if test.is_tagged_run_solo(): + # long live PEP-8 and 80 char width limitation... + c = YELLOW + test_title_colored = colorize("SOLO RUN: " + test_title, c) + + # This block may overwrite the colorized title above, + # but we want this to stand out and be fixed + if test.has_tag(TestCaseTag.FIXME_VPP_WORKERS): + c = RED + w = "FIXME with VPP workers: " + test_title_colored = colorize(w + test_title, c) + if not hasattr(test.__class__, '_header_printed'): print(double_line_delim) - print(colorize(getdoc(test).splitlines()[0], GREEN)) + print(test_title_colored) print(double_line_delim) test.__class__._header_printed = True print_header(test) - + self.start_test = time.time() unittest.TestResult.startTest(self, test) if self.verbosity > 0: self.stream.writeln( @@ -1409,14 +1475,17 @@ class VppTestResult(unittest.TestResult): """ unittest.TestResult.stopTest(self, test) + if self.verbosity > 0: self.stream.writeln(single_line_delim) self.stream.writeln("%-73s%s" % (self.getDescription(test), self.result_string)) self.stream.writeln(single_line_delim) else: - self.stream.writeln("%-73s%s" % (self.getDescription(test), - self.result_string)) + self.stream.writeln("%-68s %4.2f %s" % + (self.getDescription(test), + time.time() - self.start_test, + self.result_string)) self.send_result_through_pipe(test, TEST_RUN) @@ -1500,16 +1569,17 @@ class VppTestRunner(unittest.TextTestRunner): class Worker(Thread): - def __init__(self, args, logger, env=None): + def __init__(self, executable_args, logger, env=None, *args, **kwargs): + super(Worker, self).__init__(*args, **kwargs) self.logger = logger - self.args = args + self.args = executable_args if hasattr(self, 'testcase') and self.testcase.debug_all: if self.testcase.debug_gdbserver: self.args = ['/usr/bin/gdbserver', 'localhost:{port}' .format(port=self.testcase.gdbserver_port)] + args elif self.testcase.debug_gdb and hasattr(self, 'wait_for_gdb'): self.args.append(self.wait_for_gdb) - self.app_bin = args[0] + self.app_bin = executable_args[0] self.app_name = os.path.basename(self.app_bin) if hasattr(self, 'role'): self.app_name += ' {role}'.format(role=self.role) @@ -1517,7 +1587,6 @@ class Worker(Thread): self.result = None env = {} if env is None else env self.env = copy.deepcopy(env) - super(Worker, self).__init__() def wait_for_enter(self): if not hasattr(self, 'testcase'):