X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=test%2Fframework.py;h=c73b46c491b05d76ae95b63d096093136a3290ad;hb=e64e5fff4;hp=1be227a3ead4148b42385f1019ec1ce6157d13e3;hpb=586d3edf49f5dd8cd3e4f836afd562e21766914d;p=vpp.git diff --git a/test/framework.py b/test/framework.py index 1be227a3ead..c73b46c491b 100644 --- a/test/framework.py +++ b/test/framework.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function import gc @@ -28,6 +28,7 @@ from vpp_sub_interface import VppSubInterface from vpp_lo_interface import VppLoInterface from vpp_bvi_interface import VppBviInterface from vpp_papi_provider import VppPapiProvider +import vpp_papi from vpp_papi.vpp_stats import VPPStats from vpp_papi.vpp_transport_shmem import VppTransportShmemIOError from log import RED, GREEN, YELLOW, double_line_delim, single_line_delim, \ @@ -171,7 +172,7 @@ def pump_output(testclass): testclass.vpp_stdout_deque.extend(split[:limit]) if not testclass.cache_vpp_output: for line in split[:limit]: - testclass.logger.debug( + testclass.logger.info( "VPP STDOUT: %s" % line.rstrip("\n")) if testclass.vpp.stderr.fileno() in readable: read = os.read(testclass.vpp.stderr.fileno(), 102400) @@ -189,7 +190,7 @@ def pump_output(testclass): testclass.vpp_stderr_deque.extend(split[:limit]) if not testclass.cache_vpp_output: for line in split[:limit]: - testclass.logger.debug( + testclass.logger.error( "VPP STDERR: %s" % line.rstrip("\n")) # ignoring the dummy pipe here intentionally - the # flag will take care of properly terminating the loop @@ -216,6 +217,13 @@ def _running_extended_tests(): running_extended_tests = _running_extended_tests() +def _running_gcov_tests(): + return BoolEnvironmentVariable("GCOV_TESTS") + + +running_gcov_tests = _running_gcov_tests() + + def _running_on_centos(): os_id = os.getenv("OS_ID", "") return True if "centos" in os_id.lower() else False @@ -267,6 +275,7 @@ class VppTestCase(unittest.TestCase): extra_vpp_punt_config = [] extra_vpp_plugin_config = [] + vapi_response_timeout = 5 @property def packet_infos(self): @@ -288,20 +297,24 @@ class VppTestCase(unittest.TestCase): @classmethod def set_debug_flags(cls, d): + cls.gdbserver_port = 7777 cls.debug_core = False cls.debug_gdb = False cls.debug_gdbserver = False + cls.debug_all = False if d is None: return dl = d.lower() if dl == "core": cls.debug_core = True - elif dl == "gdb": + elif dl == "gdb" or dl == "gdb-all": cls.debug_gdb = True - elif dl == "gdbserver": + elif dl == "gdbserver" or dl == "gdbserver-all": cls.debug_gdbserver = True else: raise Exception("Unrecognized DEBUG option: '%s'" % d) + if dl == "gdb-all" or dl == "gdbserver-all": + cls.debug_all = True @staticmethod def get_least_used_cpu(): @@ -368,6 +381,12 @@ class VppTestCase(unittest.TestCase): if not hasattr(cls, "worker_config"): cls.worker_config = "" + default_variant = os.getenv("VARIANT") + if default_variant is not None: + default_variant = "defaults { %s 100 }" % default_variant + else: + default_variant = "" + cls.vpp_cmdline = [cls.vpp_bin, "unix", "{", "nodaemon", debug_cli, "full-coredump", coredump_size, "runtime-dir", cls.tempdir, "}", @@ -375,13 +394,16 @@ class VppTestCase(unittest.TestCase): "prefix", cls.shm_prefix, "}", "cpu", "{", "main-core", str(cpu_core_number), cls.worker_config, "}", + "physmem", "{", "max-size", "32m", "}", "statseg", "{", "socket-name", cls.stats_sock, "}", "socksvr", "{", "socket-name", cls.api_sock, "}", + "node { ", default_variant, "}", "plugins", "{", "plugin", "dpdk_plugin.so", "{", "disable", "}", "plugin", "rdma_plugin.so", "{", "disable", "}", "plugin", "unittest_plugin.so", "{", "enable", "}"] + cls.extra_vpp_plugin_config + ["}", ] + if cls.extra_vpp_punt_config is not None: cls.vpp_cmdline.extend(cls.extra_vpp_punt_config) if plugin_path is not None: @@ -404,17 +426,20 @@ class VppTestCase(unittest.TestCase): cls.logger.debug("Spawned VPP with PID: %d" % cls.vpp.pid) return print(single_line_delim) - print("You can debug the VPP using e.g.:") + print("You can debug VPP using:") if cls.debug_gdbserver: print("sudo gdb " + cls.vpp_bin + - " -ex 'target remote localhost:7777'") - print("Now is the time to attach a gdb by running the above " - "command, set up breakpoints etc. and then resume VPP from " + " -ex 'target remote localhost:{port}'" + .format(port=cls.gdbserver_port)) + print("Now is the time to attach gdb by running the above " + "command, set up breakpoints etc., then resume VPP from " "within gdb by issuing the 'continue' command") + cls.gdbserver_port += 1 elif cls.debug_gdb: print("sudo gdb " + cls.vpp_bin + " -ex 'attach %s'" % cls.vpp.pid) - print("Now is the time to attach a gdb by running the above " - "command and set up breakpoints etc.") + print("Now is the time to attach gdb by running the above " + "command and set up breakpoints etc., then resume VPP from" + " within gdb by issuing the 'continue' command") print(single_line_delim) input("Press ENTER to continue running the testcase...") @@ -429,14 +454,14 @@ class VppTestCase(unittest.TestCase): raise Exception("gdbserver binary '%s' does not exist or is " "not executable" % gdbserver) - cmdline = [gdbserver, 'localhost:7777'] + cls.vpp_cmdline + cmdline = [gdbserver, 'localhost:{port}' + .format(port=cls.gdbserver_port)] + cls.vpp_cmdline cls.logger.info("Gdbserver cmdline is %s", " ".join(cmdline)) try: cls.vpp = subprocess.Popen(cmdline, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=1) + stderr=subprocess.PIPE) except subprocess.CalledProcessError as e: cls.logger.critical("Subprocess returned with non-0 return code: (" "%s)", e.returncode) @@ -452,19 +477,6 @@ class VppTestCase(unittest.TestCase): cls.wait_for_enter() - @classmethod - def wait_for_stats_socket(cls): - deadline = time.time() + 300 - ok = False - while time.time() < deadline or \ - cls.debug_gdb or cls.debug_gdbserver: - if os.path.exists(cls.stats_sock): - ok = True - break - cls.sleep(0.8) - if not ok: - cls.logger.critical("Couldn't stat : {}".format(cls.stats_sock)) - @classmethod def wait_for_coredump(cls): corefile = cls.tempdir + "/core" @@ -495,8 +507,9 @@ class VppTestCase(unittest.TestCase): """ super(VppTestCase, cls).setUpClass() gc.collect() # run garbage collection first - random.seed() cls.logger = get_logger(cls.__name__) + seed = os.environ["RND_SEED"] + random.seed(seed) if hasattr(cls, 'parallel_handler'): cls.logger.addHandler(cls.parallel_handler) cls.logger.propagate = False @@ -517,6 +530,7 @@ class VppTestCase(unittest.TestCase): os.chdir(cls.tempdir) cls.logger.info("Temporary dir is %s, shm prefix is %s", cls.tempdir, cls.shm_prefix) + cls.logger.debug("Random seed is %s" % seed) cls.setUpConstants() cls.reset_packet_infos() cls._captures = [] @@ -540,17 +554,14 @@ class VppTestCase(unittest.TestCase): cls.pump_thread.daemon = True cls.pump_thread.start() if cls.debug_gdb or cls.debug_gdbserver: - read_timeout = 0 - else: - read_timeout = 5 + cls.vapi_response_timeout = 0 cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls, - read_timeout) + cls.vapi_response_timeout) if cls.step: hook = hookmodule.StepHook(cls) else: hook = hookmodule.PollHook(cls) cls.vapi.register_hook(hook) - cls.wait_for_stats_socket() cls.statistics = VPPStats(socketname=cls.stats_sock) try: hook.poll_vpp() @@ -562,35 +573,46 @@ class VppTestCase(unittest.TestCase): raise try: cls.vapi.connect() - except Exception: - try: - cls.vapi.disconnect() - except Exception: - pass + except vpp_papi.VPPIOError as e: + cls.logger.debug("Exception connecting to vapi: %s" % e) + cls.vapi.disconnect() + if cls.debug_gdbserver: print(colorize("You're running VPP inside gdbserver but " "VPP-API connection failed, did you forget " "to 'continue' VPP from within gdb?", RED)) raise + except vpp_papi.VPPRuntimeError as e: + cls.logger.debug("%s" % e) + cls.quit() + raise except Exception as e: cls.logger.debug("Exception connecting to VPP: %s" % e) - cls.quit() raise + @classmethod + def _debug_quit(cls): + if (cls.debug_gdbserver or cls.debug_gdb): + try: + cls.vpp.poll() + + if cls.vpp.returncode is None: + print() + print(double_line_delim) + print("VPP or GDB server is still running") + print(single_line_delim) + input("When done debugging, press ENTER to kill the " + "process and finish running the testcase...") + except AttributeError: + pass + @classmethod def quit(cls): """ Disconnect vpp-api, kill vpp and cleanup shared memory files """ - if (cls.debug_gdbserver or cls.debug_gdb) and hasattr(cls, 'vpp'): - cls.vpp.poll() - if cls.vpp.returncode is None: - print(double_line_delim) - print("VPP or GDB server is still running") - print(single_line_delim) - input("When done debugging, press ENTER to kill the " - "process and finish running the testcase...") + cls._debug_quit() # first signal that we want to stop the pump thread, then wake it up if hasattr(cls, 'pump_thread_stop_flag'): @@ -601,11 +623,12 @@ class VppTestCase(unittest.TestCase): cls.logger.debug("Waiting for pump thread to stop") cls.pump_thread.join() if hasattr(cls, 'vpp_stderr_reader_thread'): - cls.logger.debug("Waiting for stdderr pump to stop") + cls.logger.debug("Waiting for stderr pump to stop") cls.vpp_stderr_reader_thread.join() if hasattr(cls, 'vpp'): if hasattr(cls, 'vapi'): + cls.logger.debug(cls.vapi.vpp.get_stats()) cls.logger.debug("Disconnecting class vapi client on %s", cls.__name__) cls.vapi.disconnect() @@ -744,6 +767,25 @@ class VppTestCase(unittest.TestCase): # add to the list of captures with current timestamp cls._captures.append((time.time(), cap_name)) + @classmethod + def get_vpp_time(cls): + # processes e.g. "Time now 2.190522, Wed, 11 Mar 2020 17:29:54 GMT" + # returns float("2.190522") + timestr = cls.vapi.cli('show clock') + head, sep, tail = timestr.partition(',') + head, sep, tail = head.partition('Time now') + return float(tail) + + @classmethod + def sleep_on_vpp_time(cls, sec): + """ Sleep according to time in VPP world """ + # On a busy system with many processes + # we might end up with VPP time being slower than real world + # So take that into account when waiting for VPP to do something + start_time = cls.get_vpp_time() + while cls.get_vpp_time() - start_time < sec: + cls.sleep(0.1) + @classmethod def pg_start(cls): """ Enable the PG, wait till it is done, then clean up """ @@ -983,13 +1025,15 @@ class VppTestCase(unittest.TestCase): while True: layer = temp.getlayer(counter) if layer: + layer = layer.copy() + layer.remove_payload() for cf in checksum_fields: if hasattr(layer, cf): if ignore_zero_udp_checksums and \ 0 == getattr(layer, cf) and \ layer.name in udp_layers: continue - delattr(layer, cf) + delattr(temp.getlayer(counter), cf) checksums.append((counter, cf)) else: break @@ -1112,9 +1156,9 @@ class VppTestCase(unittest.TestCase): "Finished sleep (%s) - slept %es (wanted %es)", remark, after - before, timeout) - def pg_send(self, intf, pkts): + def pg_send(self, intf, pkts, worker=None): self.vapi.cli("clear trace") - intf.add_stream(pkts) + intf.add_stream(pkts, worker=worker) self.pg_enable_capture(self.pg_interfaces) self.pg_start() @@ -1127,10 +1171,10 @@ class VppTestCase(unittest.TestCase): i.assert_nothing_captured(remark=remark) timeout = 0.1 - def send_and_expect(self, intf, pkts, output, n_rx=None): + def send_and_expect(self, intf, pkts, output, n_rx=None, worker=None): if not n_rx: n_rx = len(pkts) - self.pg_send(intf, pkts) + self.pg_send(intf, pkts, worker=worker) rx = output.get_capture(n_rx) return rx @@ -1148,11 +1192,6 @@ class VppTestCase(unittest.TestCase): return rx - def runTest(self): - """ unittest calls runTest when TestCase is instantiated without a - test case. Use case: Writing unittests against VppTestCase""" - pass - def get_testcase_doc_name(test): return getdoc(test.__class__).splitlines()[0] @@ -1370,7 +1409,7 @@ class VppTestResult(unittest.TestResult): test.__class__._header_printed = True print_header(test) - + self.start_test = time.time() unittest.TestResult.startTest(self, test) if self.verbosity > 0: self.stream.writeln( @@ -1385,14 +1424,17 @@ class VppTestResult(unittest.TestResult): """ unittest.TestResult.stopTest(self, test) + if self.verbosity > 0: self.stream.writeln(single_line_delim) self.stream.writeln("%-73s%s" % (self.getDescription(test), self.result_string)) self.stream.writeln(single_line_delim) else: - self.stream.writeln("%-73s%s" % (self.getDescription(test), - self.result_string)) + self.stream.writeln("%-68s %4.2f %s" % + (self.getDescription(test), + time.time() - self.start_test, + self.result_string)) self.send_result_through_pipe(test, TEST_RUN) @@ -1476,14 +1518,58 @@ class VppTestRunner(unittest.TextTestRunner): class Worker(Thread): - def __init__(self, args, logger, env=None): + def __init__(self, executable_args, logger, env=None, *args, **kwargs): + super(Worker, self).__init__(*args, **kwargs) self.logger = logger - self.args = args + self.args = executable_args + if hasattr(self, 'testcase') and self.testcase.debug_all: + if self.testcase.debug_gdbserver: + self.args = ['/usr/bin/gdbserver', 'localhost:{port}' + .format(port=self.testcase.gdbserver_port)] + args + elif self.testcase.debug_gdb and hasattr(self, 'wait_for_gdb'): + self.args.append(self.wait_for_gdb) + self.app_bin = executable_args[0] + self.app_name = os.path.basename(self.app_bin) + if hasattr(self, 'role'): + self.app_name += ' {role}'.format(role=self.role) self.process = None self.result = None env = {} if env is None else env self.env = copy.deepcopy(env) - super(Worker, self).__init__() + + def wait_for_enter(self): + if not hasattr(self, 'testcase'): + return + if self.testcase.debug_all and self.testcase.debug_gdbserver: + print() + print(double_line_delim) + print("Spawned GDB Server for '{app}' with PID: {pid}" + .format(app=self.app_name, pid=self.process.pid)) + elif self.testcase.debug_all and self.testcase.debug_gdb: + print() + print(double_line_delim) + print("Spawned '{app}' with PID: {pid}" + .format(app=self.app_name, pid=self.process.pid)) + else: + return + print(single_line_delim) + print("You can debug '{app}' using:".format(app=self.app_name)) + if self.testcase.debug_gdbserver: + print("sudo gdb " + self.app_bin + + " -ex 'target remote localhost:{port}'" + .format(port=self.testcase.gdbserver_port)) + print("Now is the time to attach gdb by running the above " + "command, set up breakpoints etc., then resume from " + "within gdb by issuing the 'continue' command") + self.testcase.gdbserver_port += 1 + elif self.testcase.debug_gdb: + print("sudo gdb " + self.app_bin + + " -ex 'attach {pid}'".format(pid=self.process.pid)) + print("Now is the time to attach gdb by running the above " + "command and set up breakpoints etc., then resume from" + " within gdb by issuing the 'continue' command") + print(single_line_delim) + input("Press ENTER to continue running the testcase...") def run(self): executable = self.args[0] @@ -1494,24 +1580,28 @@ class Worker(Thread): self.result = os.EX_OSFILE raise EnvironmentError( "executable '%s' is not found or executable." % executable) - self.logger.debug("Running executable w/args `%s'" % self.args) + self.logger.debug("Running executable: '{app}'" + .format(app=' '.join(self.args))) env = os.environ.copy() env.update(self.env) env["CK_LOG_FILE_NAME"] = "-" self.process = subprocess.Popen( self.args, shell=False, env=env, preexec_fn=os.setpgrp, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.wait_for_enter() out, err = self.process.communicate() - self.logger.debug("Finished running `%s'" % executable) + self.logger.debug("Finished running `{app}'".format(app=self.app_name)) self.logger.info("Return code is `%s'" % self.process.returncode) self.logger.info(single_line_delim) - self.logger.info("Executable `%s' wrote to stdout:" % executable) + self.logger.info("Executable `{app}' wrote to stdout:" + .format(app=self.app_name)) self.logger.info(single_line_delim) - self.logger.info(out) + self.logger.info(out.decode('utf-8')) self.logger.info(single_line_delim) - self.logger.info("Executable `%s' wrote to stderr:" % executable) + self.logger.info("Executable `{app}' wrote to stderr:" + .format(app=self.app_name)) self.logger.info(single_line_delim) - self.logger.info(err) + self.logger.info(err.decode('utf-8')) self.logger.info(single_line_delim) self.result = self.process.returncode