3 from __future__ import print_function
14 from collections import deque
15 from threading import Thread, Event
16 from inspect import getdoc, isclass
17 from traceback import format_exception
18 from logging import FileHandler, DEBUG, Formatter
19 from scapy.packet import Raw
20 from hook import StepHook, PollHook
21 from vpp_pg_interface import VppPGInterface
22 from vpp_sub_interface import VppSubInterface
23 from vpp_lo_interface import VppLoInterface
24 from vpp_papi_provider import VppPapiProvider
26 from vpp_object import VppObjectRegistry
27 from vpp_punt_socket import vpp_uds_socket_name
28 if os.name == 'posix' and sys.version_info[0] < 3:
29 # using subprocess32 is recommended by python official documentation
30 # @ https://docs.python.org/2/library/subprocess.html
31 import subprocess32 as subprocess
35 debug_framework = False
36 if os.getenv('TEST_DEBUG', "0") == "1":
37 debug_framework = True
42 Test framework module.
44 The module provides a set of tools for constructing and running tests and
45 representing the results.
49 class _PacketInfo(object):
50 """Private class to create packet info object.
52 Help process information about the next packet.
53 Set variables to default values.
55 #: Store the index of the packet.
57 #: Store the index of the source packet generator interface of the packet.
59 #: Store the index of the destination packet generator interface
62 #: Store expected ip version
64 #: Store expected upper protocol
66 #: Store the copy of the former packet.
69 def __eq__(self, other):
70 index = self.index == other.index
71 src = self.src == other.src
72 dst = self.dst == other.dst
73 data = self.data == other.data
74 return index and src and dst and data
77 def pump_output(testclass):
78 """ pump output from vpp stdout/stderr to proper queues """
81 while not testclass.pump_thread_stop_flag.wait(0):
82 readable = select.select([testclass.vpp.stdout.fileno(),
83 testclass.vpp.stderr.fileno(),
84 testclass.pump_thread_wakeup_pipe[0]],
86 if testclass.vpp.stdout.fileno() in readable:
87 read = os.read(testclass.vpp.stdout.fileno(), 102400)
89 split = read.splitlines(True)
90 if len(stdout_fragment) > 0:
91 split[0] = "%s%s" % (stdout_fragment, split[0])
92 if len(split) > 0 and split[-1].endswith("\n"):
96 stdout_fragment = split[-1]
97 testclass.vpp_stdout_deque.extend(split[:limit])
98 if not testclass.cache_vpp_output:
99 for line in split[:limit]:
100 testclass.logger.debug(
101 "VPP STDOUT: %s" % line.rstrip("\n"))
102 if testclass.vpp.stderr.fileno() in readable:
103 read = os.read(testclass.vpp.stderr.fileno(), 102400)
105 split = read.splitlines(True)
106 if len(stderr_fragment) > 0:
107 split[0] = "%s%s" % (stderr_fragment, split[0])
108 if len(split) > 0 and split[-1].endswith("\n"):
112 stderr_fragment = split[-1]
113 testclass.vpp_stderr_deque.extend(split[:limit])
114 if not testclass.cache_vpp_output:
115 for line in split[:limit]:
116 testclass.logger.debug(
117 "VPP STDERR: %s" % line.rstrip("\n"))
118 # ignoring the dummy pipe here intentionally - the flag will take care
119 # of properly terminating the loop
122 def running_extended_tests():
124 s = os.getenv("EXTENDED_TESTS")
125 return True if s.lower() in ("y", "yes", "1") else False
131 def running_on_centos():
133 os_id = os.getenv("OS_ID")
134 return True if "centos" in os_id.lower() else False
140 class KeepAliveReporter(object):
142 Singleton object which reports test start to parent process
147 self.__dict__ = self._shared_state
154 def pipe(self, pipe):
155 if hasattr(self, '_pipe'):
156 raise Exception("Internal error - pipe should only be set once.")
159 def send_keep_alive(self, test):
161 Write current test tmpdir & desc to keep-alive pipe to signal liveness
163 if self.pipe is None:
164 # if not running forked..
170 desc = test.shortDescription()
174 self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid))
177 class VppTestCase(unittest.TestCase):
178 """This subclass is a base class for VPP test cases that are implemented as
179 classes. It provides methods to create and run test case.
183 def packet_infos(self):
184 """List of packet infos"""
185 return self._packet_infos
188 def get_packet_count_for_if_idx(cls, dst_if_index):
189 """Get the number of packet info for specified destination if index"""
190 if dst_if_index in cls._packet_count_for_dst_if_idx:
191 return cls._packet_count_for_dst_if_idx[dst_if_index]
197 """Return the instance of this testcase"""
198 return cls.test_instance
201 def set_debug_flags(cls, d):
202 cls.debug_core = False
203 cls.debug_gdb = False
204 cls.debug_gdbserver = False
209 cls.debug_core = True
212 elif dl == "gdbserver":
213 cls.debug_gdbserver = True
215 raise Exception("Unrecognized DEBUG option: '%s'" % d)
218 def setUpConstants(cls):
219 """ Set-up the test case class based on environment variables """
221 s = os.getenv("STEP")
222 cls.step = True if s.lower() in ("y", "yes", "1") else False
226 d = os.getenv("DEBUG")
230 c = os.getenv("CACHE_OUTPUT", "1")
231 cls.cache_vpp_output = \
232 False if c.lower() in ("n", "no", "0") else True
234 cls.cache_vpp_output = True
235 cls.set_debug_flags(d)
236 cls.vpp_bin = os.getenv('VPP_TEST_BIN', "vpp")
237 cls.plugin_path = os.getenv('VPP_TEST_PLUGIN_PATH')
238 cls.extern_plugin_path = os.getenv('EXTERN_PLUGINS')
240 if cls.plugin_path is not None:
241 if cls.extern_plugin_path is not None:
242 plugin_path = "%s:%s" % (
243 cls.plugin_path, cls.extern_plugin_path)
245 plugin_path = cls.plugin_path
246 elif cls.extern_plugin_path is not None:
247 plugin_path = cls.extern_plugin_path
249 if cls.step or cls.debug_gdb or cls.debug_gdbserver:
250 debug_cli = "cli-listen localhost:5002"
253 size = os.getenv("COREDUMP_SIZE")
255 coredump_size = "coredump-size %s" % size
258 if coredump_size is None:
259 coredump_size = "coredump-size unlimited"
260 cls.vpp_cmdline = [cls.vpp_bin, "unix",
261 "{", "nodaemon", debug_cli, "full-coredump",
262 coredump_size, "}", "api-trace", "{", "on", "}",
263 "api-segment", "{", "prefix", cls.shm_prefix, "}",
264 "plugins", "{", "plugin", "dpdk_plugin.so", "{",
266 "punt", "{", "socket", cls.punt_socket_path, "}"]
267 if plugin_path is not None:
268 cls.vpp_cmdline.extend(["plugin_path", plugin_path])
269 cls.logger.info("vpp_cmdline: %s" % cls.vpp_cmdline)
272 def wait_for_enter(cls):
273 if cls.debug_gdbserver:
274 print(double_line_delim)
275 print("Spawned GDB server with PID: %d" % cls.vpp.pid)
277 print(double_line_delim)
278 print("Spawned VPP with PID: %d" % cls.vpp.pid)
280 cls.logger.debug("Spawned VPP with PID: %d" % cls.vpp.pid)
282 print(single_line_delim)
283 print("You can debug the VPP using e.g.:")
284 if cls.debug_gdbserver:
285 print("gdb " + cls.vpp_bin + " -ex 'target remote localhost:7777'")
286 print("Now is the time to attach a gdb by running the above "
287 "command, set up breakpoints etc. and then resume VPP from "
288 "within gdb by issuing the 'continue' command")
290 print("gdb " + cls.vpp_bin + " -ex 'attach %s'" % cls.vpp.pid)
291 print("Now is the time to attach a gdb by running the above "
292 "command and set up breakpoints etc.")
293 print(single_line_delim)
294 raw_input("Press ENTER to continue running the testcase...")
298 cmdline = cls.vpp_cmdline
300 if cls.debug_gdbserver:
301 gdbserver = '/usr/bin/gdbserver'
302 if not os.path.isfile(gdbserver) or \
303 not os.access(gdbserver, os.X_OK):
304 raise Exception("gdbserver binary '%s' does not exist or is "
305 "not executable" % gdbserver)
307 cmdline = [gdbserver, 'localhost:7777'] + cls.vpp_cmdline
308 cls.logger.info("Gdbserver cmdline is %s", " ".join(cmdline))
311 cls.vpp = subprocess.Popen(cmdline,
312 stdout=subprocess.PIPE,
313 stderr=subprocess.PIPE,
315 except Exception as e:
316 cls.logger.critical("Couldn't start vpp: %s" % e)
324 Perform class setup before running the testcase
325 Remove shared memory files, start vpp and connect the vpp-api
327 gc.collect() # run garbage collection first
329 cls.logger = getLogger(cls.__name__)
330 cls.tempdir = tempfile.mkdtemp(
331 prefix='vpp-unittest-%s-' % cls.__name__)
332 cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir)
333 cls.file_handler.setFormatter(
334 Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s',
336 cls.file_handler.setLevel(DEBUG)
337 cls.logger.addHandler(cls.file_handler)
338 cls.shm_prefix = cls.tempdir.split("/")[-1]
339 cls.punt_socket_path = '%s/%s' % (cls.tempdir, vpp_uds_socket_name)
340 os.chdir(cls.tempdir)
341 cls.logger.info("Temporary dir is %s, shm prefix is %s",
342 cls.tempdir, cls.shm_prefix)
344 cls.reset_packet_infos()
346 cls._zombie_captures = []
349 cls.registry = VppObjectRegistry()
350 cls.vpp_startup_failed = False
351 cls.reporter = KeepAliveReporter()
352 # need to catch exceptions here because if we raise, then the cleanup
353 # doesn't get called and we might end with a zombie vpp
356 cls.reporter.send_keep_alive(cls)
357 cls.vpp_stdout_deque = deque()
358 cls.vpp_stderr_deque = deque()
359 cls.pump_thread_stop_flag = Event()
360 cls.pump_thread_wakeup_pipe = os.pipe()
361 cls.pump_thread = Thread(target=pump_output, args=(cls,))
362 cls.pump_thread.daemon = True
363 cls.pump_thread.start()
364 cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls)
369 cls.vapi.register_hook(hook)
370 cls.sleep(0.1, "after vpp startup, before initial poll")
374 cls.vpp_startup_failed = True
376 "VPP died shortly after startup, check the"
377 " output to standard error for possible cause")
382 if cls.debug_gdbserver:
383 print(colorize("You're running VPP inside gdbserver but "
384 "VPP-API connection failed, did you forget "
385 "to 'continue' VPP from within gdb?", RED))
388 t, v, tb = sys.exc_info()
398 Disconnect vpp-api, kill vpp and cleanup shared memory files
400 if (cls.debug_gdbserver or cls.debug_gdb) and hasattr(cls, 'vpp'):
402 if cls.vpp.returncode is None:
403 print(double_line_delim)
404 print("VPP or GDB server is still running")
405 print(single_line_delim)
406 raw_input("When done debugging, press ENTER to kill the "
407 "process and finish running the testcase...")
409 os.write(cls.pump_thread_wakeup_pipe[1], 'ding dong wake up')
410 cls.pump_thread_stop_flag.set()
411 if hasattr(cls, 'pump_thread'):
412 cls.logger.debug("Waiting for pump thread to stop")
413 cls.pump_thread.join()
414 if hasattr(cls, 'vpp_stderr_reader_thread'):
415 cls.logger.debug("Waiting for stdderr pump to stop")
416 cls.vpp_stderr_reader_thread.join()
418 if hasattr(cls, 'vpp'):
419 if hasattr(cls, 'vapi'):
420 cls.vapi.disconnect()
423 if cls.vpp.returncode is None:
424 cls.logger.debug("Sending TERM to vpp")
426 cls.logger.debug("Waiting for vpp to die")
427 cls.vpp.communicate()
430 if cls.vpp_startup_failed:
431 stdout_log = cls.logger.info
432 stderr_log = cls.logger.critical
434 stdout_log = cls.logger.info
435 stderr_log = cls.logger.info
437 if hasattr(cls, 'vpp_stdout_deque'):
438 stdout_log(single_line_delim)
439 stdout_log('VPP output to stdout while running %s:', cls.__name__)
440 stdout_log(single_line_delim)
441 vpp_output = "".join(cls.vpp_stdout_deque)
442 with open(cls.tempdir + '/vpp_stdout.txt', 'w') as f:
444 stdout_log('\n%s', vpp_output)
445 stdout_log(single_line_delim)
447 if hasattr(cls, 'vpp_stderr_deque'):
448 stderr_log(single_line_delim)
449 stderr_log('VPP output to stderr while running %s:', cls.__name__)
450 stderr_log(single_line_delim)
451 vpp_output = "".join(cls.vpp_stderr_deque)
452 with open(cls.tempdir + '/vpp_stderr.txt', 'w') as f:
454 stderr_log('\n%s', vpp_output)
455 stderr_log(single_line_delim)
458 def tearDownClass(cls):
459 """ Perform final cleanup after running all tests in this test-case """
461 cls.file_handler.close()
462 cls.reset_packet_infos()
464 debug_internal.on_tear_down_class(cls)
467 """ Show various debug prints after each test """
468 self.logger.debug("--- tearDown() for %s.%s(%s) called ---" %
469 (self.__class__.__name__, self._testMethodName,
470 self._testMethodDoc))
471 if not self.vpp_dead:
472 self.logger.debug(self.vapi.cli("show trace"))
473 self.logger.info(self.vapi.ppcli("show interface"))
474 self.logger.info(self.vapi.ppcli("show hardware"))
475 self.logger.info(self.vapi.ppcli("show error"))
476 self.logger.info(self.vapi.ppcli("show run"))
477 self.registry.remove_vpp_config(self.logger)
478 # Save/Dump VPP api trace log
479 api_trace = "vpp_api_trace.%s.log" % self._testMethodName
480 tmp_api_trace = "/tmp/%s" % api_trace
481 vpp_api_trace_log = "%s/%s" % (self.tempdir, api_trace)
482 self.logger.info(self.vapi.ppcli("api trace save %s" % api_trace))
483 self.logger.info("Moving %s to %s\n" % (tmp_api_trace,
485 os.rename(tmp_api_trace, vpp_api_trace_log)
486 self.logger.info(self.vapi.ppcli("api trace custom-dump %s" %
489 self.registry.unregister_all(self.logger)
492 """ Clear trace before running each test"""
493 self.reporter.send_keep_alive(self)
494 self.logger.debug("--- setUp() for %s.%s(%s) called ---" %
495 (self.__class__.__name__, self._testMethodName,
496 self._testMethodDoc))
498 raise Exception("VPP is dead when setting up the test")
499 self.sleep(.1, "during setUp")
500 self.vpp_stdout_deque.append(
501 "--- test setUp() for %s.%s(%s) starts here ---\n" %
502 (self.__class__.__name__, self._testMethodName,
503 self._testMethodDoc))
504 self.vpp_stderr_deque.append(
505 "--- test setUp() for %s.%s(%s) starts here ---\n" %
506 (self.__class__.__name__, self._testMethodName,
507 self._testMethodDoc))
508 self.vapi.cli("clear trace")
509 # store the test instance inside the test class - so that objects
510 # holding the class can access instance methods (like assertEqual)
511 type(self).test_instance = self
514 def pg_enable_capture(cls, interfaces=None):
516 Enable capture on packet-generator interfaces
518 :param interfaces: iterable interface indexes (if None,
519 use self.pg_interfaces)
522 if interfaces is None:
523 interfaces = cls.pg_interfaces
528 def register_capture(cls, cap_name):
529 """ Register a capture in the testclass """
530 # add to the list of captures with current timestamp
531 cls._captures.append((time.time(), cap_name))
532 # filter out from zombies
533 cls._zombie_captures = [(stamp, name)
534 for (stamp, name) in cls._zombie_captures
539 """ Remove any zombie captures and enable the packet generator """
540 # how long before capture is allowed to be deleted - otherwise vpp
541 # crashes - 100ms seems enough (this shouldn't be needed at all)
544 for stamp, cap_name in cls._zombie_captures:
545 wait = stamp + capture_ttl - now
547 cls.sleep(wait, "before deleting capture %s" % cap_name)
549 cls.logger.debug("Removing zombie capture %s" % cap_name)
550 cls.vapi.cli('packet-generator delete %s' % cap_name)
552 cls.vapi.cli("trace add pg-input 50") # 50 is maximum
553 cls.vapi.cli('packet-generator enable')
554 cls._zombie_captures = cls._captures
558 def create_pg_interfaces(cls, interfaces):
560 Create packet-generator interfaces.
562 :param interfaces: iterable indexes of the interfaces.
563 :returns: List of created interfaces.
568 intf = VppPGInterface(cls, i)
569 setattr(cls, intf.name, intf)
571 cls.pg_interfaces = result
575 def create_loopback_interfaces(cls, interfaces):
577 Create loopback interfaces.
579 :param interfaces: iterable indexes of the interfaces.
580 :returns: List of created interfaces.
584 intf = VppLoInterface(cls, i)
585 setattr(cls, intf.name, intf)
587 cls.lo_interfaces = result
591 def extend_packet(packet, size, padding=' '):
593 Extend packet to given size by padding with spaces or custom padding
594 NOTE: Currently works only when Raw layer is present.
596 :param packet: packet
597 :param size: target size
598 :param padding: padding used to extend the payload
601 packet_len = len(packet) + 4
602 extend = size - packet_len
604 num = (extend / len(padding)) + 1
605 packet[Raw].load += (padding * num)[:extend]
608 def reset_packet_infos(cls):
609 """ Reset the list of packet info objects and packet counts to zero """
610 cls._packet_infos = {}
611 cls._packet_count_for_dst_if_idx = {}
614 def create_packet_info(cls, src_if, dst_if):
616 Create packet info object containing the source and destination indexes
617 and add it to the testcase's packet info list
619 :param VppInterface src_if: source interface
620 :param VppInterface dst_if: destination interface
622 :returns: _PacketInfo object
626 info.index = len(cls._packet_infos)
627 info.src = src_if.sw_if_index
628 info.dst = dst_if.sw_if_index
629 if isinstance(dst_if, VppSubInterface):
630 dst_idx = dst_if.parent.sw_if_index
632 dst_idx = dst_if.sw_if_index
633 if dst_idx in cls._packet_count_for_dst_if_idx:
634 cls._packet_count_for_dst_if_idx[dst_idx] += 1
636 cls._packet_count_for_dst_if_idx[dst_idx] = 1
637 cls._packet_infos[info.index] = info
641 def info_to_payload(info):
643 Convert _PacketInfo object to packet payload
645 :param info: _PacketInfo object
647 :returns: string containing serialized data from packet info
649 return "%d %d %d %d %d" % (info.index, info.src, info.dst,
653 def payload_to_info(payload):
655 Convert packet payload to _PacketInfo object
657 :param payload: packet payload
659 :returns: _PacketInfo object containing de-serialized data from payload
662 numbers = payload.split()
664 info.index = int(numbers[0])
665 info.src = int(numbers[1])
666 info.dst = int(numbers[2])
667 info.ip = int(numbers[3])
668 info.proto = int(numbers[4])
671 def get_next_packet_info(self, info):
673 Iterate over the packet info list stored in the testcase
674 Start iteration with first element if info is None
675 Continue based on index in info if info is specified
677 :param info: info or None
678 :returns: next info in list or None if no more infos
683 next_index = info.index + 1
684 if next_index == len(self._packet_infos):
687 return self._packet_infos[next_index]
689 def get_next_packet_info_for_interface(self, src_index, info):
691 Search the packet info list for the next packet info with same source
694 :param src_index: source interface index to search for
695 :param info: packet info - where to start the search
696 :returns: packet info or None
700 info = self.get_next_packet_info(info)
703 if info.src == src_index:
706 def get_next_packet_info_for_interface2(self, src_index, dst_index, info):
708 Search the packet info list for the next packet info with same source
709 and destination interface indexes
711 :param src_index: source interface index to search for
712 :param dst_index: destination interface index to search for
713 :param info: packet info - where to start the search
714 :returns: packet info or None
718 info = self.get_next_packet_info_for_interface(src_index, info)
721 if info.dst == dst_index:
724 def assert_equal(self, real_value, expected_value, name_or_class=None):
725 if name_or_class is None:
726 self.assertEqual(real_value, expected_value)
729 msg = "Invalid %s: %d('%s') does not match expected value %d('%s')"
730 msg = msg % (getdoc(name_or_class).strip(),
731 real_value, str(name_or_class(real_value)),
732 expected_value, str(name_or_class(expected_value)))
734 msg = "Invalid %s: %s does not match expected value %s" % (
735 name_or_class, real_value, expected_value)
737 self.assertEqual(real_value, expected_value, msg)
739 def assert_in_range(self,
747 msg = "Invalid %s: %s out of range <%s,%s>" % (
748 name, real_value, expected_min, expected_max)
749 self.assertTrue(expected_min <= real_value <= expected_max, msg)
752 def sleep(cls, timeout, remark=None):
753 if hasattr(cls, 'logger'):
754 cls.logger.debug("Starting sleep for %ss (%s)" % (timeout, remark))
758 if after - before > 2 * timeout:
759 cls.logger.error("unexpected time.sleep() result - "
760 "slept for %ss instead of ~%ss!" % (
761 after - before, timeout))
762 if hasattr(cls, 'logger'):
764 "Finished sleep (%s) - slept %ss (wanted %ss)" % (
765 remark, after - before, timeout))
767 def send_and_assert_no_replies(self, intf, pkts, remark=""):
768 self.vapi.cli("clear trace")
769 intf.add_stream(pkts)
770 self.pg_enable_capture(self.pg_interfaces)
773 for i in self.pg_interfaces:
774 i.get_capture(0, timeout=timeout)
775 i.assert_nothing_captured(remark=remark)
778 def send_and_expect(self, input, pkts, output):
779 self.vapi.cli("clear trace")
780 input.add_stream(pkts)
781 self.pg_enable_capture(self.pg_interfaces)
783 rx = output.get_capture(len(pkts))
787 class TestCasePrinter(object):
791 self.__dict__ = self._shared_state
792 if not hasattr(self, "_test_case_set"):
793 self._test_case_set = set()
795 def print_test_case_heading_if_first_time(self, case):
796 if case.__class__ not in self._test_case_set:
797 print(double_line_delim)
798 print(colorize(getdoc(case.__class__).splitlines()[0], YELLOW))
799 print(double_line_delim)
800 self._test_case_set.add(case.__class__)
803 class VppTestResult(unittest.TestResult):
805 @property result_string
806 String variable to store the test case result string.
808 List variable containing 2-tuples of TestCase instances and strings
809 holding formatted tracebacks. Each tuple represents a test which
810 raised an unexpected exception.
812 List variable containing 2-tuples of TestCase instances and strings
813 holding formatted tracebacks. Each tuple represents a test where
814 a failure was explicitly signalled using the TestCase.assert*()
818 def __init__(self, stream, descriptions, verbosity):
820 :param stream File descriptor to store where to report test results.
821 Set to the standard error stream by default.
822 :param descriptions Boolean variable to store information if to use
823 test case descriptions.
824 :param verbosity Integer variable to store required verbosity level.
826 unittest.TestResult.__init__(self, stream, descriptions, verbosity)
828 self.descriptions = descriptions
829 self.verbosity = verbosity
830 self.result_string = None
831 self.printer = TestCasePrinter()
833 def addSuccess(self, test):
835 Record a test succeeded result
840 if hasattr(test, 'logger'):
841 test.logger.debug("--- addSuccess() %s.%s(%s) called"
842 % (test.__class__.__name__,
843 test._testMethodName,
844 test._testMethodDoc))
845 unittest.TestResult.addSuccess(self, test)
846 self.result_string = colorize("OK", GREEN)
848 def addSkip(self, test, reason):
850 Record a test skipped.
856 if hasattr(test, 'logger'):
857 test.logger.debug("--- addSkip() %s.%s(%s) called, reason is %s"
858 % (test.__class__.__name__,
859 test._testMethodName,
862 unittest.TestResult.addSkip(self, test, reason)
863 self.result_string = colorize("SKIP", YELLOW)
865 def symlink_failed(self, test):
867 if hasattr(test, 'logger'):
869 if hasattr(test, 'tempdir'):
871 failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
872 link_path = '%s/%s-FAILED' % (failed_dir,
873 test.tempdir.split("/")[-1])
875 logger.debug("creating a link to the failed test")
876 logger.debug("os.symlink(%s, %s)" %
877 (test.tempdir, link_path))
878 os.symlink(test.tempdir, link_path)
879 except Exception as e:
883 def send_failure_through_pipe(self, test):
884 if hasattr(self, 'test_framework_failed_pipe'):
885 pipe = self.test_framework_failed_pipe
887 pipe.send(test.__class__)
889 def addFailure(self, test, err):
891 Record a test failed result
894 :param err: error message
897 if hasattr(test, 'logger'):
898 test.logger.debug("--- addFailure() %s.%s(%s) called, err is %s"
899 % (test.__class__.__name__,
900 test._testMethodName,
901 test._testMethodDoc, err))
902 test.logger.debug("formatted exception is:\n%s" %
903 "".join(format_exception(*err)))
904 unittest.TestResult.addFailure(self, test, err)
905 if hasattr(test, 'tempdir'):
906 self.result_string = colorize("FAIL", RED) + \
907 ' [ temp dir used by test case: ' + test.tempdir + ' ]'
908 self.symlink_failed(test)
910 self.result_string = colorize("FAIL", RED) + ' [no temp dir]'
912 self.send_failure_through_pipe(test)
914 def addError(self, test, err):
916 Record a test error result
919 :param err: error message
922 if hasattr(test, 'logger'):
923 test.logger.debug("--- addError() %s.%s(%s) called, err is %s"
924 % (test.__class__.__name__,
925 test._testMethodName,
926 test._testMethodDoc, err))
927 test.logger.debug("formatted exception is:\n%s" %
928 "".join(format_exception(*err)))
929 unittest.TestResult.addError(self, test, err)
930 if hasattr(test, 'tempdir'):
931 self.result_string = colorize("ERROR", RED) + \
932 ' [ temp dir used by test case: ' + test.tempdir + ' ]'
933 self.symlink_failed(test)
935 self.result_string = colorize("ERROR", RED) + ' [no temp dir]'
937 self.send_failure_through_pipe(test)
939 def getDescription(self, test):
944 :returns: test description
947 # TODO: if none print warning not raise exception
948 short_description = test.shortDescription()
949 if self.descriptions and short_description:
950 return short_description
954 def startTest(self, test):
961 self.printer.print_test_case_heading_if_first_time(test)
962 unittest.TestResult.startTest(self, test)
963 if self.verbosity > 0:
965 "Starting " + self.getDescription(test) + " ...")
966 self.stream.writeln(single_line_delim)
968 def stopTest(self, test):
975 unittest.TestResult.stopTest(self, test)
976 if self.verbosity > 0:
977 self.stream.writeln(single_line_delim)
978 self.stream.writeln("%-73s%s" % (self.getDescription(test),
980 self.stream.writeln(single_line_delim)
982 self.stream.writeln("%-73s%s" % (self.getDescription(test),
985 def printErrors(self):
987 Print errors from running the test case
989 self.stream.writeln()
990 self.printErrorList('ERROR', self.errors)
991 self.printErrorList('FAIL', self.failures)
993 def printErrorList(self, flavour, errors):
995 Print error list to the output stream together with error type
996 and test case description.
998 :param flavour: error type
999 :param errors: iterable errors
1002 for test, err in errors:
1003 self.stream.writeln(double_line_delim)
1004 self.stream.writeln("%s: %s" %
1005 (flavour, self.getDescription(test)))
1006 self.stream.writeln(single_line_delim)
1007 self.stream.writeln("%s" % err)
1010 class Filter_by_test_option:
1011 def __init__(self, filter_file_name, filter_class_name, filter_func_name):
1012 self.filter_file_name = filter_file_name
1013 self.filter_class_name = filter_class_name
1014 self.filter_func_name = filter_func_name
1016 def __call__(self, file_name, class_name, func_name):
1017 if self.filter_file_name and file_name != self.filter_file_name:
1019 if self.filter_class_name and class_name != self.filter_class_name:
1021 if self.filter_func_name and func_name != self.filter_func_name:
1026 class VppTestRunner(unittest.TextTestRunner):
1028 A basic test runner implementation which prints results to standard error.
1031 def resultclass(self):
1032 """Class maintaining the results of the tests"""
1033 return VppTestResult
1035 def __init__(self, keep_alive_pipe=None, failed_pipe=None,
1036 stream=sys.stderr, descriptions=True,
1037 verbosity=1, failfast=False, buffer=False, resultclass=None):
1038 # ignore stream setting here, use hard-coded stdout to be in sync
1039 # with prints from VppTestCase methods ...
1040 super(VppTestRunner, self).__init__(sys.stdout, descriptions,
1041 verbosity, failfast, buffer,
1043 reporter = KeepAliveReporter()
1044 reporter.pipe = keep_alive_pipe
1045 # this is super-ugly, but very simple to implement and works as long
1046 # as we run only one test at the same time
1047 VppTestResult.test_framework_failed_pipe = failed_pipe
1049 test_option = "TEST"
1051 def parse_test_option(self):
1053 f = os.getenv(self.test_option)
1056 filter_file_name = None
1057 filter_class_name = None
1058 filter_func_name = None
1061 parts = f.split('.')
1063 raise Exception("Unrecognized %s option: %s" %
1064 (self.test_option, f))
1066 if parts[2] not in ('*', ''):
1067 filter_func_name = parts[2]
1068 if parts[1] not in ('*', ''):
1069 filter_class_name = parts[1]
1070 if parts[0] not in ('*', ''):
1071 if parts[0].startswith('test_'):
1072 filter_file_name = parts[0]
1074 filter_file_name = 'test_%s' % parts[0]
1076 if f.startswith('test_'):
1077 filter_file_name = f
1079 filter_file_name = 'test_%s' % f
1080 return filter_file_name, filter_class_name, filter_func_name
1083 def filter_tests(tests, filter_cb):
1084 result = unittest.suite.TestSuite()
1086 if isinstance(t, unittest.suite.TestSuite):
1087 # this is a bunch of tests, recursively filter...
1088 x = filter_tests(t, filter_cb)
1089 if x.countTestCases() > 0:
1091 elif isinstance(t, unittest.TestCase):
1092 # this is a single test
1093 parts = t.id().split('.')
1094 # t.id() for common cases like this:
1095 # test_classifier.TestClassifier.test_acl_ip
1096 # apply filtering only if it is so
1098 if not filter_cb(parts[0], parts[1], parts[2]):
1102 # unexpected object, don't touch it
1106 def run(self, test):
1113 faulthandler.enable() # emit stack trace to stderr if killed by signal
1114 print("Running tests using custom test runner") # debug message
1115 filter_file, filter_class, filter_func = self.parse_test_option()
1116 print("Active filters: file=%s, class=%s, function=%s" % (
1117 filter_file, filter_class, filter_func))
1118 filter_cb = Filter_by_test_option(
1119 filter_file, filter_class, filter_func)
1120 filtered = self.filter_tests(test, filter_cb)
1121 print("%s out of %s tests match specified filters" % (
1122 filtered.countTestCases(), test.countTestCases()))
1123 if not running_extended_tests():
1124 print("Not running extended tests (some tests will be skipped)")
1125 return super(VppTestRunner, self).run(filtered)
1128 class Worker(Thread):
1129 def __init__(self, args, logger):
1130 self.logger = logger
1134 super(Worker, self).__init__()
1137 executable = self.args[0]
1138 self.logger.debug("Running executable w/args `%s'" % self.args)
1139 env = os.environ.copy()
1140 env.update(self.env)
1141 env["CK_LOG_FILE_NAME"] = "-"
1142 self.process = subprocess.Popen(
1143 self.args, shell=False, env=env, preexec_fn=os.setpgrp,
1144 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1145 out, err = self.process.communicate()
1146 self.logger.debug("Finished running `%s'" % executable)
1147 self.logger.info("Return code is `%s'" % self.process.returncode)
1148 self.logger.info(single_line_delim)
1149 self.logger.info("Executable `%s' wrote to stdout:" % executable)
1150 self.logger.info(single_line_delim)
1151 self.logger.info(out)
1152 self.logger.info(single_line_delim)
1153 self.logger.info("Executable `%s' wrote to stderr:" % executable)
1154 self.logger.info(single_line_delim)
1155 self.logger.info(err)
1156 self.logger.info(single_line_delim)
1157 self.result = self.process.returncode