3 from __future__ import print_function
12 from collections import deque
13 from threading import Thread, Event
14 from inspect import getdoc
15 from traceback import format_exception
16 from logging import FileHandler, DEBUG, Formatter
17 from scapy.packet import Raw
18 from hook import StepHook, PollHook
19 from vpp_pg_interface import VppPGInterface
20 from vpp_sub_interface import VppSubInterface
21 from vpp_lo_interface import VppLoInterface
22 from vpp_papi_provider import VppPapiProvider
24 from vpp_object import VppObjectRegistry
25 if os.name == 'posix' and sys.version_info[0] < 3:
26 # using subprocess32 is recommended by python official documentation
27 # @ https://docs.python.org/2/library/subprocess.html
28 import subprocess32 as subprocess
33 Test framework module.
35 The module provides a set of tools for constructing and running tests and
36 representing the results.
40 class _PacketInfo(object):
41 """Private class to create packet info object.
43 Help process information about the next packet.
44 Set variables to default values.
46 #: Store the index of the packet.
48 #: Store the index of the source packet generator interface of the packet.
50 #: Store the index of the destination packet generator interface
53 #: Store the copy of the former packet.
56 def __eq__(self, other):
57 index = self.index == other.index
58 src = self.src == other.src
59 dst = self.dst == other.dst
60 data = self.data == other.data
61 return index and src and dst and data
64 def pump_output(testclass):
65 """ pump output from vpp stdout/stderr to proper queues """
66 while not testclass.pump_thread_stop_flag.wait(0):
67 readable = select.select([testclass.vpp.stdout.fileno(),
68 testclass.vpp.stderr.fileno(),
69 testclass.pump_thread_wakeup_pipe[0]],
71 if testclass.vpp.stdout.fileno() in readable:
72 read = os.read(testclass.vpp.stdout.fileno(), 1024)
73 testclass.vpp_stdout_deque.append(read)
74 if testclass.vpp.stderr.fileno() in readable:
75 read = os.read(testclass.vpp.stderr.fileno(), 1024)
76 testclass.vpp_stderr_deque.append(read)
77 # ignoring the dummy pipe here intentionally - the flag will take care
78 # of properly terminating the loop
81 class VppTestCase(unittest.TestCase):
82 """This subclass is a base class for VPP test cases that are implemented as
83 classes. It provides methods to create and run test case.
87 def packet_infos(self):
88 """List of packet infos"""
89 return self._packet_infos
92 def get_packet_count_for_if_idx(cls, dst_if_index):
93 """Get the number of packet info for specified destination if index"""
94 if dst_if_index in cls._packet_count_for_dst_if_idx:
95 return cls._packet_count_for_dst_if_idx[dst_if_index]
101 """Return the instance of this testcase"""
102 return cls.test_instance
105 def set_debug_flags(cls, d):
106 cls.debug_core = False
107 cls.debug_gdb = False
108 cls.debug_gdbserver = False
113 if resource.getrlimit(resource.RLIMIT_CORE)[0] <= 0:
114 # give a heads up if this is actually useless
115 print(colorize("WARNING: core size limit is set 0, core files "
116 "will NOT be created", RED))
117 cls.debug_core = True
120 elif dl == "gdbserver":
121 cls.debug_gdbserver = True
123 raise Exception("Unrecognized DEBUG option: '%s'" % d)
126 def setUpConstants(cls):
127 """ Set-up the test case class based on environment variables """
129 s = os.getenv("STEP")
130 cls.step = True if s.lower() in ("y", "yes", "1") else False
134 d = os.getenv("DEBUG")
137 cls.set_debug_flags(d)
138 cls.vpp_bin = os.getenv('VPP_TEST_BIN', "vpp")
139 cls.plugin_path = os.getenv('VPP_TEST_PLUGIN_PATH')
141 if cls.step or cls.debug_gdb or cls.debug_gdbserver:
142 debug_cli = "cli-listen localhost:5002"
143 cls.vpp_cmdline = [cls.vpp_bin,
144 "unix", "{", "nodaemon", debug_cli, "}",
145 "api-trace", "{", "on", "}",
146 "api-segment", "{", "prefix", cls.shm_prefix, "}"]
147 if cls.plugin_path is not None:
148 cls.vpp_cmdline.extend(["plugin_path", cls.plugin_path])
149 cls.logger.info("vpp_cmdline: %s" % cls.vpp_cmdline)
152 def wait_for_enter(cls):
153 if cls.debug_gdbserver:
154 print(double_line_delim)
155 print("Spawned GDB server with PID: %d" % cls.vpp.pid)
157 print(double_line_delim)
158 print("Spawned VPP with PID: %d" % cls.vpp.pid)
160 cls.logger.debug("Spawned VPP with PID: %d" % cls.vpp.pid)
162 print(single_line_delim)
163 print("You can debug the VPP using e.g.:")
164 if cls.debug_gdbserver:
165 print("gdb " + cls.vpp_bin + " -ex 'target remote localhost:7777'")
166 print("Now is the time to attach a gdb by running the above "
167 "command, set up breakpoints etc. and then resume VPP from "
168 "within gdb by issuing the 'continue' command")
170 print("gdb " + cls.vpp_bin + " -ex 'attach %s'" % cls.vpp.pid)
171 print("Now is the time to attach a gdb by running the above "
172 "command and set up breakpoints etc.")
173 print(single_line_delim)
174 raw_input("Press ENTER to continue running the testcase...")
178 cmdline = cls.vpp_cmdline
180 if cls.debug_gdbserver:
181 gdbserver = '/usr/bin/gdbserver'
182 if not os.path.isfile(gdbserver) or \
183 not os.access(gdbserver, os.X_OK):
184 raise Exception("gdbserver binary '%s' does not exist or is "
185 "not executable" % gdbserver)
187 cmdline = [gdbserver, 'localhost:7777'] + cls.vpp_cmdline
188 cls.logger.info("Gdbserver cmdline is %s", " ".join(cmdline))
191 cls.vpp = subprocess.Popen(cmdline,
192 stdout=subprocess.PIPE,
193 stderr=subprocess.PIPE,
195 except Exception as e:
196 cls.logger.critical("Couldn't start vpp: %s" % e)
204 Perform class setup before running the testcase
205 Remove shared memory files, start vpp and connect the vpp-api
207 gc.collect() # run garbage collection first
208 cls.logger = getLogger(cls.__name__)
209 cls.tempdir = tempfile.mkdtemp(
210 prefix='vpp-unittest-' + cls.__name__ + '-')
211 file_handler = FileHandler("%s/log.txt" % cls.tempdir)
212 file_handler.setFormatter(
213 Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s',
215 file_handler.setLevel(DEBUG)
216 cls.logger.addHandler(file_handler)
217 cls.shm_prefix = cls.tempdir.split("/")[-1]
218 os.chdir(cls.tempdir)
219 cls.logger.info("Temporary dir is %s, shm prefix is %s",
220 cls.tempdir, cls.shm_prefix)
222 cls.reset_packet_infos()
224 cls._zombie_captures = []
227 cls.registry = VppObjectRegistry()
228 print(double_line_delim)
229 print(colorize(getdoc(cls).splitlines()[0], YELLOW))
230 print(double_line_delim)
231 # need to catch exceptions here because if we raise, then the cleanup
232 # doesn't get called and we might end with a zombie vpp
235 cls.vpp_stdout_deque = deque()
236 cls.vpp_stderr_deque = deque()
237 cls.pump_thread_stop_flag = Event()
238 cls.pump_thread_wakeup_pipe = os.pipe()
239 cls.pump_thread = Thread(target=pump_output, args=(cls,))
240 cls.pump_thread.daemon = True
241 cls.pump_thread.start()
242 cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls)
247 cls.vapi.register_hook(hook)
248 cls.sleep(0.1, "after vpp startup, before initial poll")
253 if cls.debug_gdbserver:
254 print(colorize("You're running VPP inside gdbserver but "
255 "VPP-API connection failed, did you forget "
256 "to 'continue' VPP from within gdb?", RED))
259 t, v, tb = sys.exc_info()
269 Disconnect vpp-api, kill vpp and cleanup shared memory files
271 if (cls.debug_gdbserver or cls.debug_gdb) and hasattr(cls, 'vpp'):
273 if cls.vpp.returncode is None:
274 print(double_line_delim)
275 print("VPP or GDB server is still running")
276 print(single_line_delim)
277 raw_input("When done debugging, press ENTER to kill the "
278 "process and finish running the testcase...")
280 os.write(cls.pump_thread_wakeup_pipe[1], 'ding dong wake up')
281 cls.pump_thread_stop_flag.set()
282 if hasattr(cls, 'pump_thread'):
283 cls.logger.debug("Waiting for pump thread to stop")
284 cls.pump_thread.join()
285 if hasattr(cls, 'vpp_stderr_reader_thread'):
286 cls.logger.debug("Waiting for stdderr pump to stop")
287 cls.vpp_stderr_reader_thread.join()
289 if hasattr(cls, 'vpp'):
290 if hasattr(cls, 'vapi'):
291 cls.vapi.disconnect()
294 if cls.vpp.returncode is None:
295 cls.logger.debug("Sending TERM to vpp")
297 cls.logger.debug("Waiting for vpp to die")
298 cls.vpp.communicate()
301 if hasattr(cls, 'vpp_stdout_deque'):
302 cls.logger.info(single_line_delim)
303 cls.logger.info('VPP output to stdout while running %s:',
305 cls.logger.info(single_line_delim)
306 f = open(cls.tempdir + '/vpp_stdout.txt', 'w')
307 vpp_output = "".join(cls.vpp_stdout_deque)
309 cls.logger.info('\n%s', vpp_output)
310 cls.logger.info(single_line_delim)
312 if hasattr(cls, 'vpp_stderr_deque'):
313 cls.logger.info(single_line_delim)
314 cls.logger.info('VPP output to stderr while running %s:',
316 cls.logger.info(single_line_delim)
317 f = open(cls.tempdir + '/vpp_stderr.txt', 'w')
318 vpp_output = "".join(cls.vpp_stderr_deque)
320 cls.logger.info('\n%s', vpp_output)
321 cls.logger.info(single_line_delim)
324 def tearDownClass(cls):
325 """ Perform final cleanup after running all tests in this test-case """
329 """ Show various debug prints after each test """
330 self.logger.debug("--- tearDown() for %s.%s(%s) called ---" %
331 (self.__class__.__name__, self._testMethodName,
332 self._testMethodDoc))
333 if not self.vpp_dead:
334 self.logger.debug(self.vapi.cli("show trace"))
335 self.logger.info(self.vapi.ppcli("show int"))
336 self.logger.info(self.vapi.ppcli("show hardware"))
337 self.logger.info(self.vapi.ppcli("show error"))
338 self.logger.info(self.vapi.ppcli("show run"))
339 self.registry.remove_vpp_config(self.logger)
340 # Save/Dump VPP api trace log
341 api_trace = "vpp_api_trace.%s.log" % self._testMethodName
342 tmp_api_trace = "/tmp/%s" % api_trace
343 vpp_api_trace_log = "%s/%s" % (self.tempdir, api_trace)
344 self.logger.info(self.vapi.ppcli("api trace save %s" % api_trace))
345 self.logger.info("Moving %s to %s\n" % (tmp_api_trace,
347 os.rename(tmp_api_trace, vpp_api_trace_log)
348 self.logger.info(self.vapi.ppcli("api trace dump %s" %
352 """ Clear trace before running each test"""
353 self.logger.debug("--- setUp() for %s.%s(%s) called ---" %
354 (self.__class__.__name__, self._testMethodName,
355 self._testMethodDoc))
357 raise Exception("VPP is dead when setting up the test")
358 self.sleep(.1, "during setUp")
359 self.vpp_stdout_deque.append(
360 "--- test setUp() for %s.%s(%s) starts here ---\n" %
361 (self.__class__.__name__, self._testMethodName,
362 self._testMethodDoc))
363 self.vpp_stderr_deque.append(
364 "--- test setUp() for %s.%s(%s) starts here ---\n" %
365 (self.__class__.__name__, self._testMethodName,
366 self._testMethodDoc))
367 self.vapi.cli("clear trace")
368 # store the test instance inside the test class - so that objects
369 # holding the class can access instance methods (like assertEqual)
370 type(self).test_instance = self
373 def pg_enable_capture(cls, interfaces):
375 Enable capture on packet-generator interfaces
377 :param interfaces: iterable interface indexes
384 def register_capture(cls, cap_name):
385 """ Register a capture in the testclass """
386 # add to the list of captures with current timestamp
387 cls._captures.append((time.time(), cap_name))
388 # filter out from zombies
389 cls._zombie_captures = [(stamp, name)
390 for (stamp, name) in cls._zombie_captures
395 """ Remove any zombie captures and enable the packet generator """
396 # how long before capture is allowed to be deleted - otherwise vpp
397 # crashes - 100ms seems enough (this shouldn't be needed at all)
400 for stamp, cap_name in cls._zombie_captures:
401 wait = stamp + capture_ttl - now
403 cls.sleep(wait, "before deleting capture %s" % cap_name)
405 cls.logger.debug("Removing zombie capture %s" % cap_name)
406 cls.vapi.cli('packet-generator delete %s' % cap_name)
408 cls.vapi.cli("trace add pg-input 50") # 50 is maximum
409 cls.vapi.cli('packet-generator enable')
410 cls._zombie_captures = cls._captures
414 def create_pg_interfaces(cls, interfaces):
416 Create packet-generator interfaces.
418 :param interfaces: iterable indexes of the interfaces.
419 :returns: List of created interfaces.
424 intf = VppPGInterface(cls, i)
425 setattr(cls, intf.name, intf)
427 cls.pg_interfaces = result
431 def create_loopback_interfaces(cls, interfaces):
433 Create loopback interfaces.
435 :param interfaces: iterable indexes of the interfaces.
436 :returns: List of created interfaces.
440 intf = VppLoInterface(cls, i)
441 setattr(cls, intf.name, intf)
443 cls.lo_interfaces = result
447 def extend_packet(packet, size):
449 Extend packet to given size by padding with spaces
450 NOTE: Currently works only when Raw layer is present.
452 :param packet: packet
453 :param size: target size
456 packet_len = len(packet) + 4
457 extend = size - packet_len
459 packet[Raw].load += ' ' * extend
462 def reset_packet_infos(cls):
463 """ Reset the list of packet info objects and packet counts to zero """
464 cls._packet_infos = {}
465 cls._packet_count_for_dst_if_idx = {}
468 def create_packet_info(cls, src_if, dst_if):
470 Create packet info object containing the source and destination indexes
471 and add it to the testcase's packet info list
473 :param VppInterface src_if: source interface
474 :param VppInterface dst_if: destination interface
476 :returns: _PacketInfo object
480 info.index = len(cls._packet_infos)
481 info.src = src_if.sw_if_index
482 info.dst = dst_if.sw_if_index
483 if isinstance(dst_if, VppSubInterface):
484 dst_idx = dst_if.parent.sw_if_index
486 dst_idx = dst_if.sw_if_index
487 if dst_idx in cls._packet_count_for_dst_if_idx:
488 cls._packet_count_for_dst_if_idx[dst_idx] += 1
490 cls._packet_count_for_dst_if_idx[dst_idx] = 1
491 cls._packet_infos[info.index] = info
495 def info_to_payload(info):
497 Convert _PacketInfo object to packet payload
499 :param info: _PacketInfo object
501 :returns: string containing serialized data from packet info
503 return "%d %d %d" % (info.index, info.src, info.dst)
506 def payload_to_info(payload):
508 Convert packet payload to _PacketInfo object
510 :param payload: packet payload
512 :returns: _PacketInfo object containing de-serialized data from payload
515 numbers = payload.split()
517 info.index = int(numbers[0])
518 info.src = int(numbers[1])
519 info.dst = int(numbers[2])
522 def get_next_packet_info(self, info):
524 Iterate over the packet info list stored in the testcase
525 Start iteration with first element if info is None
526 Continue based on index in info if info is specified
528 :param info: info or None
529 :returns: next info in list or None if no more infos
534 next_index = info.index + 1
535 if next_index == len(self._packet_infos):
538 return self._packet_infos[next_index]
540 def get_next_packet_info_for_interface(self, src_index, info):
542 Search the packet info list for the next packet info with same source
545 :param src_index: source interface index to search for
546 :param info: packet info - where to start the search
547 :returns: packet info or None
551 info = self.get_next_packet_info(info)
554 if info.src == src_index:
557 def get_next_packet_info_for_interface2(self, src_index, dst_index, info):
559 Search the packet info list for the next packet info with same source
560 and destination interface indexes
562 :param src_index: source interface index to search for
563 :param dst_index: destination interface index to search for
564 :param info: packet info - where to start the search
565 :returns: packet info or None
569 info = self.get_next_packet_info_for_interface(src_index, info)
572 if info.dst == dst_index:
575 def assert_equal(self, real_value, expected_value, name_or_class=None):
576 if name_or_class is None:
577 self.assertEqual(real_value, expected_value)
580 msg = "Invalid %s: %d('%s') does not match expected value %d('%s')"
581 msg = msg % (getdoc(name_or_class).strip(),
582 real_value, str(name_or_class(real_value)),
583 expected_value, str(name_or_class(expected_value)))
585 msg = "Invalid %s: %s does not match expected value %s" % (
586 name_or_class, real_value, expected_value)
588 self.assertEqual(real_value, expected_value, msg)
590 def assert_in_range(self,
598 msg = "Invalid %s: %s out of range <%s,%s>" % (
599 name, real_value, expected_min, expected_max)
600 self.assertTrue(expected_min <= real_value <= expected_max, msg)
603 def sleep(cls, timeout, remark=None):
604 if hasattr(cls, 'logger'):
605 cls.logger.debug("Sleeping for %ss (%s)" % (timeout, remark))
609 class VppTestResult(unittest.TestResult):
611 @property result_string
612 String variable to store the test case result string.
614 List variable containing 2-tuples of TestCase instances and strings
615 holding formatted tracebacks. Each tuple represents a test which
616 raised an unexpected exception.
618 List variable containing 2-tuples of TestCase instances and strings
619 holding formatted tracebacks. Each tuple represents a test where
620 a failure was explicitly signalled using the TestCase.assert*()
624 def __init__(self, stream, descriptions, verbosity):
626 :param stream File descriptor to store where to report test results.
627 Set to the standard error stream by default.
628 :param descriptions Boolean variable to store information if to use
629 test case descriptions.
630 :param verbosity Integer variable to store required verbosity level.
632 unittest.TestResult.__init__(self, stream, descriptions, verbosity)
634 self.descriptions = descriptions
635 self.verbosity = verbosity
636 self.result_string = None
638 def addSuccess(self, test):
640 Record a test succeeded result
645 if hasattr(test, 'logger'):
646 test.logger.debug("--- addSuccess() %s.%s(%s) called"
647 % (test.__class__.__name__,
648 test._testMethodName,
649 test._testMethodDoc))
650 unittest.TestResult.addSuccess(self, test)
651 self.result_string = colorize("OK", GREEN)
653 def addSkip(self, test, reason):
655 Record a test skipped.
661 if hasattr(test, 'logger'):
662 test.logger.debug("--- addSkip() %s.%s(%s) called, reason is %s"
663 % (test.__class__.__name__,
664 test._testMethodName,
667 unittest.TestResult.addSkip(self, test, reason)
668 self.result_string = colorize("SKIP", YELLOW)
670 def addFailure(self, test, err):
672 Record a test failed result
675 :param err: error message
678 if hasattr(test, 'logger'):
679 test.logger.debug("--- addFailure() %s.%s(%s) called, err is %s"
680 % (test.__class__.__name__,
681 test._testMethodName,
682 test._testMethodDoc, err))
683 test.logger.debug("formatted exception is:\n%s" %
684 "".join(format_exception(*err)))
685 unittest.TestResult.addFailure(self, test, err)
686 if hasattr(test, 'tempdir'):
687 self.result_string = colorize("FAIL", RED) + \
688 ' [ temp dir used by test case: ' + test.tempdir + ' ]'
690 self.result_string = colorize("FAIL", RED) + ' [no temp dir]'
692 def addError(self, test, err):
694 Record a test error result
697 :param err: error message
700 if hasattr(test, 'logger'):
701 test.logger.debug("--- addError() %s.%s(%s) called, err is %s"
702 % (test.__class__.__name__,
703 test._testMethodName,
704 test._testMethodDoc, err))
705 test.logger.debug("formatted exception is:\n%s" %
706 "".join(format_exception(*err)))
707 unittest.TestResult.addError(self, test, err)
708 if hasattr(test, 'tempdir'):
709 self.result_string = colorize("ERROR", RED) + \
710 ' [ temp dir used by test case: ' + test.tempdir + ' ]'
712 self.result_string = colorize("ERROR", RED) + ' [no temp dir]'
714 def getDescription(self, test):
719 :returns: test description
722 # TODO: if none print warning not raise exception
723 short_description = test.shortDescription()
724 if self.descriptions and short_description:
725 return short_description
729 def startTest(self, test):
736 unittest.TestResult.startTest(self, test)
737 if self.verbosity > 0:
739 "Starting " + self.getDescription(test) + " ...")
740 self.stream.writeln(single_line_delim)
742 def stopTest(self, test):
749 unittest.TestResult.stopTest(self, test)
750 if self.verbosity > 0:
751 self.stream.writeln(single_line_delim)
752 self.stream.writeln("%-73s%s" % (self.getDescription(test),
754 self.stream.writeln(single_line_delim)
756 self.stream.writeln("%-73s%s" % (self.getDescription(test),
759 def printErrors(self):
761 Print errors from running the test case
763 self.stream.writeln()
764 self.printErrorList('ERROR', self.errors)
765 self.printErrorList('FAIL', self.failures)
767 def printErrorList(self, flavour, errors):
769 Print error list to the output stream together with error type
770 and test case description.
772 :param flavour: error type
773 :param errors: iterable errors
776 for test, err in errors:
777 self.stream.writeln(double_line_delim)
778 self.stream.writeln("%s: %s" %
779 (flavour, self.getDescription(test)))
780 self.stream.writeln(single_line_delim)
781 self.stream.writeln("%s" % err)
784 class VppTestRunner(unittest.TextTestRunner):
786 A basic test runner implementation which prints results to standard error.
789 def resultclass(self):
790 """Class maintaining the results of the tests"""
793 def __init__(self, stream=sys.stderr, descriptions=True, verbosity=1,
794 failfast=False, buffer=False, resultclass=None):
795 # ignore stream setting here, use hard-coded stdout to be in sync
796 # with prints from VppTestCase methods ...
797 super(VppTestRunner, self).__init__(sys.stdout, descriptions,
798 verbosity, failfast, buffer,
803 def parse_test_option(self):
805 f = os.getenv(self.test_option)
808 filter_file_name = None
809 filter_class_name = None
810 filter_func_name = None
815 raise Exception("Unrecognized %s option: %s" %
816 (self.test_option, f))
818 if parts[2] not in ('*', ''):
819 filter_func_name = parts[2]
820 if parts[1] not in ('*', ''):
821 filter_class_name = parts[1]
822 if parts[0] not in ('*', ''):
823 if parts[0].startswith('test_'):
824 filter_file_name = parts[0]
826 filter_file_name = 'test_%s' % parts[0]
828 if f.startswith('test_'):
831 filter_file_name = 'test_%s' % f
832 return filter_file_name, filter_class_name, filter_func_name
834 def filter_tests(self, tests, filter_file, filter_class, filter_func):
835 result = unittest.suite.TestSuite()
837 if isinstance(t, unittest.suite.TestSuite):
838 # this is a bunch of tests, recursively filter...
839 x = self.filter_tests(t, filter_file, filter_class,
841 if x.countTestCases() > 0:
843 elif isinstance(t, unittest.TestCase):
844 # this is a single test
845 parts = t.id().split('.')
846 # t.id() for common cases like this:
847 # test_classifier.TestClassifier.test_acl_ip
848 # apply filtering only if it is so
850 if filter_file and filter_file != parts[0]:
852 if filter_class and filter_class != parts[1]:
854 if filter_func and filter_func != parts[2]:
858 # unexpected object, don't touch it
869 gc.disable() # disable garbage collection, we'll do that manually
870 print("Running tests using custom test runner") # debug message
871 filter_file, filter_class, filter_func = self.parse_test_option()
872 print("Active filters: file=%s, class=%s, function=%s" % (
873 filter_file, filter_class, filter_func))
874 filtered = self.filter_tests(test, filter_file, filter_class,
876 print("%s out of %s tests match specified filters" % (
877 filtered.countTestCases(), test.countTestCases()))
878 return super(VppTestRunner, self).run(filtered)