3 from __future__ import print_function
15 from collections import deque
16 from threading import Thread, Event
17 from inspect import getdoc, isclass
18 from traceback import format_exception
19 from logging import FileHandler, DEBUG, Formatter
20 from scapy.packet import Raw
21 from hook import StepHook, PollHook, VppDiedError
22 from vpp_pg_interface import VppPGInterface
23 from vpp_sub_interface import VppSubInterface
24 from vpp_lo_interface import VppLoInterface
25 from vpp_papi_provider import VppPapiProvider
26 from vpp_papi.vpp_stats import VPPStats
27 from log import RED, GREEN, YELLOW, double_line_delim, single_line_delim, \
29 from vpp_object import VppObjectRegistry
30 from util import ppp, is_core_present
31 from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror
32 from scapy.layers.inet6 import ICMPv6DestUnreach, ICMPv6EchoRequest
33 from scapy.layers.inet6 import ICMPv6EchoReply
34 if os.name == 'posix' and sys.version_info[0] < 3:
35 # using subprocess32 is recommended by python official documentation
36 # @ https://docs.python.org/2/library/subprocess.html
37 import subprocess32 as subprocess
49 debug_framework = False
50 if os.getenv('TEST_DEBUG', "0") == "1":
51 debug_framework = True
56 Test framework module.
58 The module provides a set of tools for constructing and running tests and
59 representing the results.
63 class _PacketInfo(object):
64 """Private class to create packet info object.
66 Help process information about the next packet.
67 Set variables to default values.
69 #: Store the index of the packet.
71 #: Store the index of the source packet generator interface of the packet.
73 #: Store the index of the destination packet generator interface
76 #: Store expected ip version
78 #: Store expected upper protocol
80 #: Store the copy of the former packet.
83 def __eq__(self, other):
84 index = self.index == other.index
85 src = self.src == other.src
86 dst = self.dst == other.dst
87 data = self.data == other.data
88 return index and src and dst and data
91 def pump_output(testclass):
92 """ pump output from vpp stdout/stderr to proper queues """
95 while not testclass.pump_thread_stop_flag.is_set():
96 readable = select.select([testclass.vpp.stdout.fileno(),
97 testclass.vpp.stderr.fileno(),
98 testclass.pump_thread_wakeup_pipe[0]],
100 if testclass.vpp.stdout.fileno() in readable:
101 read = os.read(testclass.vpp.stdout.fileno(), 102400)
103 split = read.splitlines(True)
104 if len(stdout_fragment) > 0:
105 split[0] = "%s%s" % (stdout_fragment, split[0])
106 if len(split) > 0 and split[-1].endswith("\n"):
110 stdout_fragment = split[-1]
111 testclass.vpp_stdout_deque.extend(split[:limit])
112 if not testclass.cache_vpp_output:
113 for line in split[:limit]:
114 testclass.logger.debug(
115 "VPP STDOUT: %s" % line.rstrip("\n"))
116 if testclass.vpp.stderr.fileno() in readable:
117 read = os.read(testclass.vpp.stderr.fileno(), 102400)
119 split = read.splitlines(True)
120 if len(stderr_fragment) > 0:
121 split[0] = "%s%s" % (stderr_fragment, split[0])
122 if len(split) > 0 and split[-1].endswith("\n"):
126 stderr_fragment = split[-1]
127 testclass.vpp_stderr_deque.extend(split[:limit])
128 if not testclass.cache_vpp_output:
129 for line in split[:limit]:
130 testclass.logger.debug(
131 "VPP STDERR: %s" % line.rstrip("\n"))
132 # ignoring the dummy pipe here intentionally - the flag will take care
133 # of properly terminating the loop
136 def running_extended_tests():
137 s = os.getenv("EXTENDED_TESTS", "n")
138 return True if s.lower() in ("y", "yes", "1") else False
141 def running_on_centos():
142 os_id = os.getenv("OS_ID", "")
143 return True if "centos" in os_id.lower() else False
146 class KeepAliveReporter(object):
148 Singleton object which reports test start to parent process
153 self.__dict__ = self._shared_state
160 def pipe(self, pipe):
161 if hasattr(self, '_pipe'):
162 raise Exception("Internal error - pipe should only be set once.")
165 def send_keep_alive(self, test, desc=None):
167 Write current test tmpdir & desc to keep-alive pipe to signal liveness
169 if self.pipe is None:
170 # if not running forked..
174 desc = '%s (%s)' % (desc, unittest.util.strclass(test))
178 self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid))
181 class VppTestCase(unittest.TestCase):
182 """This subclass is a base class for VPP test cases that are implemented as
183 classes. It provides methods to create and run test case.
187 def packet_infos(self):
188 """List of packet infos"""
189 return self._packet_infos
192 def get_packet_count_for_if_idx(cls, dst_if_index):
193 """Get the number of packet info for specified destination if index"""
194 if dst_if_index in cls._packet_count_for_dst_if_idx:
195 return cls._packet_count_for_dst_if_idx[dst_if_index]
201 """Return the instance of this testcase"""
202 return cls.test_instance
205 def set_debug_flags(cls, d):
206 cls.debug_core = False
207 cls.debug_gdb = False
208 cls.debug_gdbserver = False
213 cls.debug_core = True
216 elif dl == "gdbserver":
217 cls.debug_gdbserver = True
219 raise Exception("Unrecognized DEBUG option: '%s'" % d)
222 def get_least_used_cpu(self):
223 cpu_usage_list = [set(range(psutil.cpu_count()))]
224 vpp_processes = [p for p in psutil.process_iter(attrs=['pid', 'name'])
225 if 'vpp_main' == p.info['name']]
226 for vpp_process in vpp_processes:
227 for cpu_usage_set in cpu_usage_list:
229 cpu_num = vpp_process.cpu_num()
230 if cpu_num in cpu_usage_set:
231 cpu_usage_set_index = cpu_usage_list.index(
233 if cpu_usage_set_index == len(cpu_usage_list) - 1:
234 cpu_usage_list.append({cpu_num})
236 cpu_usage_list[cpu_usage_set_index + 1].add(
238 cpu_usage_set.remove(cpu_num)
240 except psutil.NoSuchProcess:
243 for cpu_usage_set in cpu_usage_list:
244 if len(cpu_usage_set) > 0:
245 min_usage_set = cpu_usage_set
248 return random.choice(tuple(min_usage_set))
251 def print_header(cls):
252 if not hasattr(cls, '_header_printed'):
253 print(double_line_delim)
254 print(colorize(getdoc(cls).splitlines()[0], GREEN))
255 print(double_line_delim)
256 cls._header_printed = True
259 def setUpConstants(cls):
260 """ Set-up the test case class based on environment variables """
261 s = os.getenv("STEP", "n")
262 cls.step = True if s.lower() in ("y", "yes", "1") else False
263 d = os.getenv("DEBUG", None)
264 c = os.getenv("CACHE_OUTPUT", "1")
265 cls.cache_vpp_output = False if c.lower() in ("n", "no", "0") else True
266 cls.set_debug_flags(d)
267 cls.vpp_bin = os.getenv('VPP_TEST_BIN', "vpp")
268 cls.plugin_path = os.getenv('VPP_TEST_PLUGIN_PATH')
269 cls.extern_plugin_path = os.getenv('EXTERN_PLUGINS')
271 if cls.plugin_path is not None:
272 if cls.extern_plugin_path is not None:
273 plugin_path = "%s:%s" % (
274 cls.plugin_path, cls.extern_plugin_path)
276 plugin_path = cls.plugin_path
277 elif cls.extern_plugin_path is not None:
278 plugin_path = cls.extern_plugin_path
280 if cls.step or cls.debug_gdb or cls.debug_gdbserver:
281 debug_cli = "cli-listen localhost:5002"
283 size = os.getenv("COREDUMP_SIZE")
285 coredump_size = "coredump-size %s" % size
286 if coredump_size is None:
287 coredump_size = "coredump-size unlimited"
289 cpu_core_number = cls.get_least_used_cpu()
291 cls.vpp_cmdline = [cls.vpp_bin, "unix",
292 "{", "nodaemon", debug_cli, "full-coredump",
293 coredump_size, "runtime-dir", cls.tempdir, "}",
294 "api-trace", "{", "on", "}", "api-segment", "{",
295 "prefix", cls.shm_prefix, "}", "cpu", "{",
296 "main-core", str(cpu_core_number), "}", "statseg",
297 "{", "socket-name", cls.stats_sock, "}", "plugins",
298 "{", "plugin", "dpdk_plugin.so", "{", "disable",
299 "}", "plugin", "unittest_plugin.so", "{", "enable",
301 if plugin_path is not None:
302 cls.vpp_cmdline.extend(["plugin_path", plugin_path])
303 cls.logger.info("vpp_cmdline: %s" % cls.vpp_cmdline)
306 def wait_for_enter(cls):
307 if cls.debug_gdbserver:
308 print(double_line_delim)
309 print("Spawned GDB server with PID: %d" % cls.vpp.pid)
311 print(double_line_delim)
312 print("Spawned VPP with PID: %d" % cls.vpp.pid)
314 cls.logger.debug("Spawned VPP with PID: %d" % cls.vpp.pid)
316 print(single_line_delim)
317 print("You can debug the VPP using e.g.:")
318 if cls.debug_gdbserver:
319 print("gdb " + cls.vpp_bin + " -ex 'target remote localhost:7777'")
320 print("Now is the time to attach a gdb by running the above "
321 "command, set up breakpoints etc. and then resume VPP from "
322 "within gdb by issuing the 'continue' command")
324 print("gdb " + cls.vpp_bin + " -ex 'attach %s'" % cls.vpp.pid)
325 print("Now is the time to attach a gdb by running the above "
326 "command and set up breakpoints etc.")
327 print(single_line_delim)
328 raw_input("Press ENTER to continue running the testcase...")
332 cmdline = cls.vpp_cmdline
334 if cls.debug_gdbserver:
335 gdbserver = '/usr/bin/gdbserver'
336 if not os.path.isfile(gdbserver) or \
337 not os.access(gdbserver, os.X_OK):
338 raise Exception("gdbserver binary '%s' does not exist or is "
339 "not executable" % gdbserver)
341 cmdline = [gdbserver, 'localhost:7777'] + cls.vpp_cmdline
342 cls.logger.info("Gdbserver cmdline is %s", " ".join(cmdline))
345 cls.vpp = subprocess.Popen(cmdline,
346 stdout=subprocess.PIPE,
347 stderr=subprocess.PIPE,
349 except Exception as e:
350 cls.logger.critical("Couldn't start vpp: %s" % e)
356 def wait_for_stats_socket(cls):
357 deadline = time.time() + 3
358 while time.time() < deadline or cls.debug_gdb or cls.debug_gdbserver:
359 if os.path.exists(cls.stats_sock):
365 Perform class setup before running the testcase
366 Remove shared memory files, start vpp and connect the vpp-api
368 gc.collect() # run garbage collection first
370 cls.print_header(cls)
371 if not hasattr(cls, 'logger'):
372 cls.logger = getLogger(cls.__name__)
374 cls.logger.name = cls.__name__
375 cls.tempdir = tempfile.mkdtemp(
376 prefix='vpp-unittest-%s-' % cls.__name__)
377 cls.stats_sock = "%s/stats.sock" % cls.tempdir
378 cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir)
379 cls.file_handler.setFormatter(
380 Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s',
382 cls.file_handler.setLevel(DEBUG)
383 cls.logger.addHandler(cls.file_handler)
384 cls.shm_prefix = os.path.basename(cls.tempdir)
385 os.chdir(cls.tempdir)
386 cls.logger.info("Temporary dir is %s, shm prefix is %s",
387 cls.tempdir, cls.shm_prefix)
389 cls.reset_packet_infos()
391 cls._zombie_captures = []
394 cls.registry = VppObjectRegistry()
395 cls.vpp_startup_failed = False
396 cls.reporter = KeepAliveReporter()
397 # need to catch exceptions here because if we raise, then the cleanup
398 # doesn't get called and we might end with a zombie vpp
401 cls.reporter.send_keep_alive(cls, 'setUpClass')
402 VppTestResult.current_test_case_info = TestCaseInfo(
403 cls.logger, cls.tempdir, cls.vpp.pid, cls.vpp_bin)
404 cls.vpp_stdout_deque = deque()
405 cls.vpp_stderr_deque = deque()
406 cls.pump_thread_stop_flag = Event()
407 cls.pump_thread_wakeup_pipe = os.pipe()
408 cls.pump_thread = Thread(target=pump_output, args=(cls,))
409 cls.pump_thread.daemon = True
410 cls.pump_thread.start()
411 if cls.debug_gdb or cls.debug_gdbserver:
415 cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls,
421 cls.vapi.register_hook(hook)
422 cls.wait_for_stats_socket()
423 cls.statistics = VPPStats(socketname=cls.stats_sock)
427 cls.vpp_startup_failed = True
429 "VPP died shortly after startup, check the"
430 " output to standard error for possible cause")
436 cls.vapi.disconnect()
439 if cls.debug_gdbserver:
440 print(colorize("You're running VPP inside gdbserver but "
441 "VPP-API connection failed, did you forget "
442 "to 'continue' VPP from within gdb?", RED))
454 Disconnect vpp-api, kill vpp and cleanup shared memory files
456 if (cls.debug_gdbserver or cls.debug_gdb) and hasattr(cls, 'vpp'):
458 if cls.vpp.returncode is None:
459 print(double_line_delim)
460 print("VPP or GDB server is still running")
461 print(single_line_delim)
462 raw_input("When done debugging, press ENTER to kill the "
463 "process and finish running the testcase...")
465 # first signal that we want to stop the pump thread, then wake it up
466 if hasattr(cls, 'pump_thread_stop_flag'):
467 cls.pump_thread_stop_flag.set()
468 if hasattr(cls, 'pump_thread_wakeup_pipe'):
469 os.write(cls.pump_thread_wakeup_pipe[1], 'ding dong wake up')
470 if hasattr(cls, 'pump_thread'):
471 cls.logger.debug("Waiting for pump thread to stop")
472 cls.pump_thread.join()
473 if hasattr(cls, 'vpp_stderr_reader_thread'):
474 cls.logger.debug("Waiting for stdderr pump to stop")
475 cls.vpp_stderr_reader_thread.join()
477 if hasattr(cls, 'vpp'):
478 if hasattr(cls, 'vapi'):
479 cls.vapi.disconnect()
482 if cls.vpp.returncode is None:
483 cls.logger.debug("Sending TERM to vpp")
485 cls.logger.debug("Waiting for vpp to die")
486 cls.vpp.communicate()
489 if cls.vpp_startup_failed:
490 stdout_log = cls.logger.info
491 stderr_log = cls.logger.critical
493 stdout_log = cls.logger.info
494 stderr_log = cls.logger.info
496 if hasattr(cls, 'vpp_stdout_deque'):
497 stdout_log(single_line_delim)
498 stdout_log('VPP output to stdout while running %s:', cls.__name__)
499 stdout_log(single_line_delim)
500 vpp_output = "".join(cls.vpp_stdout_deque)
501 with open(cls.tempdir + '/vpp_stdout.txt', 'w') as f:
503 stdout_log('\n%s', vpp_output)
504 stdout_log(single_line_delim)
506 if hasattr(cls, 'vpp_stderr_deque'):
507 stderr_log(single_line_delim)
508 stderr_log('VPP output to stderr while running %s:', cls.__name__)
509 stderr_log(single_line_delim)
510 vpp_output = "".join(cls.vpp_stderr_deque)
511 with open(cls.tempdir + '/vpp_stderr.txt', 'w') as f:
513 stderr_log('\n%s', vpp_output)
514 stderr_log(single_line_delim)
517 def tearDownClass(cls):
518 """ Perform final cleanup after running all tests in this test-case """
519 cls.reporter.send_keep_alive(cls, 'tearDownClass')
521 cls.file_handler.close()
522 cls.reset_packet_infos()
524 debug_internal.on_tear_down_class(cls)
527 """ Show various debug prints after each test """
528 self.logger.debug("--- tearDown() for %s.%s(%s) called ---" %
529 (self.__class__.__name__, self._testMethodName,
530 self._testMethodDoc))
531 if not self.vpp_dead:
532 self.logger.debug(self.vapi.cli("show trace"))
533 self.logger.info(self.vapi.ppcli("show interface"))
534 self.logger.info(self.vapi.ppcli("show hardware"))
535 self.logger.info(self.statistics.set_errors_str())
536 self.logger.info(self.vapi.ppcli("show run"))
537 self.logger.info(self.vapi.ppcli("show log"))
538 self.registry.remove_vpp_config(self.logger)
539 # Save/Dump VPP api trace log
540 api_trace = "vpp_api_trace.%s.log" % self._testMethodName
541 tmp_api_trace = "/tmp/%s" % api_trace
542 vpp_api_trace_log = "%s/%s" % (self.tempdir, api_trace)
543 self.logger.info(self.vapi.ppcli("api trace save %s" % api_trace))
544 self.logger.info("Moving %s to %s\n" % (tmp_api_trace,
546 os.rename(tmp_api_trace, vpp_api_trace_log)
547 self.logger.info(self.vapi.ppcli("api trace custom-dump %s" %
550 self.registry.unregister_all(self.logger)
553 """ Clear trace before running each test"""
554 self.reporter.send_keep_alive(self)
555 self.logger.debug("--- setUp() for %s.%s(%s) called ---" %
556 (self.__class__.__name__, self._testMethodName,
557 self._testMethodDoc))
559 raise Exception("VPP is dead when setting up the test")
560 self.sleep(.1, "during setUp")
561 self.vpp_stdout_deque.append(
562 "--- test setUp() for %s.%s(%s) starts here ---\n" %
563 (self.__class__.__name__, self._testMethodName,
564 self._testMethodDoc))
565 self.vpp_stderr_deque.append(
566 "--- test setUp() for %s.%s(%s) starts here ---\n" %
567 (self.__class__.__name__, self._testMethodName,
568 self._testMethodDoc))
569 self.vapi.cli("clear trace")
570 # store the test instance inside the test class - so that objects
571 # holding the class can access instance methods (like assertEqual)
572 type(self).test_instance = self
575 def pg_enable_capture(cls, interfaces=None):
577 Enable capture on packet-generator interfaces
579 :param interfaces: iterable interface indexes (if None,
580 use self.pg_interfaces)
583 if interfaces is None:
584 interfaces = cls.pg_interfaces
589 def register_capture(cls, cap_name):
590 """ Register a capture in the testclass """
591 # add to the list of captures with current timestamp
592 cls._captures.append((time.time(), cap_name))
593 # filter out from zombies
594 cls._zombie_captures = [(stamp, name)
595 for (stamp, name) in cls._zombie_captures
600 """ Remove any zombie captures and enable the packet generator """
601 # how long before capture is allowed to be deleted - otherwise vpp
602 # crashes - 100ms seems enough (this shouldn't be needed at all)
605 for stamp, cap_name in cls._zombie_captures:
606 wait = stamp + capture_ttl - now
608 cls.sleep(wait, "before deleting capture %s" % cap_name)
610 cls.logger.debug("Removing zombie capture %s" % cap_name)
611 cls.vapi.cli('packet-generator delete %s' % cap_name)
613 cls.vapi.cli("trace add pg-input 50") # 50 is maximum
614 cls.vapi.cli('packet-generator enable')
615 cls._zombie_captures = cls._captures
619 def create_pg_interfaces(cls, interfaces):
621 Create packet-generator interfaces.
623 :param interfaces: iterable indexes of the interfaces.
624 :returns: List of created interfaces.
629 intf = VppPGInterface(cls, i)
630 setattr(cls, intf.name, intf)
632 cls.pg_interfaces = result
636 def create_loopback_interfaces(cls, count):
638 Create loopback interfaces.
640 :param count: number of interfaces created.
641 :returns: List of created interfaces.
643 result = [VppLoInterface(cls) for i in range(count)]
645 setattr(cls, intf.name, intf)
646 cls.lo_interfaces = result
650 def extend_packet(packet, size, padding=' '):
652 Extend packet to given size by padding with spaces or custom padding
653 NOTE: Currently works only when Raw layer is present.
655 :param packet: packet
656 :param size: target size
657 :param padding: padding used to extend the payload
660 packet_len = len(packet) + 4
661 extend = size - packet_len
663 num = (extend / len(padding)) + 1
664 packet[Raw].load += (padding * num)[:extend]
667 def reset_packet_infos(cls):
668 """ Reset the list of packet info objects and packet counts to zero """
669 cls._packet_infos = {}
670 cls._packet_count_for_dst_if_idx = {}
673 def create_packet_info(cls, src_if, dst_if):
675 Create packet info object containing the source and destination indexes
676 and add it to the testcase's packet info list
678 :param VppInterface src_if: source interface
679 :param VppInterface dst_if: destination interface
681 :returns: _PacketInfo object
685 info.index = len(cls._packet_infos)
686 info.src = src_if.sw_if_index
687 info.dst = dst_if.sw_if_index
688 if isinstance(dst_if, VppSubInterface):
689 dst_idx = dst_if.parent.sw_if_index
691 dst_idx = dst_if.sw_if_index
692 if dst_idx in cls._packet_count_for_dst_if_idx:
693 cls._packet_count_for_dst_if_idx[dst_idx] += 1
695 cls._packet_count_for_dst_if_idx[dst_idx] = 1
696 cls._packet_infos[info.index] = info
700 def info_to_payload(info):
702 Convert _PacketInfo object to packet payload
704 :param info: _PacketInfo object
706 :returns: string containing serialized data from packet info
708 return "%d %d %d %d %d" % (info.index, info.src, info.dst,
712 def payload_to_info(payload):
714 Convert packet payload to _PacketInfo object
716 :param payload: packet payload
718 :returns: _PacketInfo object containing de-serialized data from payload
721 numbers = payload.split()
723 info.index = int(numbers[0])
724 info.src = int(numbers[1])
725 info.dst = int(numbers[2])
726 info.ip = int(numbers[3])
727 info.proto = int(numbers[4])
730 def get_next_packet_info(self, info):
732 Iterate over the packet info list stored in the testcase
733 Start iteration with first element if info is None
734 Continue based on index in info if info is specified
736 :param info: info or None
737 :returns: next info in list or None if no more infos
742 next_index = info.index + 1
743 if next_index == len(self._packet_infos):
746 return self._packet_infos[next_index]
748 def get_next_packet_info_for_interface(self, src_index, info):
750 Search the packet info list for the next packet info with same source
753 :param src_index: source interface index to search for
754 :param info: packet info - where to start the search
755 :returns: packet info or None
759 info = self.get_next_packet_info(info)
762 if info.src == src_index:
765 def get_next_packet_info_for_interface2(self, src_index, dst_index, info):
767 Search the packet info list for the next packet info with same source
768 and destination interface indexes
770 :param src_index: source interface index to search for
771 :param dst_index: destination interface index to search for
772 :param info: packet info - where to start the search
773 :returns: packet info or None
777 info = self.get_next_packet_info_for_interface(src_index, info)
780 if info.dst == dst_index:
783 def assert_equal(self, real_value, expected_value, name_or_class=None):
784 if name_or_class is None:
785 self.assertEqual(real_value, expected_value)
788 msg = "Invalid %s: %d('%s') does not match expected value %d('%s')"
789 msg = msg % (getdoc(name_or_class).strip(),
790 real_value, str(name_or_class(real_value)),
791 expected_value, str(name_or_class(expected_value)))
793 msg = "Invalid %s: %s does not match expected value %s" % (
794 name_or_class, real_value, expected_value)
796 self.assertEqual(real_value, expected_value, msg)
798 def assert_in_range(self,
806 msg = "Invalid %s: %s out of range <%s,%s>" % (
807 name, real_value, expected_min, expected_max)
808 self.assertTrue(expected_min <= real_value <= expected_max, msg)
810 def assert_packet_checksums_valid(self, packet,
811 ignore_zero_udp_checksums=True):
812 received = packet.__class__(str(packet))
814 ppp("Verifying packet checksums for packet:", received))
815 udp_layers = ['UDP', 'UDPerror']
816 checksum_fields = ['cksum', 'chksum']
819 temp = received.__class__(str(received))
821 layer = temp.getlayer(counter)
823 for cf in checksum_fields:
824 if hasattr(layer, cf):
825 if ignore_zero_udp_checksums and \
826 0 == getattr(layer, cf) and \
827 layer.name in udp_layers:
830 checksums.append((counter, cf))
833 counter = counter + 1
834 if 0 == len(checksums):
836 temp = temp.__class__(str(temp))
837 for layer, cf in checksums:
838 calc_sum = getattr(temp[layer], cf)
840 getattr(received[layer], cf), calc_sum,
841 "packet checksum on layer #%d: %s" % (layer, temp[layer].name))
843 "Checksum field `%s` on `%s` layer has correct value `%s`" %
844 (cf, temp[layer].name, calc_sum))
846 def assert_checksum_valid(self, received_packet, layer,
848 ignore_zero_checksum=False):
849 """ Check checksum of received packet on given layer """
850 received_packet_checksum = getattr(received_packet[layer], field_name)
851 if ignore_zero_checksum and 0 == received_packet_checksum:
853 recalculated = received_packet.__class__(str(received_packet))
854 delattr(recalculated[layer], field_name)
855 recalculated = recalculated.__class__(str(recalculated))
856 self.assert_equal(received_packet_checksum,
857 getattr(recalculated[layer], field_name),
858 "packet checksum on layer: %s" % layer)
860 def assert_ip_checksum_valid(self, received_packet,
861 ignore_zero_checksum=False):
862 self.assert_checksum_valid(received_packet, 'IP',
863 ignore_zero_checksum=ignore_zero_checksum)
865 def assert_tcp_checksum_valid(self, received_packet,
866 ignore_zero_checksum=False):
867 self.assert_checksum_valid(received_packet, 'TCP',
868 ignore_zero_checksum=ignore_zero_checksum)
870 def assert_udp_checksum_valid(self, received_packet,
871 ignore_zero_checksum=True):
872 self.assert_checksum_valid(received_packet, 'UDP',
873 ignore_zero_checksum=ignore_zero_checksum)
875 def assert_embedded_icmp_checksum_valid(self, received_packet):
876 if received_packet.haslayer(IPerror):
877 self.assert_checksum_valid(received_packet, 'IPerror')
878 if received_packet.haslayer(TCPerror):
879 self.assert_checksum_valid(received_packet, 'TCPerror')
880 if received_packet.haslayer(UDPerror):
881 self.assert_checksum_valid(received_packet, 'UDPerror',
882 ignore_zero_checksum=True)
883 if received_packet.haslayer(ICMPerror):
884 self.assert_checksum_valid(received_packet, 'ICMPerror')
886 def assert_icmp_checksum_valid(self, received_packet):
887 self.assert_checksum_valid(received_packet, 'ICMP')
888 self.assert_embedded_icmp_checksum_valid(received_packet)
890 def assert_icmpv6_checksum_valid(self, pkt):
891 if pkt.haslayer(ICMPv6DestUnreach):
892 self.assert_checksum_valid(pkt, 'ICMPv6DestUnreach', 'cksum')
893 self.assert_embedded_icmp_checksum_valid(pkt)
894 if pkt.haslayer(ICMPv6EchoRequest):
895 self.assert_checksum_valid(pkt, 'ICMPv6EchoRequest', 'cksum')
896 if pkt.haslayer(ICMPv6EchoReply):
897 self.assert_checksum_valid(pkt, 'ICMPv6EchoReply', 'cksum')
900 def sleep(cls, timeout, remark=None):
901 if hasattr(cls, 'logger'):
902 cls.logger.debug("Starting sleep for %ss (%s)" % (timeout, remark))
906 if after - before > 2 * timeout:
907 cls.logger.error("unexpected time.sleep() result - "
908 "slept for %ss instead of ~%ss!" % (
909 after - before, timeout))
910 if hasattr(cls, 'logger'):
912 "Finished sleep (%s) - slept %ss (wanted %ss)" % (
913 remark, after - before, timeout))
915 def send_and_assert_no_replies(self, intf, pkts, remark="", timeout=None):
916 self.vapi.cli("clear trace")
917 intf.add_stream(pkts)
918 self.pg_enable_capture(self.pg_interfaces)
922 for i in self.pg_interfaces:
923 i.get_capture(0, timeout=timeout)
924 i.assert_nothing_captured(remark=remark)
927 def send_and_expect(self, input, pkts, output):
928 self.vapi.cli("clear trace")
929 input.add_stream(pkts)
930 self.pg_enable_capture(self.pg_interfaces)
932 rx = output.get_capture(len(pkts))
936 def get_testcase_doc_name(test):
937 return getdoc(test.__class__).splitlines()[0]
940 def get_test_description(descriptions, test):
941 short_description = test.shortDescription()
942 if descriptions and short_description:
943 return short_description
948 class TestCaseInfo(object):
949 def __init__(self, logger, tempdir, vpp_pid, vpp_bin_path):
951 self.tempdir = tempdir
952 self.vpp_pid = vpp_pid
953 self.vpp_bin_path = vpp_bin_path
954 self.core_crash_test = None
957 class VppTestResult(unittest.TestResult):
959 @property result_string
960 String variable to store the test case result string.
962 List variable containing 2-tuples of TestCase instances and strings
963 holding formatted tracebacks. Each tuple represents a test which
964 raised an unexpected exception.
966 List variable containing 2-tuples of TestCase instances and strings
967 holding formatted tracebacks. Each tuple represents a test where
968 a failure was explicitly signalled using the TestCase.assert*()
972 failed_test_cases_info = set()
973 core_crash_test_cases_info = set()
974 current_test_case_info = None
976 def __init__(self, stream, descriptions, verbosity):
978 :param stream File descriptor to store where to report test results.
979 Set to the standard error stream by default.
980 :param descriptions Boolean variable to store information if to use
981 test case descriptions.
982 :param verbosity Integer variable to store required verbosity level.
984 unittest.TestResult.__init__(self, stream, descriptions, verbosity)
986 self.descriptions = descriptions
987 self.verbosity = verbosity
988 self.result_string = None
990 def addSuccess(self, test):
992 Record a test succeeded result
997 if self.current_test_case_info:
998 self.current_test_case_info.logger.debug(
999 "--- addSuccess() %s.%s(%s) called" % (test.__class__.__name__,
1000 test._testMethodName,
1001 test._testMethodDoc))
1002 unittest.TestResult.addSuccess(self, test)
1003 self.result_string = colorize("OK", GREEN)
1005 self.send_result_through_pipe(test, PASS)
1007 def addSkip(self, test, reason):
1009 Record a test skipped.
1015 if self.current_test_case_info:
1016 self.current_test_case_info.logger.debug(
1017 "--- addSkip() %s.%s(%s) called, reason is %s" %
1018 (test.__class__.__name__, test._testMethodName,
1019 test._testMethodDoc, reason))
1020 unittest.TestResult.addSkip(self, test, reason)
1021 self.result_string = colorize("SKIP", YELLOW)
1023 self.send_result_through_pipe(test, SKIP)
1025 def symlink_failed(self):
1026 if self.current_test_case_info:
1028 failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
1029 link_path = os.path.join(
1032 os.path.basename(self.current_test_case_info.tempdir))
1033 if self.current_test_case_info.logger:
1034 self.current_test_case_info.logger.debug(
1035 "creating a link to the failed test")
1036 self.current_test_case_info.logger.debug(
1037 "os.symlink(%s, %s)" %
1038 (self.current_test_case_info.tempdir, link_path))
1039 if os.path.exists(link_path):
1040 if self.current_test_case_info.logger:
1041 self.current_test_case_info.logger.debug(
1042 'symlink already exists')
1044 os.symlink(self.current_test_case_info.tempdir, link_path)
1046 except Exception as e:
1047 if self.current_test_case_info.logger:
1048 self.current_test_case_info.logger.error(e)
1050 def send_result_through_pipe(self, test, result):
1051 if hasattr(self, 'test_framework_result_pipe'):
1052 pipe = self.test_framework_result_pipe
1054 pipe.send((test.id(), result))
1056 def log_error(self, test, err, fn_name):
1057 if self.current_test_case_info:
1058 if isinstance(test, unittest.suite._ErrorHolder):
1059 test_name = test.description
1061 test_name = '%s.%s(%s)' % (test.__class__.__name__,
1062 test._testMethodName,
1063 test._testMethodDoc)
1064 self.current_test_case_info.logger.debug(
1065 "--- %s() %s called, err is %s" %
1066 (fn_name, test_name, err))
1067 self.current_test_case_info.logger.debug(
1068 "formatted exception is:\n%s" %
1069 "".join(format_exception(*err)))
1071 def add_error(self, test, err, unittest_fn, error_type):
1072 if error_type == FAIL:
1073 self.log_error(test, err, 'addFailure')
1074 error_type_str = colorize("FAIL", RED)
1075 elif error_type == ERROR:
1076 self.log_error(test, err, 'addError')
1077 error_type_str = colorize("ERROR", RED)
1079 raise Exception('Error type %s cannot be used to record an '
1080 'error or a failure' % error_type)
1082 unittest_fn(self, test, err)
1083 if self.current_test_case_info:
1084 self.result_string = "%s [ temp dir used by test case: %s ]" % \
1086 self.current_test_case_info.tempdir)
1087 self.symlink_failed()
1088 self.failed_test_cases_info.add(self.current_test_case_info)
1089 if is_core_present(self.current_test_case_info.tempdir):
1090 if not self.current_test_case_info.core_crash_test:
1091 if isinstance(test, unittest.suite._ErrorHolder):
1092 test_name = str(test)
1094 test_name = "'{}' ({})".format(
1095 get_testcase_doc_name(test), test.id())
1096 self.current_test_case_info.core_crash_test = test_name
1097 self.core_crash_test_cases_info.add(
1098 self.current_test_case_info)
1100 self.result_string = '%s [no temp dir]' % error_type_str
1102 self.send_result_through_pipe(test, error_type)
1104 def addFailure(self, test, err):
1106 Record a test failed result
1109 :param err: error message
1112 self.add_error(test, err, unittest.TestResult.addFailure, FAIL)
1114 def addError(self, test, err):
1116 Record a test error result
1119 :param err: error message
1122 self.add_error(test, err, unittest.TestResult.addError, ERROR)
1124 def getDescription(self, test):
1126 Get test description
1129 :returns: test description
1132 return get_test_description(self.descriptions, test)
1134 def startTest(self, test):
1141 test.print_header(test.__class__)
1143 unittest.TestResult.startTest(self, test)
1144 if self.verbosity > 0:
1145 self.stream.writeln(
1146 "Starting " + self.getDescription(test) + " ...")
1147 self.stream.writeln(single_line_delim)
1149 def stopTest(self, test):
1151 Called when the given test has been run
1156 unittest.TestResult.stopTest(self, test)
1157 if self.verbosity > 0:
1158 self.stream.writeln(single_line_delim)
1159 self.stream.writeln("%-73s%s" % (self.getDescription(test),
1160 self.result_string))
1161 self.stream.writeln(single_line_delim)
1163 self.stream.writeln("%-73s%s" % (self.getDescription(test),
1164 self.result_string))
1166 self.send_result_through_pipe(test, TEST_RUN)
1168 def printErrors(self):
1170 Print errors from running the test case
1172 self.stream.writeln()
1173 self.printErrorList('ERROR', self.errors)
1174 self.printErrorList('FAIL', self.failures)
1176 def printErrorList(self, flavour, errors):
1178 Print error list to the output stream together with error type
1179 and test case description.
1181 :param flavour: error type
1182 :param errors: iterable errors
1185 for test, err in errors:
1186 self.stream.writeln(double_line_delim)
1187 self.stream.writeln("%s: %s" %
1188 (flavour, self.getDescription(test)))
1189 self.stream.writeln(single_line_delim)
1190 self.stream.writeln("%s" % err)
1193 class VppTestRunner(unittest.TextTestRunner):
1195 A basic test runner implementation which prints results to standard error.
1198 def resultclass(self):
1199 """Class maintaining the results of the tests"""
1200 return VppTestResult
1202 def __init__(self, keep_alive_pipe=None, descriptions=True, verbosity=1,
1203 result_pipe=None, failfast=False, buffer=False,
1206 # ignore stream setting here, use hard-coded stdout to be in sync
1207 # with prints from VppTestCase methods ...
1208 super(VppTestRunner, self).__init__(sys.stdout, descriptions,
1209 verbosity, failfast, buffer,
1211 reporter = KeepAliveReporter()
1212 reporter.pipe = keep_alive_pipe
1214 VppTestResult.test_framework_result_pipe = result_pipe
1216 def run(self, test):
1223 faulthandler.enable() # emit stack trace to stderr if killed by signal
1225 result = super(VppTestRunner, self).run(test)
1229 class Worker(Thread):
1230 def __init__(self, args, logger, env={}):
1231 self.logger = logger
1234 self.env = copy.deepcopy(env)
1235 super(Worker, self).__init__()
1238 executable = self.args[0]
1239 self.logger.debug("Running executable w/args `%s'" % self.args)
1240 env = os.environ.copy()
1241 env.update(self.env)
1242 env["CK_LOG_FILE_NAME"] = "-"
1243 self.process = subprocess.Popen(
1244 self.args, shell=False, env=env, preexec_fn=os.setpgrp,
1245 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1246 out, err = self.process.communicate()
1247 self.logger.debug("Finished running `%s'" % executable)
1248 self.logger.info("Return code is `%s'" % self.process.returncode)
1249 self.logger.info(single_line_delim)
1250 self.logger.info("Executable `%s' wrote to stdout:" % executable)
1251 self.logger.info(single_line_delim)
1252 self.logger.info(out)
1253 self.logger.info(single_line_delim)
1254 self.logger.info("Executable `%s' wrote to stderr:" % executable)
1255 self.logger.info(single_line_delim)
1256 self.logger.info(err)
1257 self.logger.info(single_line_delim)
1258 self.result = self.process.returncode