test/framework.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import print_function
   4 import gc
   5 import sys
   6 import os
   7 import select
   8 import unittest
   9 import tempfile
  10 import time
  11 import resource
  12 from collections import deque
  13 from threading import Thread, Event
  14 from inspect import getdoc
  15 from traceback import format_exception
  16 from logging import FileHandler, DEBUG, Formatter
  17 from scapy.packet import Raw
  18 from hook import StepHook, PollHook
  19 from vpp_pg_interface import VppPGInterface
  20 from vpp_sub_interface import VppSubInterface
  21 from vpp_lo_interface import VppLoInterface
  22 from vpp_papi_provider import VppPapiProvider
  23 from log import *
  24 from vpp_object import VppObjectRegistry
  25 if os.name == 'posix' and sys.version_info[0] < 3:
  26     # using subprocess32 is recommended by python official documentation
  27     # @ https://docs.python.org/2/library/subprocess.html
  28     import subprocess32 as subprocess
  29 else:
  30     import subprocess
  31
  32 """
  33   Test framework module.
  34
  35   The module provides a set of tools for constructing and running tests and
  36   representing the results.
  37 """
  38
  39
  40 class _PacketInfo(object):
  41     """Private class to create packet info object.
  42
  43     Help process information about the next packet.
  44     Set variables to default values.
  45     """
  46     #: Store the index of the packet.
  47     index = -1
  48     #: Store the index of the source packet generator interface of the packet.
  49     src = -1
  50     #: Store the index of the destination packet generator interface
  51     #: of the packet.
  52     dst = -1
  53     #: Store the copy of the former packet.
  54     data = None
  55
  56     def __eq__(self, other):
  57         index = self.index == other.index
  58         src = self.src == other.src
  59         dst = self.dst == other.dst
  60         data = self.data == other.data
  61         return index and src and dst and data
  62
  63
  64 def pump_output(testclass):
  65     """ pump output from vpp stdout/stderr to proper queues """
  66     while not testclass.pump_thread_stop_flag.wait(0):
  67         readable = select.select([testclass.vpp.stdout.fileno(),
  68                                   testclass.vpp.stderr.fileno(),
  69                                   testclass.pump_thread_wakeup_pipe[0]],
  70                                  [], [])[0]
  71         if testclass.vpp.stdout.fileno() in readable:
  72             read = os.read(testclass.vpp.stdout.fileno(), 1024)
  73             testclass.vpp_stdout_deque.append(read)
  74         if testclass.vpp.stderr.fileno() in readable:
  75             read = os.read(testclass.vpp.stderr.fileno(), 1024)
  76             testclass.vpp_stderr_deque.append(read)
  77         # ignoring the dummy pipe here intentionally - the flag will take care
  78         # of properly terminating the loop
  79
  80
  81 class VppTestCase(unittest.TestCase):
  82     """This subclass is a base class for VPP test cases that are implemented as
  83     classes. It provides methods to create and run test case.
  84     """
  85
  86     @property
  87     def packet_infos(self):
  88         """List of packet infos"""
  89         return self._packet_infos
  90
  91     @classmethod
  92     def get_packet_count_for_if_idx(cls, dst_if_index):
  93         """Get the number of packet info for specified destination if index"""
  94         if dst_if_index in cls._packet_count_for_dst_if_idx:
  95             return cls._packet_count_for_dst_if_idx[dst_if_index]
  96         else:
  97             return 0
  98
  99     @classmethod
 100     def instance(cls):
 101         """Return the instance of this testcase"""
 102         return cls.test_instance
 103
 104     @classmethod
 105     def set_debug_flags(cls, d):
 106         cls.debug_core = False
 107         cls.debug_gdb = False
 108         cls.debug_gdbserver = False
 109         if d is None:
 110             return
 111         dl = d.lower()
 112         if dl == "core":
 113             if resource.getrlimit(resource.RLIMIT_CORE)[0] <= 0:
 114                 # give a heads up if this is actually useless
 115                 print(colorize("WARNING: core size limit is set 0, core files "
 116                                "will NOT be created", RED))
 117             cls.debug_core = True
 118         elif dl == "gdb":
 119             cls.debug_gdb = True
 120         elif dl == "gdbserver":
 121             cls.debug_gdbserver = True
 122         else:
 123             raise Exception("Unrecognized DEBUG option: '%s'" % d)
 124
 125     @classmethod
 126     def setUpConstants(cls):
 127         """ Set-up the test case class based on environment variables """
 128         try:
 129             s = os.getenv("STEP")
 130             cls.step = True if s.lower() in ("y", "yes", "1") else False
 131         except:
 132             cls.step = False
 133         try:
 134             d = os.getenv("DEBUG")
 135         except:
 136             d = None
 137         cls.set_debug_flags(d)
 138         cls.vpp_bin = os.getenv('VPP_TEST_BIN', "vpp")
 139         cls.plugin_path = os.getenv('VPP_TEST_PLUGIN_PATH')
 140         debug_cli = ""
 141         if cls.step or cls.debug_gdb or cls.debug_gdbserver:
 142             debug_cli = "cli-listen localhost:5002"
 143         cls.vpp_cmdline = [cls.vpp_bin,
 144                            "unix", "{", "nodaemon", debug_cli, "}",
 145                            "api-trace", "{", "on", "}",
 146                            "api-segment", "{", "prefix", cls.shm_prefix, "}"]
 147         if cls.plugin_path is not None:
 148             cls.vpp_cmdline.extend(["plugin_path", cls.plugin_path])
 149         cls.logger.info("vpp_cmdline: %s" % cls.vpp_cmdline)
 150
 151     @classmethod
 152     def wait_for_enter(cls):
 153         if cls.debug_gdbserver:
 154             print(double_line_delim)
 155             print("Spawned GDB server with PID: %d" % cls.vpp.pid)
 156         elif cls.debug_gdb:
 157             print(double_line_delim)
 158             print("Spawned VPP with PID: %d" % cls.vpp.pid)
 159         else:
 160             cls.logger.debug("Spawned VPP with PID: %d" % cls.vpp.pid)
 161             return
 162         print(single_line_delim)
 163         print("You can debug the VPP using e.g.:")
 164         if cls.debug_gdbserver:
 165             print("gdb " + cls.vpp_bin + " -ex 'target remote localhost:7777'")
 166             print("Now is the time to attach a gdb by running the above "
 167                   "command, set up breakpoints etc. and then resume VPP from "
 168                   "within gdb by issuing the 'continue' command")
 169         elif cls.debug_gdb:
 170             print("gdb " + cls.vpp_bin + " -ex 'attach %s'" % cls.vpp.pid)
 171             print("Now is the time to attach a gdb by running the above "
 172                   "command and set up breakpoints etc.")
 173         print(single_line_delim)
 174         raw_input("Press ENTER to continue running the testcase...")
 175
 176     @classmethod
 177     def run_vpp(cls):
 178         cmdline = cls.vpp_cmdline
 179
 180         if cls.debug_gdbserver:
 181             gdbserver = '/usr/bin/gdbserver'
 182             if not os.path.isfile(gdbserver) or \
 183                     not os.access(gdbserver, os.X_OK):
 184                 raise Exception("gdbserver binary '%s' does not exist or is "
 185                                 "not executable" % gdbserver)
 186
 187             cmdline = [gdbserver, 'localhost:7777'] + cls.vpp_cmdline
 188             cls.logger.info("Gdbserver cmdline is %s", " ".join(cmdline))
 189
 190         try:
 191             cls.vpp = subprocess.Popen(cmdline,
 192                                        stdout=subprocess.PIPE,
 193                                        stderr=subprocess.PIPE,
 194                                        bufsize=1)
 195         except Exception as e:
 196             cls.logger.critical("Couldn't start vpp: %s" % e)
 197             raise
 198
 199         cls.wait_for_enter()
 200
 201     @classmethod
 202     def setUpClass(cls):
 203         """
 204         Perform class setup before running the testcase
 205         Remove shared memory files, start vpp and connect the vpp-api
 206         """
 207         gc.collect()  # run garbage collection first
 208         cls.logger = getLogger(cls.__name__)
 209         cls.tempdir = tempfile.mkdtemp(
 210             prefix='vpp-unittest-' + cls.__name__ + '-')
 211         file_handler = FileHandler("%s/log.txt" % cls.tempdir)
 212         file_handler.setFormatter(
 213             Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s',
 214                       datefmt="%H:%M:%S"))
 215         file_handler.setLevel(DEBUG)
 216         cls.logger.addHandler(file_handler)
 217         cls.shm_prefix = cls.tempdir.split("/")[-1]
 218         os.chdir(cls.tempdir)
 219         cls.logger.info("Temporary dir is %s, shm prefix is %s",
 220                         cls.tempdir, cls.shm_prefix)
 221         cls.setUpConstants()
 222         cls.reset_packet_infos()
 223         cls._captures = []
 224         cls._zombie_captures = []
 225         cls.verbose = 0
 226         cls.vpp_dead = False
 227         cls.registry = VppObjectRegistry()
 228         print(double_line_delim)
 229         print(colorize(getdoc(cls).splitlines()[0], YELLOW))
 230         print(double_line_delim)
 231         # need to catch exceptions here because if we raise, then the cleanup
 232         # doesn't get called and we might end with a zombie vpp
 233         try:
 234             cls.run_vpp()
 235             cls.vpp_stdout_deque = deque()
 236             cls.vpp_stderr_deque = deque()
 237             cls.pump_thread_stop_flag = Event()
 238             cls.pump_thread_wakeup_pipe = os.pipe()
 239             cls.pump_thread = Thread(target=pump_output, args=(cls,))
 240             cls.pump_thread.daemon = True
 241             cls.pump_thread.start()
 242             cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls)
 243             if cls.step:
 244                 hook = StepHook(cls)
 245             else:
 246                 hook = PollHook(cls)
 247             cls.vapi.register_hook(hook)
 248             cls.sleep(0.1, "after vpp startup, before initial poll")
 249             hook.poll_vpp()
 250             try:
 251                 cls.vapi.connect()
 252             except:
 253                 if cls.debug_gdbserver:
 254                     print(colorize("You're running VPP inside gdbserver but "
 255                                    "VPP-API connection failed, did you forget "
 256                                    "to 'continue' VPP from within gdb?", RED))
 257                 raise
 258         except:
 259             t, v, tb = sys.exc_info()
 260             try:
 261                 cls.quit()
 262             except:
 263                 pass
 264             raise t, v, tb
 265
 266     @classmethod
 267     def quit(cls):
 268         """
 269         Disconnect vpp-api, kill vpp and cleanup shared memory files
 270         """
 271         if (cls.debug_gdbserver or cls.debug_gdb) and hasattr(cls, 'vpp'):
 272             cls.vpp.poll()
 273             if cls.vpp.returncode is None:
 274                 print(double_line_delim)
 275                 print("VPP or GDB server is still running")
 276                 print(single_line_delim)
 277                 raw_input("When done debugging, press ENTER to kill the "
 278                           "process and finish running the testcase...")
 279
 280         os.write(cls.pump_thread_wakeup_pipe[1], 'ding dong wake up')
 281         cls.pump_thread_stop_flag.set()
 282         if hasattr(cls, 'pump_thread'):
 283             cls.logger.debug("Waiting for pump thread to stop")
 284             cls.pump_thread.join()
 285         if hasattr(cls, 'vpp_stderr_reader_thread'):
 286             cls.logger.debug("Waiting for stdderr pump to stop")
 287             cls.vpp_stderr_reader_thread.join()
 288
 289         if hasattr(cls, 'vpp'):
 290             if hasattr(cls, 'vapi'):
 291                 cls.vapi.disconnect()
 292                 del cls.vapi
 293             cls.vpp.poll()
 294             if cls.vpp.returncode is None:
 295                 cls.logger.debug("Sending TERM to vpp")
 296                 cls.vpp.terminate()
 297                 cls.logger.debug("Waiting for vpp to die")
 298                 cls.vpp.communicate()
 299             del cls.vpp
 300
 301         if hasattr(cls, 'vpp_stdout_deque'):
 302             cls.logger.info(single_line_delim)
 303             cls.logger.info('VPP output to stdout while running %s:',
 304                             cls.__name__)
 305             cls.logger.info(single_line_delim)
 306             f = open(cls.tempdir + '/vpp_stdout.txt', 'w')
 307             vpp_output = "".join(cls.vpp_stdout_deque)
 308             f.write(vpp_output)
 309             cls.logger.info('\n%s', vpp_output)
 310             cls.logger.info(single_line_delim)
 311
 312         if hasattr(cls, 'vpp_stderr_deque'):
 313             cls.logger.info(single_line_delim)
 314             cls.logger.info('VPP output to stderr while running %s:',
 315                             cls.__name__)
 316             cls.logger.info(single_line_delim)
 317             f = open(cls.tempdir + '/vpp_stderr.txt', 'w')
 318             vpp_output = "".join(cls.vpp_stderr_deque)
 319             f.write(vpp_output)
 320             cls.logger.info('\n%s', vpp_output)
 321             cls.logger.info(single_line_delim)
 322
 323     @classmethod
 324     def tearDownClass(cls):
 325         """ Perform final cleanup after running all tests in this test-case """
 326         cls.quit()
 327
 328     def tearDown(self):
 329         """ Show various debug prints after each test """
 330         self.logger.debug("--- tearDown() for %s.%s(%s) called ---" %
 331                           (self.__class__.__name__, self._testMethodName,
 332                            self._testMethodDoc))
 333         if not self.vpp_dead:
 334             self.logger.debug(self.vapi.cli("show trace"))
 335             self.logger.info(self.vapi.ppcli("show int"))
 336             self.logger.info(self.vapi.ppcli("show hardware"))
 337             self.logger.info(self.vapi.ppcli("show error"))
 338             self.logger.info(self.vapi.ppcli("show run"))
 339             self.registry.remove_vpp_config(self.logger)
 340             # Save/Dump VPP api trace log
 341             api_trace = "vpp_api_trace.%s.log" % self._testMethodName
 342             tmp_api_trace = "/tmp/%s" % api_trace
 343             vpp_api_trace_log = "%s/%s" % (self.tempdir, api_trace)
 344             self.logger.info(self.vapi.ppcli("api trace save %s" % api_trace))
 345             self.logger.info("Moving %s to %s\n" % (tmp_api_trace,
 346                                                     vpp_api_trace_log))
 347             os.rename(tmp_api_trace, vpp_api_trace_log)
 348             self.logger.info(self.vapi.ppcli("api trace dump %s" %
 349                                              vpp_api_trace_log))
 350
 351     def setUp(self):
 352         """ Clear trace before running each test"""
 353         self.logger.debug("--- setUp() for %s.%s(%s) called ---" %
 354                           (self.__class__.__name__, self._testMethodName,
 355                            self._testMethodDoc))
 356         if self.vpp_dead:
 357             raise Exception("VPP is dead when setting up the test")
 358         self.sleep(.1, "during setUp")
 359         self.vpp_stdout_deque.append(
 360             "--- test setUp() for %s.%s(%s) starts here ---\n" %
 361             (self.__class__.__name__, self._testMethodName,
 362              self._testMethodDoc))
 363         self.vpp_stderr_deque.append(
 364             "--- test setUp() for %s.%s(%s) starts here ---\n" %
 365             (self.__class__.__name__, self._testMethodName,
 366              self._testMethodDoc))
 367         self.vapi.cli("clear trace")
 368         # store the test instance inside the test class - so that objects
 369         # holding the class can access instance methods (like assertEqual)
 370         type(self).test_instance = self
 371
 372     @classmethod
 373     def pg_enable_capture(cls, interfaces):
 374         """
 375         Enable capture on packet-generator interfaces
 376
 377         :param interfaces: iterable interface indexes
 378
 379         """
 380         for i in interfaces:
 381             i.enable_capture()
 382
 383     @classmethod
 384     def register_capture(cls, cap_name):
 385         """ Register a capture in the testclass """
 386         # add to the list of captures with current timestamp
 387         cls._captures.append((time.time(), cap_name))
 388         # filter out from zombies
 389         cls._zombie_captures = [(stamp, name)
 390                                 for (stamp, name) in cls._zombie_captures
 391                                 if name != cap_name]
 392
 393     @classmethod
 394     def pg_start(cls):
 395         """ Remove any zombie captures and enable the packet generator """
 396         # how long before capture is allowed to be deleted - otherwise vpp
 397         # crashes - 100ms seems enough (this shouldn't be needed at all)
 398         capture_ttl = 0.1
 399         now = time.time()
 400         for stamp, cap_name in cls._zombie_captures:
 401             wait = stamp + capture_ttl - now
 402             if wait > 0:
 403                 cls.sleep(wait, "before deleting capture %s" % cap_name)
 404                 now = time.time()
 405             cls.logger.debug("Removing zombie capture %s" % cap_name)
 406             cls.vapi.cli('packet-generator delete %s' % cap_name)
 407
 408         cls.vapi.cli("trace add pg-input 50")  # 50 is maximum
 409         cls.vapi.cli('packet-generator enable')
 410         cls._zombie_captures = cls._captures
 411         cls._captures = []
 412
 413     @classmethod
 414     def create_pg_interfaces(cls, interfaces):
 415         """
 416         Create packet-generator interfaces.
 417
 418         :param interfaces: iterable indexes of the interfaces.
 419         :returns: List of created interfaces.
 420
 421         """
 422         result = []
 423         for i in interfaces:
 424             intf = VppPGInterface(cls, i)
 425             setattr(cls, intf.name, intf)
 426             result.append(intf)
 427         cls.pg_interfaces = result
 428         return result
 429
 430     @classmethod
 431     def create_loopback_interfaces(cls, interfaces):
 432         """
 433         Create loopback interfaces.
 434
 435         :param interfaces: iterable indexes of the interfaces.
 436         :returns: List of created interfaces.
 437         """
 438         result = []
 439         for i in interfaces:
 440             intf = VppLoInterface(cls, i)
 441             setattr(cls, intf.name, intf)
 442             result.append(intf)
 443         cls.lo_interfaces = result
 444         return result
 445
 446     @staticmethod
 447     def extend_packet(packet, size):
 448         """
 449         Extend packet to given size by padding with spaces
 450         NOTE: Currently works only when Raw layer is present.
 451
 452         :param packet: packet
 453         :param size: target size
 454
 455         """
 456         packet_len = len(packet) + 4
 457         extend = size - packet_len
 458         if extend > 0:
 459             packet[Raw].load += ' ' * extend
 460
 461     @classmethod
 462     def reset_packet_infos(cls):
 463         """ Reset the list of packet info objects and packet counts to zero """
 464         cls._packet_infos = {}
 465         cls._packet_count_for_dst_if_idx = {}
 466
 467     @classmethod
 468     def create_packet_info(cls, src_if, dst_if):
 469         """
 470         Create packet info object containing the source and destination indexes
 471         and add it to the testcase's packet info list
 472
 473         :param VppInterface src_if: source interface
 474         :param VppInterface dst_if: destination interface
 475
 476         :returns: _PacketInfo object
 477
 478         """
 479         info = _PacketInfo()
 480         info.index = len(cls._packet_infos)
 481         info.src = src_if.sw_if_index
 482         info.dst = dst_if.sw_if_index
 483         if isinstance(dst_if, VppSubInterface):
 484             dst_idx = dst_if.parent.sw_if_index
 485         else:
 486             dst_idx = dst_if.sw_if_index
 487         if dst_idx in cls._packet_count_for_dst_if_idx:
 488             cls._packet_count_for_dst_if_idx[dst_idx] += 1
 489         else:
 490             cls._packet_count_for_dst_if_idx[dst_idx] = 1
 491         cls._packet_infos[info.index] = info
 492         return info
 493
 494     @staticmethod
 495     def info_to_payload(info):
 496         """
 497         Convert _PacketInfo object to packet payload
 498
 499         :param info: _PacketInfo object
 500
 501         :returns: string containing serialized data from packet info
 502         """
 503         return "%d %d %d" % (info.index, info.src, info.dst)
 504
 505     @staticmethod
 506     def payload_to_info(payload):
 507         """
 508         Convert packet payload to _PacketInfo object
 509
 510         :param payload: packet payload
 511
 512         :returns: _PacketInfo object containing de-serialized data from payload
 513
 514         """
 515         numbers = payload.split()
 516         info = _PacketInfo()
 517         info.index = int(numbers[0])
 518         info.src = int(numbers[1])
 519         info.dst = int(numbers[2])
 520         return info
 521
 522     def get_next_packet_info(self, info):
 523         """
 524         Iterate over the packet info list stored in the testcase
 525         Start iteration with first element if info is None
 526         Continue based on index in info if info is specified
 527
 528         :param info: info or None
 529         :returns: next info in list or None if no more infos
 530         """
 531         if info is None:
 532             next_index = 0
 533         else:
 534             next_index = info.index + 1
 535         if next_index == len(self._packet_infos):
 536             return None
 537         else:
 538             return self._packet_infos[next_index]
 539
 540     def get_next_packet_info_for_interface(self, src_index, info):
 541         """
 542         Search the packet info list for the next packet info with same source
 543         interface index
 544
 545         :param src_index: source interface index to search for
 546         :param info: packet info - where to start the search
 547         :returns: packet info or None
 548
 549         """
 550         while True:
 551             info = self.get_next_packet_info(info)
 552             if info is None:
 553                 return None
 554             if info.src == src_index:
 555                 return info
 556
 557     def get_next_packet_info_for_interface2(self, src_index, dst_index, info):
 558         """
 559         Search the packet info list for the next packet info with same source
 560         and destination interface indexes
 561
 562         :param src_index: source interface index to search for
 563         :param dst_index: destination interface index to search for
 564         :param info: packet info - where to start the search
 565         :returns: packet info or None
 566
 567         """
 568         while True:
 569             info = self.get_next_packet_info_for_interface(src_index, info)
 570             if info is None:
 571                 return None
 572             if info.dst == dst_index:
 573                 return info
 574
 575     def assert_equal(self, real_value, expected_value, name_or_class=None):
 576         if name_or_class is None:
 577             self.assertEqual(real_value, expected_value)
 578             return
 579         try:
 580             msg = "Invalid %s: %d('%s') does not match expected value %d('%s')"
 581             msg = msg % (getdoc(name_or_class).strip(),
 582                          real_value, str(name_or_class(real_value)),
 583                          expected_value, str(name_or_class(expected_value)))
 584         except:
 585             msg = "Invalid %s: %s does not match expected value %s" % (
 586                 name_or_class, real_value, expected_value)
 587
 588         self.assertEqual(real_value, expected_value, msg)
 589
 590     def assert_in_range(self,
 591                         real_value,
 592                         expected_min,
 593                         expected_max,
 594                         name=None):
 595         if name is None:
 596             msg = None
 597         else:
 598             msg = "Invalid %s: %s out of range <%s,%s>" % (
 599                 name, real_value, expected_min, expected_max)
 600         self.assertTrue(expected_min <= real_value <= expected_max, msg)
 601
 602     @classmethod
 603     def sleep(cls, timeout, remark=None):
 604         if hasattr(cls, 'logger'):
 605             cls.logger.debug("Sleeping for %ss (%s)" % (timeout, remark))
 606         time.sleep(timeout)
 607
 608
 609 class VppTestResult(unittest.TestResult):
 610     """
 611     @property result_string
 612      String variable to store the test case result string.
 613     @property errors
 614      List variable containing 2-tuples of TestCase instances and strings
 615      holding formatted tracebacks. Each tuple represents a test which
 616      raised an unexpected exception.
 617     @property failures
 618      List variable containing 2-tuples of TestCase instances and strings
 619      holding formatted tracebacks. Each tuple represents a test where
 620      a failure was explicitly signalled using the TestCase.assert*()
 621      methods.
 622     """
 623
 624     def __init__(self, stream, descriptions, verbosity):
 625         """
 626         :param stream File descriptor to store where to report test results.
 627             Set to the standard error stream by default.
 628         :param descriptions Boolean variable to store information if to use
 629             test case descriptions.
 630         :param verbosity Integer variable to store required verbosity level.
 631         """
 632         unittest.TestResult.__init__(self, stream, descriptions, verbosity)
 633         self.stream = stream
 634         self.descriptions = descriptions
 635         self.verbosity = verbosity
 636         self.result_string = None
 637
 638     def addSuccess(self, test):
 639         """
 640         Record a test succeeded result
 641
 642         :param test:
 643
 644         """
 645         if hasattr(test, 'logger'):
 646             test.logger.debug("--- addSuccess() %s.%s(%s) called"
 647                               % (test.__class__.__name__,
 648                                  test._testMethodName,
 649                                  test._testMethodDoc))
 650         unittest.TestResult.addSuccess(self, test)
 651         self.result_string = colorize("OK", GREEN)
 652
 653     def addSkip(self, test, reason):
 654         """
 655         Record a test skipped.
 656
 657         :param test:
 658         :param reason:
 659
 660         """
 661         if hasattr(test, 'logger'):
 662             test.logger.debug("--- addSkip() %s.%s(%s) called, reason is %s"
 663                               % (test.__class__.__name__,
 664                                  test._testMethodName,
 665                                  test._testMethodDoc,
 666                                  reason))
 667         unittest.TestResult.addSkip(self, test, reason)
 668         self.result_string = colorize("SKIP", YELLOW)
 669
 670     def addFailure(self, test, err):
 671         """
 672         Record a test failed result
 673
 674         :param test:
 675         :param err: error message
 676
 677         """
 678         if hasattr(test, 'logger'):
 679             test.logger.debug("--- addFailure() %s.%s(%s) called, err is %s"
 680                               % (test.__class__.__name__,
 681                                  test._testMethodName,
 682                                  test._testMethodDoc, err))
 683             test.logger.debug("formatted exception is:\n%s" %
 684                               "".join(format_exception(*err)))
 685         unittest.TestResult.addFailure(self, test, err)
 686         if hasattr(test, 'tempdir'):
 687             self.result_string = colorize("FAIL", RED) + \
 688                 ' [ temp dir used by test case: ' + test.tempdir + ' ]'
 689         else:
 690             self.result_string = colorize("FAIL", RED) + ' [no temp dir]'
 691
 692     def addError(self, test, err):
 693         """
 694         Record a test error result
 695
 696         :param test:
 697         :param err: error message
 698
 699         """
 700         if hasattr(test, 'logger'):
 701             test.logger.debug("--- addError() %s.%s(%s) called, err is %s"
 702                               % (test.__class__.__name__,
 703                                  test._testMethodName,
 704                                  test._testMethodDoc, err))
 705             test.logger.debug("formatted exception is:\n%s" %
 706                               "".join(format_exception(*err)))
 707         unittest.TestResult.addError(self, test, err)
 708         if hasattr(test, 'tempdir'):
 709             self.result_string = colorize("ERROR", RED) + \
 710                 ' [ temp dir used by test case: ' + test.tempdir + ' ]'
 711         else:
 712             self.result_string = colorize("ERROR", RED) + ' [no temp dir]'
 713
 714     def getDescription(self, test):
 715         """
 716         Get test description
 717
 718         :param test:
 719         :returns: test description
 720
 721         """
 722         # TODO: if none print warning not raise exception
 723         short_description = test.shortDescription()
 724         if self.descriptions and short_description:
 725             return short_description
 726         else:
 727             return str(test)
 728
 729     def startTest(self, test):
 730         """
 731         Start a test
 732
 733         :param test:
 734
 735         """
 736         unittest.TestResult.startTest(self, test)
 737         if self.verbosity > 0:
 738             self.stream.writeln(
 739                 "Starting " + self.getDescription(test) + " ...")
 740             self.stream.writeln(single_line_delim)
 741
 742     def stopTest(self, test):
 743         """
 744         Stop a test
 745
 746         :param test:
 747
 748         """
 749         unittest.TestResult.stopTest(self, test)
 750         if self.verbosity > 0:
 751             self.stream.writeln(single_line_delim)
 752             self.stream.writeln("%-73s%s" % (self.getDescription(test),
 753                                              self.result_string))
 754             self.stream.writeln(single_line_delim)
 755         else:
 756             self.stream.writeln("%-73s%s" % (self.getDescription(test),
 757                                              self.result_string))
 758
 759     def printErrors(self):
 760         """
 761         Print errors from running the test case
 762         """
 763         self.stream.writeln()
 764         self.printErrorList('ERROR', self.errors)
 765         self.printErrorList('FAIL', self.failures)
 766
 767     def printErrorList(self, flavour, errors):
 768         """
 769         Print error list to the output stream together with error type
 770         and test case description.
 771
 772         :param flavour: error type
 773         :param errors: iterable errors
 774
 775         """
 776         for test, err in errors:
 777             self.stream.writeln(double_line_delim)
 778             self.stream.writeln("%s: %s" %
 779                                 (flavour, self.getDescription(test)))
 780             self.stream.writeln(single_line_delim)
 781             self.stream.writeln("%s" % err)
 782
 783
 784 class VppTestRunner(unittest.TextTestRunner):
 785     """
 786     A basic test runner implementation which prints results to standard error.
 787     """
 788     @property
 789     def resultclass(self):
 790         """Class maintaining the results of the tests"""
 791         return VppTestResult
 792
 793     def __init__(self, stream=sys.stderr, descriptions=True, verbosity=1,
 794                  failfast=False, buffer=False, resultclass=None):
 795         # ignore stream setting here, use hard-coded stdout to be in sync
 796         # with prints from VppTestCase methods ...
 797         super(VppTestRunner, self).__init__(sys.stdout, descriptions,
 798                                             verbosity, failfast, buffer,
 799                                             resultclass)
 800
 801     test_option = "TEST"
 802
 803     def parse_test_option(self):
 804         try:
 805             f = os.getenv(self.test_option)
 806         except:
 807             f = None
 808         filter_file_name = None
 809         filter_class_name = None
 810         filter_func_name = None
 811         if f:
 812             if '.' in f:
 813                 parts = f.split('.')
 814                 if len(parts) > 3:
 815                     raise Exception("Unrecognized %s option: %s" %
 816                                     (self.test_option, f))
 817                 if len(parts) > 2:
 818                     if parts[2] not in ('*', ''):
 819                         filter_func_name = parts[2]
 820                 if parts[1] not in ('*', ''):
 821                     filter_class_name = parts[1]
 822                 if parts[0] not in ('*', ''):
 823                     if parts[0].startswith('test_'):
 824                         filter_file_name = parts[0]
 825                     else:
 826                         filter_file_name = 'test_%s' % parts[0]
 827             else:
 828                 if f.startswith('test_'):
 829                     filter_file_name = f
 830                 else:
 831                     filter_file_name = 'test_%s' % f
 832         return filter_file_name, filter_class_name, filter_func_name
 833
 834     def filter_tests(self, tests, filter_file, filter_class, filter_func):
 835         result = unittest.suite.TestSuite()
 836         for t in tests:
 837             if isinstance(t, unittest.suite.TestSuite):
 838                 # this is a bunch of tests, recursively filter...
 839                 x = self.filter_tests(t, filter_file, filter_class,
 840                                       filter_func)
 841                 if x.countTestCases() > 0:
 842                     result.addTest(x)
 843             elif isinstance(t, unittest.TestCase):
 844                 # this is a single test
 845                 parts = t.id().split('.')
 846                 # t.id() for common cases like this:
 847                 # test_classifier.TestClassifier.test_acl_ip
 848                 # apply filtering only if it is so
 849                 if len(parts) == 3:
 850                     if filter_file and filter_file != parts[0]:
 851                         continue
 852                     if filter_class and filter_class != parts[1]:
 853                         continue
 854                     if filter_func and filter_func != parts[2]:
 855                         continue
 856                 result.addTest(t)
 857             else:
 858                 # unexpected object, don't touch it
 859                 result.addTest(t)
 860         return result
 861
 862     def run(self, test):
 863         """
 864         Run the tests
 865
 866         :param test:
 867
 868         """
 869         gc.disable()  # disable garbage collection, we'll do that manually
 870         print("Running tests using custom test runner")  # debug message
 871         filter_file, filter_class, filter_func = self.parse_test_option()
 872         print("Active filters: file=%s, class=%s, function=%s" % (
 873             filter_file, filter_class, filter_func))
 874         filtered = self.filter_tests(test, filter_file, filter_class,
 875                                      filter_func)
 876         print("%s out of %s tests match specified filters" % (
 877             filtered.countTestCases(), test.countTestCases()))
 878         return super(VppTestRunner, self).run(filtered)