test/framework.py

   1 #!/usr/bin/env python
   2
   3 import subprocess
   4 import unittest
   5 import tempfile
   6 import time
   7 import resource
   8 from collections import deque
   9 from threading import Thread
  10 from inspect import getdoc
  11 from hook import StepHook, PollHook
  12 from vpp_pg_interface import VppPGInterface
  13 from vpp_sub_interface import VppSubInterface
  14 from vpp_lo_interface import VppLoInterface
  15 from vpp_papi_provider import VppPapiProvider
  16 from scapy.packet import Raw
  17 from logging import FileHandler, DEBUG
  18 from log import *
  19
  20 """
  21   Test framework module.
  22
  23   The module provides a set of tools for constructing and running tests and
  24   representing the results.
  25 """
  26
  27
  28 class _PacketInfo(object):
  29     """Private class to create packet info object.
  30
  31     Help process information about the next packet.
  32     Set variables to default values.
  33     """
  34     #: Store the index of the packet.
  35     index = -1
  36     #: Store the index of the source packet generator interface of the packet.
  37     src = -1
  38     #: Store the index of the destination packet generator interface
  39     #: of the packet.
  40     dst = -1
  41     #: Store the copy of the former packet.
  42     data = None
  43
  44     def __eq__(self, other):
  45         index = self.index == other.index
  46         src = self.src == other.src
  47         dst = self.dst == other.dst
  48         data = self.data == other.data
  49         return index and src and dst and data
  50
  51
  52 def pump_output(out, deque):
  53     for line in iter(out.readline, b''):
  54         deque.append(line)
  55
  56
  57 class VppTestCase(unittest.TestCase):
  58     """This subclass is a base class for VPP test cases that are implemented as
  59     classes. It provides methods to create and run test case.
  60     """
  61
  62     @property
  63     def packet_infos(self):
  64         """List of packet infos"""
  65         return self._packet_infos
  66
  67     @classmethod
  68     def get_packet_count_for_if_idx(cls, dst_if_index):
  69         """Get the number of packet info for specified destination if index"""
  70         if dst_if_index in cls._packet_count_for_dst_if_idx:
  71             return cls._packet_count_for_dst_if_idx[dst_if_index]
  72         else:
  73             return 0
  74
  75     @classmethod
  76     def instance(cls):
  77         """Return the instance of this testcase"""
  78         return cls.test_instance
  79
  80     @classmethod
  81     def set_debug_flags(cls, d):
  82         cls.debug_core = False
  83         cls.debug_gdb = False
  84         cls.debug_gdbserver = False
  85         if d is None:
  86             return
  87         dl = d.lower()
  88         if dl == "core":
  89             if resource.getrlimit(resource.RLIMIT_CORE)[0] <= 0:
  90                 # give a heads up if this is actually useless
  91                 cls.logger.critical("WARNING: core size limit is set 0, core "
  92                                     "files will NOT be created")
  93             cls.debug_core = True
  94         elif dl == "gdb":
  95             cls.debug_gdb = True
  96         elif dl == "gdbserver":
  97             cls.debug_gdbserver = True
  98         else:
  99             raise Exception("Unrecognized DEBUG option: '%s'" % d)
 100
 101     @classmethod
 102     def setUpConstants(cls):
 103         """ Set-up the test case class based on environment variables """
 104         try:
 105             s = os.getenv("STEP")
 106             cls.step = True if s.lower() in ("y", "yes", "1") else False
 107         except:
 108             cls.step = False
 109         try:
 110             d = os.getenv("DEBUG")
 111         except:
 112             d = None
 113         cls.set_debug_flags(d)
 114         cls.vpp_bin = os.getenv('VPP_TEST_BIN', "vpp")
 115         cls.plugin_path = os.getenv('VPP_TEST_PLUGIN_PATH')
 116         debug_cli = ""
 117         if cls.step or cls.debug_gdb or cls.debug_gdbserver:
 118             debug_cli = "cli-listen localhost:5002"
 119         cls.vpp_cmdline = [cls.vpp_bin, "unix", "{", "nodaemon", debug_cli, "}",
 120                            "api-segment", "{", "prefix", cls.shm_prefix, "}"]
 121         if cls.plugin_path is not None:
 122             cls.vpp_cmdline.extend(["plugin_path", cls.plugin_path])
 123         cls.logger.info("vpp_cmdline: %s" % cls.vpp_cmdline)
 124
 125     @classmethod
 126     def wait_for_enter(cls):
 127         if cls.debug_gdbserver:
 128             print(double_line_delim)
 129             print("Spawned GDB server with PID: %d" % cls.vpp.pid)
 130         elif cls.debug_gdb:
 131             print(double_line_delim)
 132             print("Spawned VPP with PID: %d" % cls.vpp.pid)
 133         else:
 134             cls.logger.debug("Spawned VPP with PID: %d" % cls.vpp.pid)
 135             return
 136         print(single_line_delim)
 137         print("You can debug the VPP using e.g.:")
 138         if cls.debug_gdbserver:
 139             print("gdb " + cls.vpp_bin + " -ex 'target remote localhost:7777'")
 140             print("Now is the time to attach a gdb by running the above "
 141                   "command, set up breakpoints etc. and then resume VPP from "
 142                   "within gdb by issuing the 'continue' command")
 143         elif cls.debug_gdb:
 144             print("gdb " + cls.vpp_bin + " -ex 'attach %s'" % cls.vpp.pid)
 145             print("Now is the time to attach a gdb by running the above "
 146                   "command and set up breakpoints etc.")
 147         print(single_line_delim)
 148         raw_input("Press ENTER to continue running the testcase...")
 149
 150     @classmethod
 151     def run_vpp(cls):
 152         cmdline = cls.vpp_cmdline
 153
 154         if cls.debug_gdbserver:
 155             gdbserver = '/usr/bin/gdbserver'
 156             if not os.path.isfile(gdbserver) or \
 157                     not os.access(gdbserver, os.X_OK):
 158                 raise Exception("gdbserver binary '%s' does not exist or is "
 159                                 "not executable" % gdbserver)
 160
 161             cmdline = [gdbserver, 'localhost:7777'] + cls.vpp_cmdline
 162             cls.logger.info("Gdbserver cmdline is %s", " ".join(cmdline))
 163
 164         try:
 165             cls.vpp = subprocess.Popen(cmdline,
 166                                        stdout=subprocess.PIPE,
 167                                        stderr=subprocess.PIPE,
 168                                        bufsize=1)
 169         except Exception as e:
 170             cls.logger.critical("Couldn't start vpp: %s" % e)
 171             raise
 172
 173         cls.wait_for_enter()
 174
 175     @classmethod
 176     def setUpClass(cls):
 177         """
 178         Perform class setup before running the testcase
 179         Remove shared memory files, start vpp and connect the vpp-api
 180         """
 181         cls.logger = getLogger(cls.__name__)
 182         cls.tempdir = tempfile.mkdtemp(
 183             prefix='vpp-unittest-' + cls.__name__ + '-')
 184         file_handler = FileHandler("%s/log.txt" % cls.tempdir)
 185         file_handler.setLevel(DEBUG)
 186         cls.logger.addHandler(file_handler)
 187         cls.shm_prefix = cls.tempdir.split("/")[-1]
 188         os.chdir(cls.tempdir)
 189         cls.logger.info("Temporary dir is %s, shm prefix is %s",
 190                         cls.tempdir, cls.shm_prefix)
 191         cls.setUpConstants()
 192         cls.reset_packet_infos()
 193         cls._captures = []
 194         cls._zombie_captures = []
 195         cls.verbose = 0
 196         cls.vpp_dead = False
 197         print(double_line_delim)
 198         print(colorize(getdoc(cls).splitlines()[0], YELLOW))
 199         print(double_line_delim)
 200         # need to catch exceptions here because if we raise, then the cleanup
 201         # doesn't get called and we might end with a zombie vpp
 202         try:
 203             cls.run_vpp()
 204             cls.vpp_stdout_deque = deque()
 205             cls.vpp_stdout_reader_thread = Thread(target=pump_output, args=(
 206                 cls.vpp.stdout, cls.vpp_stdout_deque))
 207             cls.vpp_stdout_reader_thread.start()
 208             cls.vpp_stderr_deque = deque()
 209             cls.vpp_stderr_reader_thread = Thread(target=pump_output, args=(
 210                 cls.vpp.stderr, cls.vpp_stderr_deque))
 211             cls.vpp_stderr_reader_thread.start()
 212             cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls)
 213             if cls.step:
 214                 hook = StepHook(cls)
 215             else:
 216                 hook = PollHook(cls)
 217             cls.vapi.register_hook(hook)
 218             time.sleep(0.1)
 219             hook.poll_vpp()
 220             try:
 221                 cls.vapi.connect()
 222             except:
 223                 if cls.debug_gdbserver:
 224                     print(colorize("You're running VPP inside gdbserver but "
 225                                    "VPP-API connection failed, did you forget "
 226                                    "to 'continue' VPP from within gdb?", RED))
 227                 raise
 228         except:
 229             t, v, tb = sys.exc_info()
 230             try:
 231                 cls.quit()
 232             except:
 233                 pass
 234             raise t, v, tb
 235
 236     @classmethod
 237     def quit(cls):
 238         """
 239         Disconnect vpp-api, kill vpp and cleanup shared memory files
 240         """
 241         if (cls.debug_gdbserver or cls.debug_gdb) and hasattr(cls, 'vpp'):
 242             cls.vpp.poll()
 243             if cls.vpp.returncode is None:
 244                 print(double_line_delim)
 245                 print("VPP or GDB server is still running")
 246                 print(single_line_delim)
 247                 raw_input("When done debugging, press ENTER to kill the process"
 248                           " and finish running the testcase...")
 249
 250         if hasattr(cls, 'vpp'):
 251             if hasattr(cls, 'vapi'):
 252                 cls.vapi.disconnect()
 253             cls.vpp.poll()
 254             if cls.vpp.returncode is None:
 255                 cls.vpp.terminate()
 256             del cls.vpp
 257
 258         if hasattr(cls, 'vpp_stdout_deque'):
 259             cls.logger.info(single_line_delim)
 260             cls.logger.info('VPP output to stdout while running %s:',
 261                             cls.__name__)
 262             cls.logger.info(single_line_delim)
 263             f = open(cls.tempdir + '/vpp_stdout.txt', 'w')
 264             vpp_output = "".join(cls.vpp_stdout_deque)
 265             f.write(vpp_output)
 266             cls.logger.info('\n%s', vpp_output)
 267             cls.logger.info(single_line_delim)
 268
 269         if hasattr(cls, 'vpp_stderr_deque'):
 270             cls.logger.info(single_line_delim)
 271             cls.logger.info('VPP output to stderr while running %s:',
 272                             cls.__name__)
 273             cls.logger.info(single_line_delim)
 274             f = open(cls.tempdir + '/vpp_stderr.txt', 'w')
 275             vpp_output = "".join(cls.vpp_stderr_deque)
 276             f.write(vpp_output)
 277             cls.logger.info('\n%s', vpp_output)
 278             cls.logger.info(single_line_delim)
 279
 280     @classmethod
 281     def tearDownClass(cls):
 282         """ Perform final cleanup after running all tests in this test-case """
 283         cls.quit()
 284
 285     def tearDown(self):
 286         """ Show various debug prints after each test """
 287         if not self.vpp_dead:
 288             self.logger.debug(self.vapi.cli("show trace"))
 289             self.logger.info(self.vapi.ppcli("show int"))
 290             self.logger.info(self.vapi.ppcli("show hardware"))
 291             self.logger.info(self.vapi.ppcli("show error"))
 292             self.logger.info(self.vapi.ppcli("show run"))
 293
 294     def setUp(self):
 295         """ Clear trace before running each test"""
 296         if self.vpp_dead:
 297             raise Exception("VPP is dead when setting up the test")
 298         time.sleep(.1)
 299         self.vpp_stdout_deque.append(
 300             "--- test setUp() for %s.%s(%s) starts here ---\n" %
 301             (self.__class__.__name__, self._testMethodName,
 302              self._testMethodDoc))
 303         self.vpp_stderr_deque.append(
 304             "--- test setUp() for %s.%s(%s) starts here ---\n" %
 305             (self.__class__.__name__, self._testMethodName,
 306              self._testMethodDoc))
 307         self.vapi.cli("clear trace")
 308         # store the test instance inside the test class - so that objects
 309         # holding the class can access instance methods (like assertEqual)
 310         type(self).test_instance = self
 311
 312     @classmethod
 313     def pg_enable_capture(cls, interfaces):
 314         """
 315         Enable capture on packet-generator interfaces
 316
 317         :param interfaces: iterable interface indexes
 318
 319         """
 320         for i in interfaces:
 321             i.enable_capture()
 322
 323     @classmethod
 324     def register_capture(cls, cap_name):
 325         """ Register a capture in the testclass """
 326         # add to the list of captures with current timestamp
 327         cls._captures.append((time.time(), cap_name))
 328         # filter out from zombies
 329         cls._zombie_captures = [(stamp, name)
 330                                 for (stamp, name) in cls._zombie_captures
 331                                 if name != cap_name]
 332
 333     @classmethod
 334     def pg_start(cls):
 335         """ Remove any zombie captures and enable the packet generator """
 336         # how long before capture is allowed to be deleted - otherwise vpp
 337         # crashes - 100ms seems enough (this shouldn't be needed at all)
 338         capture_ttl = 0.1
 339         now = time.time()
 340         for stamp, cap_name in cls._zombie_captures:
 341             wait = stamp + capture_ttl - now
 342             if wait > 0:
 343                 cls.logger.debug("Waiting for %ss before deleting capture %s",
 344                                  wait, cap_name)
 345                 time.sleep(wait)
 346                 now = time.time()
 347             cls.logger.debug("Removing zombie capture %s" % cap_name)
 348             cls.vapi.cli('packet-generator delete %s' % cap_name)
 349
 350         cls.vapi.cli("trace add pg-input 50")  # 50 is maximum
 351         cls.vapi.cli('packet-generator enable')
 352         cls._zombie_captures = cls._captures
 353         cls._captures = []
 354
 355     @classmethod
 356     def create_pg_interfaces(cls, interfaces):
 357         """
 358         Create packet-generator interfaces.
 359
 360         :param interfaces: iterable indexes of the interfaces.
 361         :returns: List of created interfaces.
 362
 363         """
 364         result = []
 365         for i in interfaces:
 366             intf = VppPGInterface(cls, i)
 367             setattr(cls, intf.name, intf)
 368             result.append(intf)
 369         cls.pg_interfaces = result
 370         return result
 371
 372     @classmethod
 373     def create_loopback_interfaces(cls, interfaces):
 374         """
 375         Create loopback interfaces.
 376
 377         :param interfaces: iterable indexes of the interfaces.
 378         :returns: List of created interfaces.
 379         """
 380         result = []
 381         for i in interfaces:
 382             intf = VppLoInterface(cls, i)
 383             setattr(cls, intf.name, intf)
 384             result.append(intf)
 385         cls.lo_interfaces = result
 386         return result
 387
 388     @staticmethod
 389     def extend_packet(packet, size):
 390         """
 391         Extend packet to given size by padding with spaces
 392         NOTE: Currently works only when Raw layer is present.
 393
 394         :param packet: packet
 395         :param size: target size
 396
 397         """
 398         packet_len = len(packet) + 4
 399         extend = size - packet_len
 400         if extend > 0:
 401             packet[Raw].load += ' ' * extend
 402
 403     @classmethod
 404     def reset_packet_infos(cls):
 405         """ Reset the list of packet info objects and packet counts to zero """
 406         cls._packet_infos = {}
 407         cls._packet_count_for_dst_if_idx = {}
 408
 409     @classmethod
 410     def create_packet_info(cls, src_if, dst_if):
 411         """
 412         Create packet info object containing the source and destination indexes
 413         and add it to the testcase's packet info list
 414
 415         :param VppInterface src_if: source interface
 416         :param VppInterface dst_if: destination interface
 417
 418         :returns: _PacketInfo object
 419
 420         """
 421         info = _PacketInfo()
 422         info.index = len(cls._packet_infos)
 423         info.src = src_if.sw_if_index
 424         info.dst = dst_if.sw_if_index
 425         if isinstance(dst_if, VppSubInterface):
 426             dst_idx = dst_if.parent.sw_if_index
 427         else:
 428             dst_idx = dst_if.sw_if_index
 429         if dst_idx in cls._packet_count_for_dst_if_idx:
 430             cls._packet_count_for_dst_if_idx[dst_idx] += 1
 431         else:
 432             cls._packet_count_for_dst_if_idx[dst_idx] = 1
 433         cls._packet_infos[info.index] = info
 434         return info
 435
 436     @staticmethod
 437     def info_to_payload(info):
 438         """
 439         Convert _PacketInfo object to packet payload
 440
 441         :param info: _PacketInfo object
 442
 443         :returns: string containing serialized data from packet info
 444         """
 445         return "%d %d %d" % (info.index, info.src, info.dst)
 446
 447     @staticmethod
 448     def payload_to_info(payload):
 449         """
 450         Convert packet payload to _PacketInfo object
 451
 452         :param payload: packet payload
 453
 454         :returns: _PacketInfo object containing de-serialized data from payload
 455
 456         """
 457         numbers = payload.split()
 458         info = _PacketInfo()
 459         info.index = int(numbers[0])
 460         info.src = int(numbers[1])
 461         info.dst = int(numbers[2])
 462         return info
 463
 464     def get_next_packet_info(self, info):
 465         """
 466         Iterate over the packet info list stored in the testcase
 467         Start iteration with first element if info is None
 468         Continue based on index in info if info is specified
 469
 470         :param info: info or None
 471         :returns: next info in list or None if no more infos
 472         """
 473         if info is None:
 474             next_index = 0
 475         else:
 476             next_index = info.index + 1
 477         if next_index == len(self._packet_infos):
 478             return None
 479         else:
 480             return self._packet_infos[next_index]
 481
 482     def get_next_packet_info_for_interface(self, src_index, info):
 483         """
 484         Search the packet info list for the next packet info with same source
 485         interface index
 486
 487         :param src_index: source interface index to search for
 488         :param info: packet info - where to start the search
 489         :returns: packet info or None
 490
 491         """
 492         while True:
 493             info = self.get_next_packet_info(info)
 494             if info is None:
 495                 return None
 496             if info.src == src_index:
 497                 return info
 498
 499     def get_next_packet_info_for_interface2(self, src_index, dst_index, info):
 500         """
 501         Search the packet info list for the next packet info with same source
 502         and destination interface indexes
 503
 504         :param src_index: source interface index to search for
 505         :param dst_index: destination interface index to search for
 506         :param info: packet info - where to start the search
 507         :returns: packet info or None
 508
 509         """
 510         while True:
 511             info = self.get_next_packet_info_for_interface(src_index, info)
 512             if info is None:
 513                 return None
 514             if info.dst == dst_index:
 515                 return info
 516
 517     def assert_equal(self, real_value, expected_value, name_or_class=None):
 518         if name_or_class is None:
 519             self.assertEqual(real_value, expected_value, msg)
 520             return
 521         try:
 522             msg = "Invalid %s: %d('%s') does not match expected value %d('%s')"
 523             msg = msg % (getdoc(name_or_class).strip(),
 524                          real_value, str(name_or_class(real_value)),
 525                          expected_value, str(name_or_class(expected_value)))
 526         except:
 527             msg = "Invalid %s: %s does not match expected value %s" % (
 528                 name_or_class, real_value, expected_value)
 529
 530         self.assertEqual(real_value, expected_value, msg)
 531
 532     def assert_in_range(
 533             self,
 534             real_value,
 535             expected_min,
 536             expected_max,
 537             name=None):
 538         if name is None:
 539             msg = None
 540         else:
 541             msg = "Invalid %s: %s out of range <%s,%s>" % (
 542                 name, real_value, expected_min, expected_max)
 543         self.assertTrue(expected_min <= real_value <= expected_max, msg)
 544
 545
 546 class VppTestResult(unittest.TestResult):
 547     """
 548     @property result_string
 549      String variable to store the test case result string.
 550     @property errors
 551      List variable containing 2-tuples of TestCase instances and strings
 552      holding formatted tracebacks. Each tuple represents a test which
 553      raised an unexpected exception.
 554     @property failures
 555      List variable containing 2-tuples of TestCase instances and strings
 556      holding formatted tracebacks. Each tuple represents a test where
 557      a failure was explicitly signalled using the TestCase.assert*()
 558      methods.
 559     """
 560
 561     def __init__(self, stream, descriptions, verbosity):
 562         """
 563         :param stream File descriptor to store where to report test results. Set
 564             to the standard error stream by default.
 565         :param descriptions Boolean variable to store information if to use test
 566             case descriptions.
 567         :param verbosity Integer variable to store required verbosity level.
 568         """
 569         unittest.TestResult.__init__(self, stream, descriptions, verbosity)
 570         self.stream = stream
 571         self.descriptions = descriptions
 572         self.verbosity = verbosity
 573         self.result_string = None
 574
 575     def addSuccess(self, test):
 576         """
 577         Record a test succeeded result
 578
 579         :param test:
 580
 581         """
 582         unittest.TestResult.addSuccess(self, test)
 583         self.result_string = colorize("OK", GREEN)
 584
 585     def addSkip(self, test, reason):
 586         """
 587         Record a test skipped.
 588
 589         :param test:
 590         :param reason:
 591
 592         """
 593         unittest.TestResult.addSkip(self, test, reason)
 594         self.result_string = colorize("SKIP", YELLOW)
 595
 596     def addFailure(self, test, err):
 597         """
 598         Record a test failed result
 599
 600         :param test:
 601         :param err: error message
 602
 603         """
 604         unittest.TestResult.addFailure(self, test, err)
 605         if hasattr(test, 'tempdir'):
 606             self.result_string = colorize("FAIL", RED) + \
 607                 ' [ temp dir used by test case: ' + test.tempdir + ' ]'
 608         else:
 609             self.result_string = colorize("FAIL", RED) + ' [no temp dir]'
 610
 611     def addError(self, test, err):
 612         """
 613         Record a test error result
 614
 615         :param test:
 616         :param err: error message
 617
 618         """
 619         unittest.TestResult.addError(self, test, err)
 620         if hasattr(test, 'tempdir'):
 621             self.result_string = colorize("ERROR", RED) + \
 622                 ' [ temp dir used by test case: ' + test.tempdir + ' ]'
 623         else:
 624             self.result_string = colorize("ERROR", RED) + ' [no temp dir]'
 625
 626     def getDescription(self, test):
 627         """
 628         Get test description
 629
 630         :param test:
 631         :returns: test description
 632
 633         """
 634         # TODO: if none print warning not raise exception
 635         short_description = test.shortDescription()
 636         if self.descriptions and short_description:
 637             return short_description
 638         else:
 639             return str(test)
 640
 641     def startTest(self, test):
 642         """
 643         Start a test
 644
 645         :param test:
 646
 647         """
 648         unittest.TestResult.startTest(self, test)
 649         if self.verbosity > 0:
 650             self.stream.writeln(
 651                 "Starting " + self.getDescription(test) + " ...")
 652             self.stream.writeln(single_line_delim)
 653
 654     def stopTest(self, test):
 655         """
 656         Stop a test
 657
 658         :param test:
 659
 660         """
 661         unittest.TestResult.stopTest(self, test)
 662         if self.verbosity > 0:
 663             self.stream.writeln(single_line_delim)
 664             self.stream.writeln("%-60s%s" %
 665                                 (self.getDescription(test), self.result_string))
 666             self.stream.writeln(single_line_delim)
 667         else:
 668             self.stream.writeln("%-60s%s" %
 669                                 (self.getDescription(test), self.result_string))
 670
 671     def printErrors(self):
 672         """
 673         Print errors from running the test case
 674         """
 675         self.stream.writeln()
 676         self.printErrorList('ERROR', self.errors)
 677         self.printErrorList('FAIL', self.failures)
 678
 679     def printErrorList(self, flavour, errors):
 680         """
 681         Print error list to the output stream together with error type
 682         and test case description.
 683
 684         :param flavour: error type
 685         :param errors: iterable errors
 686
 687         """
 688         for test, err in errors:
 689             self.stream.writeln(double_line_delim)
 690             self.stream.writeln("%s: %s" %
 691                                 (flavour, self.getDescription(test)))
 692             self.stream.writeln(single_line_delim)
 693             self.stream.writeln("%s" % err)
 694
 695
 696 class VppTestRunner(unittest.TextTestRunner):
 697     """
 698     A basic test runner implementation which prints results on standard error.
 699     """
 700     @property
 701     def resultclass(self):
 702         """Class maintaining the results of the tests"""
 703         return VppTestResult
 704
 705     def run(self, test):
 706         """
 707         Run the tests
 708
 709         :param test:
 710
 711         """
 712         print("Running tests using custom test runner")  # debug message
 713         return super(VppTestRunner, self).run(test)