X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=test%2Fframework.py;h=008bda3b7ceb8e47eb0b70a5c5be950336acbab3;hb=f413bef1358e014c9a6cb75bd2ec3e1f351e64ff;hp=fc263e703eb748ef16125bb55d47d24692e01413;hpb=3747c75a215f082bc52198a7229e1b1e529d7666;p=vpp.git diff --git a/test/framework.py b/test/framework.py index fc263e703eb..008bda3b7ce 100644 --- a/test/framework.py +++ b/test/framework.py @@ -9,9 +9,10 @@ import unittest import tempfile import time import resource +import faulthandler from collections import deque from threading import Thread, Event -from inspect import getdoc +from inspect import getdoc, isclass from traceback import format_exception from logging import FileHandler, DEBUG, Formatter from scapy.packet import Raw @@ -91,6 +92,43 @@ def running_extended_tests(): return False +class KeepAliveReporter(object): + """ + Singleton object which reports test start to parent process + """ + _shared_state = {} + + def __init__(self): + self.__dict__ = self._shared_state + + @property + def pipe(self): + return self._pipe + + @pipe.setter + def pipe(self, pipe): + if hasattr(self, '_pipe'): + raise Exception("Internal error - pipe should only be set once.") + self._pipe = pipe + + def send_keep_alive(self, test): + """ + Write current test tmpdir & desc to keep-alive pipe to signal liveness + """ + if self.pipe is None: + # if not running forked.. + return + + if isclass(test): + desc = test.__name__ + else: + desc = test.shortDescription() + if not desc: + desc = str(test) + + self.pipe.send((desc, test.vpp_bin, test.tempdir)) + + class VppTestCase(unittest.TestCase): """This subclass is a base class for VPP test cases that are implemented as classes. It provides methods to create and run test case. @@ -237,13 +275,13 @@ class VppTestCase(unittest.TestCase): gc.collect() # run garbage collection first cls.logger = getLogger(cls.__name__) cls.tempdir = tempfile.mkdtemp( - prefix='vpp-unittest-' + cls.__name__ + '-') - file_handler = FileHandler("%s/log.txt" % cls.tempdir) - file_handler.setFormatter( + prefix='vpp-unittest-%s-' % cls.__name__) + cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir) + cls.file_handler.setFormatter( Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s', datefmt="%H:%M:%S")) - file_handler.setLevel(DEBUG) - cls.logger.addHandler(file_handler) + cls.file_handler.setLevel(DEBUG) + cls.logger.addHandler(cls.file_handler) cls.shm_prefix = cls.tempdir.split("/")[-1] os.chdir(cls.tempdir) cls.logger.info("Temporary dir is %s, shm prefix is %s", @@ -256,6 +294,8 @@ class VppTestCase(unittest.TestCase): cls.vpp_dead = False cls.registry = VppObjectRegistry() cls.vpp_startup_failed = False + cls.reporter = KeepAliveReporter() + cls.reporter.send_keep_alive(cls) # need to catch exceptions here because if we raise, then the cleanup # doesn't get called and we might end with a zombie vpp try: @@ -344,9 +384,9 @@ class VppTestCase(unittest.TestCase): stdout_log(single_line_delim) stdout_log('VPP output to stdout while running %s:', cls.__name__) stdout_log(single_line_delim) - f = open(cls.tempdir + '/vpp_stdout.txt', 'w') vpp_output = "".join(cls.vpp_stdout_deque) - f.write(vpp_output) + with open(cls.tempdir + '/vpp_stdout.txt', 'w') as f: + f.write(vpp_output) stdout_log('\n%s', vpp_output) stdout_log(single_line_delim) @@ -354,9 +394,9 @@ class VppTestCase(unittest.TestCase): stderr_log(single_line_delim) stderr_log('VPP output to stderr while running %s:', cls.__name__) stderr_log(single_line_delim) - f = open(cls.tempdir + '/vpp_stderr.txt', 'w') vpp_output = "".join(cls.vpp_stderr_deque) - f.write(vpp_output) + with open(cls.tempdir + '/vpp_stderr.txt', 'w') as f: + f.write(vpp_output) stderr_log('\n%s', vpp_output) stderr_log(single_line_delim) @@ -364,6 +404,7 @@ class VppTestCase(unittest.TestCase): def tearDownClass(cls): """ Perform final cleanup after running all tests in this test-case """ cls.quit() + cls.file_handler.close() def tearDown(self): """ Show various debug prints after each test """ @@ -392,6 +433,7 @@ class VppTestCase(unittest.TestCase): def setUp(self): """ Clear trace before running each test""" + self.reporter.send_keep_alive(self) self.logger.debug("--- setUp() for %s.%s(%s) called ---" % (self.__class__.__name__, self._testMethodName, self._testMethodDoc)) @@ -647,8 +689,18 @@ class VppTestCase(unittest.TestCase): @classmethod def sleep(cls, timeout, remark=None): if hasattr(cls, 'logger'): - cls.logger.debug("Sleeping for %ss (%s)" % (timeout, remark)) + cls.logger.debug("Starting sleep for %ss (%s)" % (timeout, remark)) + before = time.time() time.sleep(timeout) + after = time.time() + if after - before > 2 * timeout: + cls.logger.error("unexpected time.sleep() result - " + "slept for %ss instead of ~%ss!" % ( + after - before, timeout)) + if hasattr(cls, 'logger'): + cls.logger.debug( + "Finished sleep (%s) - slept %ss (wanted %ss)" % ( + remark, after - before, timeout)) class TestCasePrinter(object): @@ -729,6 +781,24 @@ class VppTestResult(unittest.TestResult): unittest.TestResult.addSkip(self, test, reason) self.result_string = colorize("SKIP", YELLOW) + def symlink_failed(self, test): + logger = None + if hasattr(test, 'logger'): + logger = test.logger + if hasattr(test, 'tempdir'): + try: + failed_dir = os.getenv('VPP_TEST_FAILED_DIR') + link_path = '%s/%s-FAILED' % (failed_dir, + test.tempdir.split("/")[-1]) + if logger: + logger.debug("creating a link to the failed test") + logger.debug("os.symlink(%s, %s)" % + (test.tempdir, link_path)) + os.symlink(test.tempdir, link_path) + except Exception as e: + if logger: + logger.error(e) + def addFailure(self, test, err): """ Record a test failed result @@ -748,6 +818,7 @@ class VppTestResult(unittest.TestResult): if hasattr(test, 'tempdir'): self.result_string = colorize("FAIL", RED) + \ ' [ temp dir used by test case: ' + test.tempdir + ' ]' + self.symlink_failed(test) else: self.result_string = colorize("FAIL", RED) + ' [no temp dir]' @@ -770,6 +841,7 @@ class VppTestResult(unittest.TestResult): if hasattr(test, 'tempdir'): self.result_string = colorize("ERROR", RED) + \ ' [ temp dir used by test case: ' + test.tempdir + ' ]' + self.symlink_failed(test) else: self.result_string = colorize("ERROR", RED) + ' [no temp dir]' @@ -853,13 +925,15 @@ class VppTestRunner(unittest.TextTestRunner): """Class maintaining the results of the tests""" return VppTestResult - def __init__(self, stream=sys.stderr, descriptions=True, verbosity=1, - failfast=False, buffer=False, resultclass=None): + def __init__(self, pipe=None, stream=sys.stderr, descriptions=True, + verbosity=1, failfast=False, buffer=False, resultclass=None): # ignore stream setting here, use hard-coded stdout to be in sync # with prints from VppTestCase methods ... super(VppTestRunner, self).__init__(sys.stdout, descriptions, verbosity, failfast, buffer, resultclass) + reporter = KeepAliveReporter() + reporter.pipe = pipe test_option = "TEST" @@ -929,7 +1003,7 @@ class VppTestRunner(unittest.TextTestRunner): :param test: """ - gc.disable() # disable garbage collection, we'll do that manually + faulthandler.enable() # emit stack trace to stderr if killed by signal print("Running tests using custom test runner") # debug message filter_file, filter_class, filter_func = self.parse_test_option() print("Active filters: file=%s, class=%s, function=%s" % (