From: selias Date: Mon, 14 Nov 2016 12:57:40 +0000 (+0100) Subject: CSIT-454: HC Test: Improve logging on test failure X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=commitdiff_plain;h=7a0a0ec3986ad03444cd4864ae8d35019007be60 CSIT-454: HC Test: Improve logging on test failure - set Honeycomb log level to TRACE before running Honeycomb tests - on a failed restconf request, print the last few lines of the log - show saved configuration during persistence suite restarts Change-Id: I07ee37a5403cfca946fd7ef25a42861273248fbe Signed-off-by: selias --- diff --git a/resources/libraries/python/honeycomb/HcPersistence.py b/resources/libraries/python/honeycomb/HcPersistence.py index 3bbc52fa91..c76a276098 100644 --- a/resources/libraries/python/honeycomb/HcPersistence.py +++ b/resources/libraries/python/honeycomb/HcPersistence.py @@ -83,3 +83,22 @@ class HcPersistence(object): if ret_code != 0: raise HoneycombError("Failed to modify persistence file on node" " {0}, {1}".format(node, stderr)) + + @staticmethod + def log_persisted_configuration(node): + """Read contents of Honeycomb persistence files and print them to log. + + :param node: Honeycomb node. + :type node: dict + """ + + commands = [ + "cat {0}/config/data.json".format(Const.REMOTE_HC_PERSIST), + "cat {0}/context/data.json".format(Const.REMOTE_HC_PERSIST), + ] + + ssh = SSH() + ssh.connect(node) + for command in commands: + (_, stdout, _) = ssh.exec_command_sudo(command) + logger.info(stdout) diff --git a/resources/libraries/python/honeycomb/HoneycombSetup.py b/resources/libraries/python/honeycomb/HoneycombSetup.py index 8a05ad7583..06b35db8a1 100644 --- a/resources/libraries/python/honeycomb/HoneycombSetup.py +++ b/resources/libraries/python/honeycomb/HoneycombSetup.py @@ -243,8 +243,7 @@ class HoneycombSetup(object): "which java", "java -version", "dpkg --list | grep openjdk", - "ls -la /opt/honeycomb", - "ls -la /opt/honeycomb/v3po-karaf-1.0.0-SNAPSHOT") + "ls -la /opt/honeycomb") for node in nodes: if node['type'] == NodeType.DUT: @@ -274,3 +273,26 @@ class HoneycombSetup(object): ssh.connect(node) ssh.exec_command_sudo(cmd) + @staticmethod + def configure_log_level(node, level): + """Set Honeycomb logging to the specified level. + + :param node: Honeycomb node. + :param level: Log level (INFO, DEBUG, TRACE). + :type node: dict + :type level: str + """ + + find = 'logger name=\\"io.fd\\"' + replace = ''.format(level) + + argument = '"/{0}/c\\ {1}"'.format(find, replace) + path = "{0}/config/logback.xml".format(Const.REMOTE_HC_DIR) + command = "sed -i {0} {1}".format(argument, path) + + ssh = SSH() + ssh.connect(node) + (ret_code, _, stderr) = ssh.exec_command_sudo(command) + if ret_code != 0: + raise HoneycombError("Failed to modify configuration on " + "node {0}, {1}".format(node, stderr)) diff --git a/resources/libraries/python/honeycomb/HoneycombUtil.py b/resources/libraries/python/honeycomb/HoneycombUtil.py index f2f012ea9b..22a7e7ef1e 100644 --- a/resources/libraries/python/honeycomb/HoneycombUtil.py +++ b/resources/libraries/python/honeycomb/HoneycombUtil.py @@ -27,7 +27,8 @@ from enum import Enum, unique from robot.api import logger -from resources.libraries.python.HTTPRequest import HTTPRequest +from resources.libraries.python.ssh import SSH +from resources.libraries.python.HTTPRequest import HTTPRequest, HTTPCodes from resources.libraries.python.constants import Constants as Const @@ -301,7 +302,10 @@ class HoneycombUtil(object): base_path = HoneycombUtil.read_path_from_url_file(url_file) path = base_path + path status_code, resp = HTTPRequest.get(node, path) - return status_code, loads(resp) + (status_node, response) = status_code, loads(resp) + if status_code != HTTPCodes.OK: + HoneycombUtil.read_log_tail(node) + return status_code, response @staticmethod def put_honeycomb_data(node, url_file, data, path="", @@ -339,8 +343,12 @@ class HoneycombUtil(object): base_path = HoneycombUtil.read_path_from_url_file(url_file) path = base_path + path logger.trace(path) - return HTTPRequest.put(node=node, path=path, headers=header, - payload=data) + (status_code, response) = HTTPRequest.put( + node=node, path=path, headers=header, payload=data) + + if status_code not in (HTTPCodes.OK, HTTPCodes.ACCEPTED): + HoneycombUtil.read_log_tail(node) + return status_code, response @staticmethod def post_honeycomb_data(node, url_file, data=None, @@ -375,8 +383,12 @@ class HoneycombUtil(object): data = dumps(data) path = HoneycombUtil.read_path_from_url_file(url_file) - return HTTPRequest.post(node=node, path=path, headers=header, - payload=data, timeout=timeout) + (status_code, response) = HTTPRequest.post( + node=node, path=path, headers=header, payload=data, timeout=timeout) + + if status_code not in (HTTPCodes.OK, HTTPCodes.ACCEPTED): + HoneycombUtil.read_log_tail(node) + return status_code, response @staticmethod def delete_honeycomb_data(node, url_file, path=""): @@ -395,4 +407,33 @@ class HoneycombUtil(object): base_path = HoneycombUtil.read_path_from_url_file(url_file) path = base_path + path - return HTTPRequest.delete(node, path) + (status_code, response) = HTTPRequest.delete(node, path) + + if status_code != HTTPCodes.OK: + HoneycombUtil.read_log_tail(node) + return status_code, response + + @staticmethod + def read_log_tail(node, lines=120): + """Read the last N lines of the Honeycomb log file and print them + to robot log. + + :param node: Honeycomb node. + :param lines: Number of lines to read. + :type node: dict + :type lines: int + :return: Last N log lines. + :rtype: str + """ + + logger.trace( + "HTTP request failed, " + "obtaining last {0} lines of Honeycomb log...".format(lines)) + + ssh = SSH() + ssh.connect(node) + cmd = "tail -n {0} /var/log/honeycomb/honeycomb.log".format(lines) + # ssh also logs the reply on trace level + (_, stdout, _) = ssh.exec_command(cmd, timeout=30) + + return stdout diff --git a/resources/libraries/robot/honeycomb/interfaces.robot b/resources/libraries/robot/honeycomb/interfaces.robot index a3c8a13b1b..9940966fb9 100644 --- a/resources/libraries/robot/honeycomb/interfaces.robot +++ b/resources/libraries/robot/honeycomb/interfaces.robot @@ -237,7 +237,7 @@ | IPv4 neighbor from Honeycomb should be | | [Documentation] | Retrieves ipv4 neighbor list through Honeycomb\ -| | and compares with neighbor list supplied in argument. +| | ... | and compares with neighbor list supplied in argument. | | ... | | ... | *Arguments:* | | ... | - node - information about a DUT node. Type: dictionary diff --git a/resources/libraries/robot/honeycomb/persistence.robot b/resources/libraries/robot/honeycomb/persistence.robot index 6d2cc1f2e3..84607a8991 100644 --- a/resources/libraries/robot/honeycomb/persistence.robot +++ b/resources/libraries/robot/honeycomb/persistence.robot @@ -37,6 +37,7 @@ | | ... | \| Honeycomb is restarted \| ${nodes['DUT1']} \| | | [Arguments] | ${node} | | Stop Honeycomb service on DUTs | ${node} +| | Log Persisted Configuration | ${node} | | Setup Honeycomb service on DUTs | ${node} | VPP is restarted @@ -78,6 +79,7 @@ | | ... | \| Honeycomb and VPP are restarted \| ${nodes['DUT1']} \| | | [Arguments] | ${node} | | Stop Honeycomb service on DUTs | ${node} +| | Log Persisted Configuration | ${node} | | Setup DUT | ${node} | | Setup Honeycomb service on DUTs | ${node} diff --git a/tests/func/honeycomb/__init__.robot b/tests/func/honeycomb/__init__.robot index a38d53a464..6a553ac790 100644 --- a/tests/func/honeycomb/__init__.robot +++ b/tests/func/honeycomb/__init__.robot @@ -22,6 +22,7 @@ | Suite Setup | Run Keywords | Setup All DUTs Before Test | AND | ... | Clear Persisted Honeycomb Configuration | ${node} | AND | ... | Configure Unsecured Access | ${node} | AND +| ... | Configure Log Level | ${node} | TRACE | AND | ... | Setup Honeycomb Service On DUTs | ${node} | AND | ... | Set Global Variable | ${node} | Suite Teardown | Stop Honeycomb Service On DUTs | ${node}