From: pmikus Date: Tue, 3 May 2016 05:21:15 +0000 (+0200) Subject: Parse robot output.xml for performance reporting X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=commitdiff_plain;h=a881153db68401e040f37262d60b5d5e3cc486ac Parse robot output.xml for performance reporting - JIRA: CSIT-58 - parse robot framework output.xml file - find performance related data - write formatted json to specified file - copy archive artifact to directory Change-Id: I47e45bcb68c06044a23192cb1fca46f43782941e Signed-off-by: pmikus Signed-off-by: Peter Mikus --- diff --git a/bootstrap-verify-perf.sh b/bootstrap-verify-perf.sh index de478e5f11..16dd536456 100755 --- a/bootstrap-verify-perf.sh +++ b/bootstrap-verify-perf.sh @@ -25,6 +25,8 @@ INSTALLATION_DIR="/tmp/install_dir" PYBOT_ARGS="--noncritical MULTI_THREAD" +ARCHIVE_ARTIFACTS=(log.html output.xml report.html output_perf_data.json) + # If we run this script from CSIT jobs we want to use stable vpp version if [[ ${JOB_NAME} == csit-* ]] ; then @@ -160,3 +162,18 @@ case "$TEST_TAG" in tests/ esac +# Pybot output post-processing +python ${CUR_DIR}/resources/tools/robot_output_parser.py \ + -i ${CUR_DIR}/output.xml \ + -o ${CUR_DIR}/output_perf_data.json \ + -v ${VPP_STABLE_VER} +if [ ! $? -eq 0 ]; then + echo "Parsing ${CUR_DIR}/output.xml failed" +fi + +# Archive artifacts +mkdir archive +for i in ${ARCHIVE_ARTIFACTS[@]}; do + cp $( readlink -f ${i} | tr '\n' ' ' ) archive/ +done + diff --git a/resources/tools/robot_output_parser.py b/resources/tools/robot_output_parser.py new file mode 100755 index 0000000000..28a9f268ef --- /dev/null +++ b/resources/tools/robot_output_parser.py @@ -0,0 +1,161 @@ +#!/usr/bin/python + +# Copyright (c) 2016 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script parses the data taken by robot framework (output.xml) and dumps +intereted values into JSON output file.""" + +import json +import re +import sys, getopt + +from robot.api import ExecutionResult, ResultVisitor + + +class ExecutionTestChecker(ResultVisitor): + """Iterates through test cases.""" + + def __init__(self, vDeviceVersion): + self.vDeviceVersion = vDeviceVersion + self.out = [] + + def visit_test(self, test): + """Overloaded function. Called when test is found to process data. + + :param test: Test to process. + :type test: ExecutionTestChecker + """ + + test_id = test.longname + test_status = 'Failed' + framesize = '' + throughput = '' + throughput_units = '' + workers_per_nic = '' + workers = '' + + if any("PERFTEST" in tag for tag in test.tags): + if test.status == 'PASS': + test_status = 'Passed' + if any("PERFTEST_LONG" in tag for tag in test.tags): + throughput = test.message.split(' ')[1] + throughput_units = test.message.split(' ')[2] + elif any("PERFTEST_SHORT" in tag for tag in test.tags): + for keyword in test.keywords: + for assign in keyword.assign: + if assign == '${rate}': + temp = re.findall(r"(\d*\.\d+|\d+)([A-Za-z]*)", + keyword.args[0]) + throughput = temp[0][0] + throughput_units = temp[0][1] + + for keyword in test.keywords: + for assign in keyword.assign: + if assign == '${framesize}': + framesize = keyword.args[0] + if 'worker threads' in keyword.name: + workers = keyword.name.split('\'')[1] + workers_per_nic = keyword.name.split('\'')[3] + + self.out.append({'testCase': { + 'testId': test_id, + 'testStatus': test_status, + 'workerThreads': workers, + 'workerThreadsPerNic': workers_per_nic, + 'testTags': [tag for tag in test.tags], + 'l2FrameSize': {'value': framesize, + 'units': 'Bytes'}, + 'throughput': {'value': throughput, + 'units': throughput_units}, + 'vDevice': {'version': self.vDeviceVersion}}}) + + +def parse_tests(xml_file, vDeviceVersion): + """Parser result of robot output file and return. + + :param xml_file: Output.xml from robot run. + :param vDeviceVersion: vDevice version. + :type xml_file: file + :type vDeviceVersion: str + + :return: JSON formatted output. + :rtype: dict + """ + + result = ExecutionResult(xml_file) + checker = ExecutionTestChecker(vDeviceVersion) + result.visit(checker) + + return checker.out + + +def print_help(): + """Print help on stdout.""" + + print "args: [-h] -i -o " + \ + " -v " + + +def print_error(msg): + """Print error message on stderr. + + :param msg: Error message to print. + :type msg: str + :return: nothing + """ + + sys.stderr.write(msg+'\n') + + +def main(argv): + """Main function.""" + + _log_file = None + _json_file = None + _vpp = None + + try: + opts, _ = getopt.getopt(argv, "hi:o:v:", ["help"]) + except getopt.GetoptError: + print_help() + sys.exit(1) + + for opt, arg in opts: + if opt in ('-h', "--help"): + print_help() + sys.exit() + elif opt == '-i': + _log_file = arg + elif opt == '-o': + _json_file = arg + elif opt == '-v': + _vpp = arg + + if _log_file is None or _json_file is None or _vpp is None: + print_help() + sys.exit(1) + + try: + with open(_log_file, 'r') as input_file: + with open(_json_file, 'w') as output_file: + out = parse_tests(input_file, _vpp) + json.dump(out, fp=output_file, sort_keys=True, + indent=4, separators=(',', ': ')) + except IOError as ex_error: + print_error(str(ex_error)) + sys.exit(1) + + +if __name__ == "__main__": + main(sys.argv[1:])