1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
24 import xml.etree.ElementTree as ET
26 from robot.api import ExecutionResult, ResultVisitor
27 from robot import errors
28 from collections import OrderedDict
29 from string import replace
32 class ExecutionChecker(ResultVisitor):
33 """Class to traverse through the test suite structure.
35 The functionality implemented in this class generates a json structure:
40 "metadata": { # Optional
41 "version": "VPP version",
42 "job": "Jenkins job name",
43 "build": "Information about the build"
47 "doc": "Suite 1 documentation",
48 "parent": "Suite 1 parent",
49 "level": "Level of the suite in the suite hierarchy"
52 "doc": "Suite N documentation",
53 "parent": "Suite 2 parent",
54 "level": "Level of the suite in the suite hierarchy"
60 "parent": "Name of the parent of the test",
61 "doc": "Test documentation"
63 "tags": ["tag 1", "tag 2", "tag n"],
64 "type": "PDR" | "NDR",
67 "unit": "pps" | "bps" | "percentage"
76 "50": { # Only for NDR
81 "10": { # Only for NDR
93 "50": { # Only for NDR
98 "10": { # Only for NDR
105 "lossTolerance": "lossTolerance", # Only for PDR
106 "vat-history": "DUT1 and DUT2 VAT History"
108 "show-run": "Show Run"
120 "metadata": { # Optional
121 "version": "VPP version",
122 "job": "Jenkins job name",
123 "build": "Information about the build"
127 "doc": "Suite 1 documentation",
128 "parent": "Suite 1 parent",
129 "level": "Level of the suite in the suite hierarchy"
132 "doc": "Suite N documentation",
133 "parent": "Suite 2 parent",
134 "level": "Level of the suite in the suite hierarchy"
140 "parent": "Name of the parent of the test",
141 "doc": "Test documentation"
142 "msg": "Test message"
143 "tags": ["tag 1", "tag 2", "tag n"],
144 "vat-history": "DUT1 and DUT2 VAT History"
145 "show-run": "Show Run"
146 "status": "PASS" | "FAIL"
154 .. note:: ID is the lowercase full path to the test.
157 REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
159 REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
160 r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+/-?\d+)\','
161 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
162 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
163 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
164 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
165 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
167 REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
168 r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
169 r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
171 REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
174 REGEX_VERSION = re.compile(r"(stdout: 'vat# vat# Version:)(\s*)(.*)")
176 REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
178 REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
179 r'tx\s(\d*),\srx\s(\d*)')
181 def __init__(self, **metadata):
184 :param metadata: Key-value pairs to be included in "metadata" part of
189 # Type of message to parse out from the test messages
190 self._msg_type = None
195 # Number of VAT History messages found:
197 # 1 - VAT History of DUT1
198 # 2 - VAT History of DUT2
199 self._lookup_kw_nr = 0
200 self._vat_history_lookup_nr = 0
202 # Number of Show Running messages found
204 # 1 - Show run message found
205 self._show_run_lookup_nr = 0
207 # Test ID of currently processed test- the lowercase full path to the
211 # The main data structure
213 "metadata": OrderedDict(),
214 "suites": OrderedDict(),
215 "tests": OrderedDict()
218 # Save the provided metadata
219 for key, val in metadata.items():
220 self._data["metadata"][key] = val
222 # Dictionary defining the methods used to parse different types of
225 "setup-version": self._get_version,
226 "teardown-vat-history": self._get_vat_history,
227 "test-show-runtime": self._get_show_run
232 """Getter - Data parsed from the XML file.
234 :returns: Data parsed from the XML file.
239 def _get_version(self, msg):
240 """Called when extraction of VPP version is required.
242 :param msg: Message to process.
247 if msg.message.count("stdout: 'vat# vat# Version:"):
248 self._version = str(re.search(self.REGEX_VERSION, msg.message).
250 self._data["metadata"]["version"] = self._version
251 self._msg_type = None
253 logging.debug(" VPP version: {0}".format(self._version))
255 def _get_vat_history(self, msg):
256 """Called when extraction of VAT command history is required.
258 :param msg: Message to process.
262 if msg.message.count("VAT command history:"):
263 self._vat_history_lookup_nr += 1
264 if self._vat_history_lookup_nr == 1:
265 self._data["tests"][self._test_ID]["vat-history"] = str()
267 self._msg_type = None
268 text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
269 "VAT command history:", "", msg.message, count=1). \
270 replace("\n\n", "\n").replace('\n', ' |br| ').\
271 replace('\r', '').replace('"', "'")
273 self._data["tests"][self._test_ID]["vat-history"] += " |br| "
274 self._data["tests"][self._test_ID]["vat-history"] += \
275 "**DUT" + str(self._vat_history_lookup_nr) + ":** " + text
277 def _get_show_run(self, msg):
278 """Called when extraction of VPP operational data (output of CLI command
279 Show Runtime) is required.
281 :param msg: Message to process.
285 if msg.message.count("return STDOUT Thread "):
286 self._show_run_lookup_nr += 1
287 if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
288 self._data["tests"][self._test_ID]["show-run"] = str()
289 if self._lookup_kw_nr > 1:
290 self._msg_type = None
291 if self._show_run_lookup_nr == 1:
292 text = msg.message.replace("vat# ", "").\
293 replace("return STDOUT ", "").replace("\n\n", "\n").\
294 replace('\n', ' |br| ').\
295 replace('\r', '').replace('"', "'")
297 self._data["tests"][self._test_ID]["show-run"] += " |br| "
298 self._data["tests"][self._test_ID]["show-run"] += \
299 "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
303 def _get_latency(self, msg, test_type):
304 """Get the latency data from the test message.
306 :param msg: Message to be parsed.
307 :param test_type: Type of the test - NDR or PDR.
310 :returns: Latencies parsed from the message.
314 if test_type == "NDR":
315 groups = re.search(self.REGEX_LAT_NDR, msg)
316 groups_range = range(1, 7)
317 elif test_type == "PDR":
318 groups = re.search(self.REGEX_LAT_PDR, msg)
319 groups_range = range(1, 3)
324 for idx in groups_range:
326 lat = [int(item) for item in str(groups.group(idx)).split('/')]
327 except (AttributeError, ValueError):
329 latencies.append(lat)
331 keys = ("min", "avg", "max")
339 latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
340 latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
341 if test_type == "NDR":
342 latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
343 latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
344 latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
345 latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
349 def visit_suite(self, suite):
350 """Implements traversing through the suite and its direct children.
352 :param suite: Suite to process.
356 if self.start_suite(suite) is not False:
357 suite.suites.visit(self)
358 suite.tests.visit(self)
359 self.end_suite(suite)
361 def start_suite(self, suite):
362 """Called when suite starts.
364 :param suite: Suite to process.
370 parent_name = suite.parent.name
371 except AttributeError:
374 doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
375 replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
376 doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
378 self._data["suites"][suite.longname.lower().replace('"', "'").
379 replace(" ", "_")] = {
380 "name": suite.name.lower(),
382 "parent": parent_name,
383 "level": len(suite.longname.split("."))
386 suite.keywords.visit(self)
388 def end_suite(self, suite):
389 """Called when suite ends.
391 :param suite: Suite to process.
397 def visit_test(self, test):
398 """Implements traversing through the test.
400 :param test: Test to process.
404 if self.start_test(test) is not False:
405 test.keywords.visit(self)
408 def start_test(self, test):
409 """Called when test starts.
411 :param test: Test to process.
416 tags = [str(tag) for tag in test.tags]
418 test_result["name"] = test.name.lower()
419 test_result["parent"] = test.parent.name.lower()
420 test_result["tags"] = tags
421 doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
422 replace('\r', '').replace('[', ' |br| [')
423 test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
424 test_result["msg"] = test.message.replace('\n', ' |br| '). \
425 replace('\r', '').replace('"', "'")
426 if test.status == "PASS" and ("NDRPDRDISC" in tags or
429 if "NDRDISC" in tags:
431 elif "PDRDISC" in tags:
440 test_result["type"] = test_type
442 if test_type in ("NDR", "PDR"):
444 rate_value = str(re.search(
445 self.REGEX_RATE, test.message).group(1))
446 except AttributeError:
449 rate_unit = str(re.search(
450 self.REGEX_RATE, test.message).group(2))
451 except AttributeError:
454 test_result["throughput"] = dict()
455 test_result["throughput"]["value"] = \
456 int(rate_value.split('.')[0])
457 test_result["throughput"]["unit"] = rate_unit
458 test_result["latency"] = \
459 self._get_latency(test.message, test_type)
460 if test_type == "PDR":
461 test_result["lossTolerance"] = str(re.search(
462 self.REGEX_TOLERANCE, test.message).group(1))
464 elif test_type in ("TCP", ):
465 groups = re.search(self.REGEX_TCP, test.message)
466 test_result["result"] = dict()
467 test_result["result"]["value"] = int(groups.group(2))
468 test_result["result"]["unit"] = groups.group(1)
469 elif test_type in ("MRR", ):
470 groups = re.search(self.REGEX_MRR, test.message)
471 test_result["result"] = dict()
472 test_result["result"]["duration"] = int(groups.group(1))
473 test_result["result"]["tx"] = int(groups.group(2))
474 test_result["result"]["rx"] = int(groups.group(3))
475 test_result["result"]["throughput"] = int(
476 test_result["result"]["rx"] /
477 test_result["result"]["duration"])
479 test_result["status"] = test.status
481 self._test_ID = test.longname.lower()
482 self._data["tests"][self._test_ID] = test_result
484 def end_test(self, test):
485 """Called when test ends.
487 :param test: Test to process.
493 def visit_keyword(self, keyword):
494 """Implements traversing through the keyword and its child keywords.
496 :param keyword: Keyword to process.
497 :type keyword: Keyword
500 if self.start_keyword(keyword) is not False:
501 self.end_keyword(keyword)
503 def start_keyword(self, keyword):
504 """Called when keyword starts. Default implementation does nothing.
506 :param keyword: Keyword to process.
507 :type keyword: Keyword
511 if keyword.type == "setup":
512 self.visit_setup_kw(keyword)
513 elif keyword.type == "teardown":
514 self._lookup_kw_nr = 0
515 self.visit_teardown_kw(keyword)
517 self._lookup_kw_nr = 0
518 self.visit_test_kw(keyword)
519 except AttributeError:
522 def end_keyword(self, keyword):
523 """Called when keyword ends. Default implementation does nothing.
525 :param keyword: Keyword to process.
526 :type keyword: Keyword
531 def visit_test_kw(self, test_kw):
532 """Implements traversing through the test keyword and its child
535 :param test_kw: Keyword to process.
536 :type test_kw: Keyword
539 for keyword in test_kw.keywords:
540 if self.start_test_kw(keyword) is not False:
541 self.visit_test_kw(keyword)
542 self.end_test_kw(keyword)
544 def start_test_kw(self, test_kw):
545 """Called when test keyword starts. Default implementation does
548 :param test_kw: Keyword to process.
549 :type test_kw: Keyword
552 if test_kw.name.count("Show Runtime Counters On All Duts"):
553 self._lookup_kw_nr += 1
554 self._show_run_lookup_nr = 0
555 self._msg_type = "test-show-runtime"
556 test_kw.messages.visit(self)
558 def end_test_kw(self, test_kw):
559 """Called when keyword ends. Default implementation does nothing.
561 :param test_kw: Keyword to process.
562 :type test_kw: Keyword
567 def visit_setup_kw(self, setup_kw):
568 """Implements traversing through the teardown keyword and its child
571 :param setup_kw: Keyword to process.
572 :type setup_kw: Keyword
575 for keyword in setup_kw.keywords:
576 if self.start_setup_kw(keyword) is not False:
577 self.visit_setup_kw(keyword)
578 self.end_setup_kw(keyword)
580 def start_setup_kw(self, setup_kw):
581 """Called when teardown keyword starts. Default implementation does
584 :param setup_kw: Keyword to process.
585 :type setup_kw: Keyword
588 if setup_kw.name.count("Vpp Show Version Verbose") \
589 and not self._version:
590 self._msg_type = "setup-version"
591 setup_kw.messages.visit(self)
593 def end_setup_kw(self, setup_kw):
594 """Called when keyword ends. Default implementation does nothing.
596 :param setup_kw: Keyword to process.
597 :type setup_kw: Keyword
602 def visit_teardown_kw(self, teardown_kw):
603 """Implements traversing through the teardown keyword and its child
606 :param teardown_kw: Keyword to process.
607 :type teardown_kw: Keyword
610 for keyword in teardown_kw.keywords:
611 if self.start_teardown_kw(keyword) is not False:
612 self.visit_teardown_kw(keyword)
613 self.end_teardown_kw(keyword)
615 def start_teardown_kw(self, teardown_kw):
616 """Called when teardown keyword starts. Default implementation does
619 :param teardown_kw: Keyword to process.
620 :type teardown_kw: Keyword
624 if teardown_kw.name.count("Show Vat History On All Duts"):
625 self._vat_history_lookup_nr = 0
626 self._msg_type = "teardown-vat-history"
627 teardown_kw.messages.visit(self)
629 def end_teardown_kw(self, teardown_kw):
630 """Called when keyword ends. Default implementation does nothing.
632 :param teardown_kw: Keyword to process.
633 :type teardown_kw: Keyword
638 def visit_message(self, msg):
639 """Implements visiting the message.
641 :param msg: Message to process.
645 if self.start_message(msg) is not False:
646 self.end_message(msg)
648 def start_message(self, msg):
649 """Called when message starts. Get required information from messages:
652 :param msg: Message to process.
658 self.parse_msg[self._msg_type](msg)
660 def end_message(self, msg):
661 """Called when message ends. Default implementation does nothing.
663 :param msg: Message to process.
670 class InputData(object):
673 The data is extracted from output.xml files generated by Jenkins jobs and
674 stored in pandas' DataFrames.
685 - ID: test data (as described in ExecutionChecker documentation)
688 def __init__(self, spec):
691 :param spec: Specification.
692 :type spec: Specification
699 self._input_data = None
703 """Getter - Input data.
706 :rtype: pandas.Series
708 return self._input_data
710 def metadata(self, job, build):
713 :param job: Job which metadata we want.
714 :param build: Build which metadata we want.
718 :rtype: pandas.Series
721 return self.data[job][build]["metadata"]
723 def suites(self, job, build):
726 :param job: Job which suites we want.
727 :param build: Build which suites we want.
731 :rtype: pandas.Series
734 return self.data[job][str(build)]["suites"]
736 def tests(self, job, build):
739 :param job: Job which tests we want.
740 :param build: Build which tests we want.
744 :rtype: pandas.Series
747 return self.data[job][build]["tests"]
750 def _parse_tests(job, build):
751 """Process data from robot output.xml file and return JSON structured
754 :param job: The name of job which build output data will be processed.
755 :param build: The build which output data will be processed.
758 :returns: JSON data structure.
762 tree = ET.parse(build["file-name"])
763 root = tree.getroot()
764 generated = root.attrib["generated"]
766 with open(build["file-name"], 'r') as data_file:
768 result = ExecutionResult(data_file)
769 except errors.DataError as err:
770 logging.error("Error occurred while parsing output.xml: {0}".
773 checker = ExecutionChecker(job=job, build=build, generated=generated)
774 result.visit(checker)
779 """Parse input data from input files and store in pandas' Series.
782 logging.info("Parsing input files ...")
785 for job, builds in self._cfg.builds.items():
786 logging.info(" Extracting data from the job '{0}' ...'".
790 if build["status"] == "failed" \
791 or build["status"] == "not found":
793 logging.info(" Extracting data from the build '{0}'".
794 format(build["build"]))
795 logging.info(" Processing the file '{0}'".
796 format(build["file-name"]))
797 data = InputData._parse_tests(job, build)
799 logging.error("Input data file from the job '{job}', build "
800 "'{build}' is damaged. Skipped.".
801 format(job=job, build=build["build"]))
804 build_data = pd.Series({
805 "metadata": pd.Series(data["metadata"].values(),
806 index=data["metadata"].keys()),
807 "suites": pd.Series(data["suites"].values(),
808 index=data["suites"].keys()),
809 "tests": pd.Series(data["tests"].values(),
810 index=data["tests"].keys()),
812 builds_data[str(build["build"])] = build_data
813 logging.info(" Done.")
815 job_data[job] = pd.Series(builds_data.values(),
816 index=builds_data.keys())
817 logging.info(" Done.")
819 self._input_data = pd.Series(job_data.values(), index=job_data.keys())
820 logging.info("Done.")
823 def _end_of_tag(tag_filter, start=0, closer="'"):
824 """Return the index of character in the string which is the end of tag.
826 :param tag_filter: The string where the end of tag is being searched.
827 :param start: The index where the searching is stated.
828 :param closer: The character which is the tag closer.
829 :type tag_filter: str
832 :returns: The index of the tag closer.
837 idx_opener = tag_filter.index(closer, start)
838 return tag_filter.index(closer, idx_opener + 1)
843 def _condition(tag_filter):
844 """Create a conditional statement from the given tag filter.
846 :param tag_filter: Filter based on tags from the element specification.
847 :type tag_filter: str
848 :returns: Conditional statement which can be evaluated.
854 index = InputData._end_of_tag(tag_filter, index)
858 tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
860 def filter_data(self, element, params=None, data_set="tests",
861 continue_on_error=False):
862 """Filter required data from the given jobs and builds.
864 The output data structure is:
881 :param element: Element which will use the filtered data.
882 :param params: Parameters which will be included in the output. If None,
883 all parameters are included.
884 :param data_set: The set of data to be filtered: tests, suites,
886 :param continue_on_error: Continue if there is error while reading the
887 data. The Item will be empty then
888 :type element: pandas.Series
891 :type continue_on_error: bool
892 :returns: Filtered data.
896 logging.info(" Creating the data set for the {0} '{1}'.".
897 format(element.get("type", ""), element.get("title", "")))
900 if element["filter"] in ("all", "template"):
903 cond = InputData._condition(element["filter"])
904 logging.debug(" Filter: {0}".format(cond))
906 logging.error(" No filter defined.")
910 params = element.get("parameters", None)
914 for job, builds in element["data"].items():
915 data[job] = pd.Series()
917 data[job][str(build)] = pd.Series()
919 data_iter = self.data[job][str(build)][data_set].\
922 if continue_on_error:
926 for test_ID, test_data in data_iter:
927 if eval(cond, {"tags": test_data.get("tags", "")}):
928 data[job][str(build)][test_ID] = pd.Series()
930 for param, val in test_data.items():
931 data[job][str(build)][test_ID][param] = val
935 data[job][str(build)][test_ID][param] =\
938 data[job][str(build)][test_ID][param] =\
942 except (KeyError, IndexError, ValueError) as err:
943 logging.error(" Missing mandatory parameter in the element "
944 "specification: {0}".format(err))
946 except AttributeError:
949 logging.error(" The filter '{0}' is not correct. Check if all "
950 "tags are enclosed by apostrophes.".format(cond))
954 def merge_data(data):
955 """Merge data from more jobs and builds to a simple data structure.
957 The output data structure is:
968 :param data: Data to merge.
969 :type data: pandas.Series
970 :returns: Merged data.
971 :rtype: pandas.Series
974 logging.info(" Merging data ...")
976 merged_data = pd.Series()
977 for _, builds in data.iteritems():
978 for _, item in builds.iteritems():
979 for ID, item_data in item.iteritems():
980 merged_data[ID] = item_data