1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
24 import xml.etree.ElementTree as ET
26 from robot.api import ExecutionResult, ResultVisitor
27 from robot import errors
28 from collections import OrderedDict
29 from string import replace
33 class ExecutionChecker(ResultVisitor):
34 """Class to traverse through the test suite structure.
36 The functionality implemented in this class generates a json structure:
41 "metadata": { # Optional
42 "version": "VPP version",
43 "job": "Jenkins job name",
44 "build": "Information about the build"
48 "doc": "Suite 1 documentation",
49 "parent": "Suite 1 parent",
50 "level": "Level of the suite in the suite hierarchy"
53 "doc": "Suite N documentation",
54 "parent": "Suite 2 parent",
55 "level": "Level of the suite in the suite hierarchy"
61 "parent": "Name of the parent of the test",
62 "doc": "Test documentation"
64 "tags": ["tag 1", "tag 2", "tag n"],
65 "type": "PDR" | "NDR",
68 "unit": "pps" | "bps" | "percentage"
77 "50": { # Only for NDR
82 "10": { # Only for NDR
94 "50": { # Only for NDR
99 "10": { # Only for NDR
106 "lossTolerance": "lossTolerance", # Only for PDR
107 "vat-history": "DUT1 and DUT2 VAT History"
109 "show-run": "Show Run"
121 "metadata": { # Optional
122 "version": "VPP version",
123 "job": "Jenkins job name",
124 "build": "Information about the build"
128 "doc": "Suite 1 documentation",
129 "parent": "Suite 1 parent",
130 "level": "Level of the suite in the suite hierarchy"
133 "doc": "Suite N documentation",
134 "parent": "Suite 2 parent",
135 "level": "Level of the suite in the suite hierarchy"
141 "parent": "Name of the parent of the test",
142 "doc": "Test documentation"
143 "msg": "Test message"
144 "tags": ["tag 1", "tag 2", "tag n"],
145 "vat-history": "DUT1 and DUT2 VAT History"
146 "show-run": "Show Run"
147 "status": "PASS" | "FAIL"
155 .. note:: ID is the lowercase full path to the test.
158 REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
160 REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
161 r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+/-?\d+)\','
162 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
163 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
164 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
165 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
166 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
168 REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
169 r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
170 r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
172 REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
175 REGEX_VERSION = re.compile(r"(return STDOUT Version:\s*)(.*)")
177 REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
179 REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
180 r'tx\s(\d*),\srx\s(\d*)')
182 def __init__(self, **metadata):
185 :param metadata: Key-value pairs to be included in "metadata" part of
190 # Type of message to parse out from the test messages
191 self._msg_type = None
196 # Number of VAT History messages found:
198 # 1 - VAT History of DUT1
199 # 2 - VAT History of DUT2
200 self._lookup_kw_nr = 0
201 self._vat_history_lookup_nr = 0
203 # Number of Show Running messages found
205 # 1 - Show run message found
206 self._show_run_lookup_nr = 0
208 # Test ID of currently processed test- the lowercase full path to the
212 # The main data structure
214 "metadata": OrderedDict(),
215 "suites": OrderedDict(),
216 "tests": OrderedDict()
219 # Save the provided metadata
220 for key, val in metadata.items():
221 self._data["metadata"][key] = val
223 # Dictionary defining the methods used to parse different types of
226 "setup-version": self._get_version,
227 "teardown-vat-history": self._get_vat_history,
228 "test-show-runtime": self._get_show_run
233 """Getter - Data parsed from the XML file.
235 :returns: Data parsed from the XML file.
240 def _get_version(self, msg):
241 """Called when extraction of VPP version is required.
243 :param msg: Message to process.
248 if msg.message.count("return STDOUT Version:"):
249 self._version = str(re.search(self.REGEX_VERSION, msg.message).
251 self._data["metadata"]["version"] = self._version
252 self._msg_type = None
254 logging.info(" VPP version: {0}".format(self._version))
256 def _get_vat_history(self, msg):
257 """Called when extraction of VAT command history is required.
259 :param msg: Message to process.
263 if msg.message.count("VAT command history:"):
264 self._vat_history_lookup_nr += 1
265 if self._vat_history_lookup_nr == 1:
266 self._data["tests"][self._test_ID]["vat-history"] = str()
268 self._msg_type = None
269 text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
270 "VAT command history:", "", msg.message, count=1). \
271 replace("\n\n", "\n").replace('\n', ' |br| ').\
272 replace('\r', '').replace('"', "'")
274 self._data["tests"][self._test_ID]["vat-history"] += " |br| "
275 self._data["tests"][self._test_ID]["vat-history"] += \
276 "**DUT" + str(self._vat_history_lookup_nr) + ":** " + text
278 def _get_show_run(self, msg):
279 """Called when extraction of VPP operational data (output of CLI command
280 Show Runtime) is required.
282 :param msg: Message to process.
286 if msg.message.count("return STDOUT Thread "):
287 self._show_run_lookup_nr += 1
288 if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
289 self._data["tests"][self._test_ID]["show-run"] = str()
290 if self._lookup_kw_nr > 1:
291 self._msg_type = None
292 if self._show_run_lookup_nr == 1:
293 text = msg.message.replace("vat# ", "").\
294 replace("return STDOUT ", "").replace("\n\n", "\n").\
295 replace('\n', ' |br| ').\
296 replace('\r', '').replace('"', "'")
298 self._data["tests"][self._test_ID]["show-run"] += " |br| "
299 self._data["tests"][self._test_ID]["show-run"] += \
300 "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
304 def _get_latency(self, msg, test_type):
305 """Get the latency data from the test message.
307 :param msg: Message to be parsed.
308 :param test_type: Type of the test - NDR or PDR.
311 :returns: Latencies parsed from the message.
315 if test_type == "NDR":
316 groups = re.search(self.REGEX_LAT_NDR, msg)
317 groups_range = range(1, 7)
318 elif test_type == "PDR":
319 groups = re.search(self.REGEX_LAT_PDR, msg)
320 groups_range = range(1, 3)
325 for idx in groups_range:
327 lat = [int(item) for item in str(groups.group(idx)).split('/')]
328 except (AttributeError, ValueError):
330 latencies.append(lat)
332 keys = ("min", "avg", "max")
340 latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
341 latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
342 if test_type == "NDR":
343 latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
344 latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
345 latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
346 latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
350 def visit_suite(self, suite):
351 """Implements traversing through the suite and its direct children.
353 :param suite: Suite to process.
357 if self.start_suite(suite) is not False:
358 suite.suites.visit(self)
359 suite.tests.visit(self)
360 self.end_suite(suite)
362 def start_suite(self, suite):
363 """Called when suite starts.
365 :param suite: Suite to process.
371 parent_name = suite.parent.name
372 except AttributeError:
375 doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
376 replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
377 doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
379 self._data["suites"][suite.longname.lower().replace('"', "'").
380 replace(" ", "_")] = {
381 "name": suite.name.lower(),
383 "parent": parent_name,
384 "level": len(suite.longname.split("."))
387 suite.keywords.visit(self)
389 def end_suite(self, suite):
390 """Called when suite ends.
392 :param suite: Suite to process.
398 def visit_test(self, test):
399 """Implements traversing through the test.
401 :param test: Test to process.
405 if self.start_test(test) is not False:
406 test.keywords.visit(self)
409 def start_test(self, test):
410 """Called when test starts.
412 :param test: Test to process.
417 tags = [str(tag) for tag in test.tags]
419 test_result["name"] = test.name.lower()
420 test_result["parent"] = test.parent.name.lower()
421 test_result["tags"] = tags
422 doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
423 replace('\r', '').replace('[', ' |br| [')
424 test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
425 test_result["msg"] = test.message.replace('\n', ' |br| '). \
426 replace('\r', '').replace('"', "'")
427 if test.status == "PASS" and ("NDRPDRDISC" in tags or
430 if "NDRDISC" in tags:
432 elif "PDRDISC" in tags:
441 test_result["type"] = test_type
443 if test_type in ("NDR", "PDR"):
445 rate_value = str(re.search(
446 self.REGEX_RATE, test.message).group(1))
447 except AttributeError:
450 rate_unit = str(re.search(
451 self.REGEX_RATE, test.message).group(2))
452 except AttributeError:
455 test_result["throughput"] = dict()
456 test_result["throughput"]["value"] = \
457 int(rate_value.split('.')[0])
458 test_result["throughput"]["unit"] = rate_unit
459 test_result["latency"] = \
460 self._get_latency(test.message, test_type)
461 if test_type == "PDR":
462 test_result["lossTolerance"] = str(re.search(
463 self.REGEX_TOLERANCE, test.message).group(1))
465 elif test_type in ("TCP", ):
466 groups = re.search(self.REGEX_TCP, test.message)
467 test_result["result"] = dict()
468 test_result["result"]["value"] = int(groups.group(2))
469 test_result["result"]["unit"] = groups.group(1)
470 elif test_type in ("MRR", ):
471 groups = re.search(self.REGEX_MRR, test.message)
472 test_result["result"] = dict()
473 test_result["result"]["duration"] = int(groups.group(1))
474 test_result["result"]["tx"] = int(groups.group(2))
475 test_result["result"]["rx"] = int(groups.group(3))
476 test_result["result"]["throughput"] = int(
477 test_result["result"]["rx"] /
478 test_result["result"]["duration"])
480 test_result["status"] = test.status
482 self._test_ID = test.longname.lower()
483 self._data["tests"][self._test_ID] = test_result
485 def end_test(self, test):
486 """Called when test ends.
488 :param test: Test to process.
494 def visit_keyword(self, keyword):
495 """Implements traversing through the keyword and its child keywords.
497 :param keyword: Keyword to process.
498 :type keyword: Keyword
501 if self.start_keyword(keyword) is not False:
502 self.end_keyword(keyword)
504 def start_keyword(self, keyword):
505 """Called when keyword starts. Default implementation does nothing.
507 :param keyword: Keyword to process.
508 :type keyword: Keyword
512 if keyword.type == "setup":
513 self.visit_setup_kw(keyword)
514 elif keyword.type == "teardown":
515 self._lookup_kw_nr = 0
516 self.visit_teardown_kw(keyword)
518 self._lookup_kw_nr = 0
519 self.visit_test_kw(keyword)
520 except AttributeError:
523 def end_keyword(self, keyword):
524 """Called when keyword ends. Default implementation does nothing.
526 :param keyword: Keyword to process.
527 :type keyword: Keyword
532 def visit_test_kw(self, test_kw):
533 """Implements traversing through the test keyword and its child
536 :param test_kw: Keyword to process.
537 :type test_kw: Keyword
540 for keyword in test_kw.keywords:
541 if self.start_test_kw(keyword) is not False:
542 self.visit_test_kw(keyword)
543 self.end_test_kw(keyword)
545 def start_test_kw(self, test_kw):
546 """Called when test keyword starts. Default implementation does
549 :param test_kw: Keyword to process.
550 :type test_kw: Keyword
553 if test_kw.name.count("Show Runtime Counters On All Duts"):
554 self._lookup_kw_nr += 1
555 self._show_run_lookup_nr = 0
556 self._msg_type = "test-show-runtime"
557 test_kw.messages.visit(self)
559 def end_test_kw(self, test_kw):
560 """Called when keyword ends. Default implementation does nothing.
562 :param test_kw: Keyword to process.
563 :type test_kw: Keyword
568 def visit_setup_kw(self, setup_kw):
569 """Implements traversing through the teardown keyword and its child
572 :param setup_kw: Keyword to process.
573 :type setup_kw: Keyword
576 for keyword in setup_kw.keywords:
577 if self.start_setup_kw(keyword) is not False:
578 self.visit_setup_kw(keyword)
579 self.end_setup_kw(keyword)
581 def start_setup_kw(self, setup_kw):
582 """Called when teardown keyword starts. Default implementation does
585 :param setup_kw: Keyword to process.
586 :type setup_kw: Keyword
589 if setup_kw.name.count("Show Vpp Version On All Duts") \
590 and not self._version:
591 self._msg_type = "setup-version"
592 setup_kw.messages.visit(self)
594 def end_setup_kw(self, setup_kw):
595 """Called when keyword ends. Default implementation does nothing.
597 :param setup_kw: Keyword to process.
598 :type setup_kw: Keyword
603 def visit_teardown_kw(self, teardown_kw):
604 """Implements traversing through the teardown keyword and its child
607 :param teardown_kw: Keyword to process.
608 :type teardown_kw: Keyword
611 for keyword in teardown_kw.keywords:
612 if self.start_teardown_kw(keyword) is not False:
613 self.visit_teardown_kw(keyword)
614 self.end_teardown_kw(keyword)
616 def start_teardown_kw(self, teardown_kw):
617 """Called when teardown keyword starts. Default implementation does
620 :param teardown_kw: Keyword to process.
621 :type teardown_kw: Keyword
625 if teardown_kw.name.count("Show Vat History On All Duts"):
626 self._vat_history_lookup_nr = 0
627 self._msg_type = "teardown-vat-history"
628 teardown_kw.messages.visit(self)
630 def end_teardown_kw(self, teardown_kw):
631 """Called when keyword ends. Default implementation does nothing.
633 :param teardown_kw: Keyword to process.
634 :type teardown_kw: Keyword
639 def visit_message(self, msg):
640 """Implements visiting the message.
642 :param msg: Message to process.
646 if self.start_message(msg) is not False:
647 self.end_message(msg)
649 def start_message(self, msg):
650 """Called when message starts. Get required information from messages:
653 :param msg: Message to process.
659 self.parse_msg[self._msg_type](msg)
661 def end_message(self, msg):
662 """Called when message ends. Default implementation does nothing.
664 :param msg: Message to process.
671 class InputData(object):
674 The data is extracted from output.xml files generated by Jenkins jobs and
675 stored in pandas' DataFrames.
686 - ID: test data (as described in ExecutionChecker documentation)
689 def __init__(self, spec):
692 :param spec: Specification.
693 :type spec: Specification
700 self._input_data = None
704 """Getter - Input data.
707 :rtype: pandas.Series
709 return self._input_data
711 def metadata(self, job, build):
714 :param job: Job which metadata we want.
715 :param build: Build which metadata we want.
719 :rtype: pandas.Series
722 return self.data[job][build]["metadata"]
724 def suites(self, job, build):
727 :param job: Job which suites we want.
728 :param build: Build which suites we want.
732 :rtype: pandas.Series
735 return self.data[job][str(build)]["suites"]
737 def tests(self, job, build):
740 :param job: Job which tests we want.
741 :param build: Build which tests we want.
745 :rtype: pandas.Series
748 return self.data[job][build]["tests"]
751 def _parse_tests(job, build):
752 """Process data from robot output.xml file and return JSON structured
755 :param job: The name of job which build output data will be processed.
756 :param build: The build which output data will be processed.
759 :returns: JSON data structure.
763 tree = ET.parse(build["file-name"])
764 root = tree.getroot()
765 generated = root.attrib["generated"]
767 with open(build["file-name"], 'r') as data_file:
769 result = ExecutionResult(data_file)
770 except errors.DataError as err:
771 logging.error("Error occurred while parsing output.xml: {0}".
774 checker = ExecutionChecker(job=job, build=build, generated=generated)
775 result.visit(checker)
780 """Parse input data from input files and store in pandas' Series.
783 logging.info("Parsing input files ...")
786 for job, builds in self._cfg.builds.items():
787 logging.info(" Extracting data from the job '{0}' ...'".
791 if build["status"] == "failed" \
792 or build["status"] == "not found":
794 logging.info(" Extracting data from the build '{0}'".
795 format(build["build"]))
796 logging.info(" Processing the file '{0}'".
797 format(build["file-name"]))
799 data = InputData._parse_tests(job, build)
801 logging.info(" Removing the file '{0}'".
802 format(build["file-name"]))
804 remove(build["file-name"])
805 build["status"] = "processed"
806 except OSError as err:
807 logging.error(" Cannot remove the file '{0}': {1}".
808 format(build["file-name"], err))
810 logging.error("Input data file from the job '{job}', build "
811 "'{build}' is damaged. Skipped.".
812 format(job=job, build=build["build"]))
815 build_data = pd.Series({
816 "metadata": pd.Series(data["metadata"].values(),
817 index=data["metadata"].keys()),
818 "suites": pd.Series(data["suites"].values(),
819 index=data["suites"].keys()),
820 "tests": pd.Series(data["tests"].values(),
821 index=data["tests"].keys()),
823 builds_data[str(build["build"])] = build_data
824 logging.info(" Done.")
826 job_data[job] = pd.Series(builds_data.values(),
827 index=builds_data.keys())
828 logging.info(" Done.")
830 self._input_data = pd.Series(job_data.values(), index=job_data.keys())
831 logging.info("Done.")
834 def _end_of_tag(tag_filter, start=0, closer="'"):
835 """Return the index of character in the string which is the end of tag.
837 :param tag_filter: The string where the end of tag is being searched.
838 :param start: The index where the searching is stated.
839 :param closer: The character which is the tag closer.
840 :type tag_filter: str
843 :returns: The index of the tag closer.
848 idx_opener = tag_filter.index(closer, start)
849 return tag_filter.index(closer, idx_opener + 1)
854 def _condition(tag_filter):
855 """Create a conditional statement from the given tag filter.
857 :param tag_filter: Filter based on tags from the element specification.
858 :type tag_filter: str
859 :returns: Conditional statement which can be evaluated.
865 index = InputData._end_of_tag(tag_filter, index)
869 tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
871 def filter_data(self, element, params=None, data_set="tests",
872 continue_on_error=False):
873 """Filter required data from the given jobs and builds.
875 The output data structure is:
892 :param element: Element which will use the filtered data.
893 :param params: Parameters which will be included in the output. If None,
894 all parameters are included.
895 :param data_set: The set of data to be filtered: tests, suites,
897 :param continue_on_error: Continue if there is error while reading the
898 data. The Item will be empty then
899 :type element: pandas.Series
902 :type continue_on_error: bool
903 :returns: Filtered data.
907 logging.info(" Creating the data set for the {0} '{1}'.".
908 format(element.get("type", ""), element.get("title", "")))
911 if element["filter"] in ("all", "template"):
914 cond = InputData._condition(element["filter"])
915 logging.debug(" Filter: {0}".format(cond))
917 logging.error(" No filter defined.")
921 params = element.get("parameters", None)
925 for job, builds in element["data"].items():
926 data[job] = pd.Series()
928 data[job][str(build)] = pd.Series()
930 data_iter = self.data[job][str(build)][data_set].\
933 if continue_on_error:
937 for test_ID, test_data in data_iter:
938 if eval(cond, {"tags": test_data.get("tags", "")}):
939 data[job][str(build)][test_ID] = pd.Series()
941 for param, val in test_data.items():
942 data[job][str(build)][test_ID][param] = val
946 data[job][str(build)][test_ID][param] =\
949 data[job][str(build)][test_ID][param] =\
953 except (KeyError, IndexError, ValueError) as err:
954 logging.error(" Missing mandatory parameter in the element "
955 "specification: {0}".format(err))
957 except AttributeError:
960 logging.error(" The filter '{0}' is not correct. Check if all "
961 "tags are enclosed by apostrophes.".format(cond))
965 def merge_data(data):
966 """Merge data from more jobs and builds to a simple data structure.
968 The output data structure is:
979 :param data: Data to merge.
980 :type data: pandas.Series
981 :returns: Merged data.
982 :rtype: pandas.Series
985 logging.info(" Merging data ...")
987 merged_data = pd.Series()
988 for _, builds in data.iteritems():
989 for _, item in builds.iteritems():
990 for ID, item_data in item.iteritems():
991 merged_data[ID] = item_data