1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
25 from robot.api import ExecutionResult, ResultVisitor
26 from robot import errors
27 from collections import OrderedDict
28 from string import replace
31 class ExecutionChecker(ResultVisitor):
32 """Class to traverse through the test suite structure.
34 The functionality implemented in this class generates a json structure:
39 "metadata": { # Optional
40 "version": "VPP version",
41 "job": "Jenkins job name",
42 "build": "Information about the build"
46 "doc": "Suite 1 documentation",
47 "parent": "Suite 1 parent",
48 "level": "Level of the suite in the suite hierarchy"
51 "doc": "Suite N documentation",
52 "parent": "Suite 2 parent",
53 "level": "Level of the suite in the suite hierarchy"
59 "parent": "Name of the parent of the test",
60 "doc": "Test documentation"
62 "tags": ["tag 1", "tag 2", "tag n"],
63 "type": "PDR" | "NDR",
66 "unit": "pps" | "bps" | "percentage"
75 "50": { # Only for NDR
80 "10": { # Only for NDR
92 "50": { # Only for NDR
97 "10": { # Only for NDR
104 "lossTolerance": "lossTolerance", # Only for PDR
105 "vat-history": "DUT1 and DUT2 VAT History"
107 "show-run": "Show Run"
119 "metadata": { # Optional
120 "version": "VPP version",
121 "job": "Jenkins job name",
122 "build": "Information about the build"
126 "doc": "Suite 1 documentation",
127 "parent": "Suite 1 parent",
128 "level": "Level of the suite in the suite hierarchy"
131 "doc": "Suite N documentation",
132 "parent": "Suite 2 parent",
133 "level": "Level of the suite in the suite hierarchy"
139 "parent": "Name of the parent of the test",
140 "doc": "Test documentation"
141 "msg": "Test message"
142 "tags": ["tag 1", "tag 2", "tag n"],
143 "vat-history": "DUT1 and DUT2 VAT History"
144 "show-run": "Show Run"
145 "status": "PASS" | "FAIL"
153 .. note:: ID is the lowercase full path to the test.
156 REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
158 REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
159 r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+/-?\d+)\','
160 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
161 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
162 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
163 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
164 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
166 REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
167 r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
168 r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
170 REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
173 REGEX_VERSION = re.compile(r"(stdout: 'vat# vat# Version:)(\s*)(.*)")
175 REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
177 REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
178 r'tx\s(\d*),\srx\s(\d*)')
180 def __init__(self, **metadata):
183 :param metadata: Key-value pairs to be included in "metadata" part of
188 # Type of message to parse out from the test messages
189 self._msg_type = None
194 # Number of VAT History messages found:
196 # 1 - VAT History of DUT1
197 # 2 - VAT History of DUT2
198 self._lookup_kw_nr = 0
199 self._vat_history_lookup_nr = 0
201 # Number of Show Running messages found
203 # 1 - Show run message found
204 self._show_run_lookup_nr = 0
206 # Test ID of currently processed test- the lowercase full path to the
210 # The main data structure
212 "metadata": OrderedDict(),
213 "suites": OrderedDict(),
214 "tests": OrderedDict()
217 # Save the provided metadata
218 for key, val in metadata.items():
219 self._data["metadata"][key] = val
221 # Dictionary defining the methods used to parse different types of
224 "setup-version": self._get_version,
225 "teardown-vat-history": self._get_vat_history,
226 "test-show-runtime": self._get_show_run
231 """Getter - Data parsed from the XML file.
233 :returns: Data parsed from the XML file.
238 def _get_version(self, msg):
239 """Called when extraction of VPP version is required.
241 :param msg: Message to process.
246 if msg.message.count("stdout: 'vat# vat# Version:"):
247 self._version = str(re.search(self.REGEX_VERSION, msg.message).
249 self._data["metadata"]["version"] = self._version
250 self._msg_type = None
252 logging.debug(" VPP version: {0}".format(self._version))
254 def _get_vat_history(self, msg):
255 """Called when extraction of VAT command history is required.
257 :param msg: Message to process.
261 if msg.message.count("VAT command history:"):
262 self._vat_history_lookup_nr += 1
263 if self._vat_history_lookup_nr == 1:
264 self._data["tests"][self._test_ID]["vat-history"] = str()
266 self._msg_type = None
267 text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
268 "VAT command history:", "", msg.message, count=1). \
269 replace("\n\n", "\n").replace('\n', ' |br| ').\
270 replace('\r', '').replace('"', "'")
272 self._data["tests"][self._test_ID]["vat-history"] += " |br| "
273 self._data["tests"][self._test_ID]["vat-history"] += \
274 "**DUT" + str(self._vat_history_lookup_nr) + ":** " + text
276 def _get_show_run(self, msg):
277 """Called when extraction of VPP operational data (output of CLI command
278 Show Runtime) is required.
280 :param msg: Message to process.
284 if msg.message.count("return STDOUT Thread "):
285 self._show_run_lookup_nr += 1
286 if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
287 self._data["tests"][self._test_ID]["show-run"] = str()
288 if self._lookup_kw_nr > 1:
289 self._msg_type = None
290 if self._show_run_lookup_nr == 1:
291 text = msg.message.replace("vat# ", "").\
292 replace("return STDOUT ", "").replace("\n\n", "\n").\
293 replace('\n', ' |br| ').\
294 replace('\r', '').replace('"', "'")
296 self._data["tests"][self._test_ID]["show-run"] += " |br| "
297 self._data["tests"][self._test_ID]["show-run"] += \
298 "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
302 def _get_latency(self, msg, test_type):
303 """Get the latency data from the test message.
305 :param msg: Message to be parsed.
306 :param test_type: Type of the test - NDR or PDR.
309 :returns: Latencies parsed from the message.
313 if test_type == "NDR":
314 groups = re.search(self.REGEX_LAT_NDR, msg)
315 groups_range = range(1, 7)
316 elif test_type == "PDR":
317 groups = re.search(self.REGEX_LAT_PDR, msg)
318 groups_range = range(1, 3)
323 for idx in groups_range:
325 lat = [int(item) for item in str(groups.group(idx)).split('/')]
326 except (AttributeError, ValueError):
328 latencies.append(lat)
330 keys = ("min", "avg", "max")
338 latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
339 latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
340 if test_type == "NDR":
341 latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
342 latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
343 latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
344 latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
348 def visit_suite(self, suite):
349 """Implements traversing through the suite and its direct children.
351 :param suite: Suite to process.
355 if self.start_suite(suite) is not False:
356 suite.suites.visit(self)
357 suite.tests.visit(self)
358 self.end_suite(suite)
360 def start_suite(self, suite):
361 """Called when suite starts.
363 :param suite: Suite to process.
369 parent_name = suite.parent.name
370 except AttributeError:
373 doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
374 replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
375 doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
377 self._data["suites"][suite.longname.lower().replace('"', "'").
378 replace(" ", "_")] = {
379 "name": suite.name.lower(),
381 "parent": parent_name,
382 "level": len(suite.longname.split("."))
385 suite.keywords.visit(self)
387 def end_suite(self, suite):
388 """Called when suite ends.
390 :param suite: Suite to process.
396 def visit_test(self, test):
397 """Implements traversing through the test.
399 :param test: Test to process.
403 if self.start_test(test) is not False:
404 test.keywords.visit(self)
407 def start_test(self, test):
408 """Called when test starts.
410 :param test: Test to process.
415 tags = [str(tag) for tag in test.tags]
417 test_result["name"] = test.name.lower()
418 test_result["parent"] = test.parent.name.lower()
419 test_result["tags"] = tags
420 doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
421 replace('\r', '').replace('[', ' |br| [')
422 test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
423 test_result["msg"] = test.message.replace('\n', ' |br| '). \
424 replace('\r', '').replace('"', "'")
425 if test.status == "PASS" and ("NDRPDRDISC" in tags or
428 if "NDRDISC" in tags:
430 elif "PDRDISC" in tags:
439 test_result["type"] = test_type
441 if test_type in ("NDR", "PDR"):
443 rate_value = str(re.search(
444 self.REGEX_RATE, test.message).group(1))
445 except AttributeError:
448 rate_unit = str(re.search(
449 self.REGEX_RATE, test.message).group(2))
450 except AttributeError:
453 test_result["throughput"] = dict()
454 test_result["throughput"]["value"] = \
455 int(rate_value.split('.')[0])
456 test_result["throughput"]["unit"] = rate_unit
457 test_result["latency"] = \
458 self._get_latency(test.message, test_type)
459 if test_type == "PDR":
460 test_result["lossTolerance"] = str(re.search(
461 self.REGEX_TOLERANCE, test.message).group(1))
463 elif test_type in ("TCP", ):
464 groups = re.search(self.REGEX_TCP, test.message)
465 test_result["result"] = dict()
466 test_result["result"]["value"] = int(groups.group(2))
467 test_result["result"]["unit"] = groups.group(1)
468 elif test_type in ("MRR", ):
469 groups = re.search(self.REGEX_MRR, test.message)
470 test_result["result"] = dict()
471 test_result["result"]["duration"] = int(groups.group(1))
472 test_result["result"]["tx"] = int(groups.group(2))
473 test_result["result"]["rx"] = int(groups.group(3))
474 test_result["result"]["throughput"] = int(
475 test_result["result"]["rx"] /
476 test_result["result"]["duration"])
478 test_result["status"] = test.status
480 self._test_ID = test.longname.lower()
481 self._data["tests"][self._test_ID] = test_result
483 def end_test(self, test):
484 """Called when test ends.
486 :param test: Test to process.
492 def visit_keyword(self, keyword):
493 """Implements traversing through the keyword and its child keywords.
495 :param keyword: Keyword to process.
496 :type keyword: Keyword
499 if self.start_keyword(keyword) is not False:
500 self.end_keyword(keyword)
502 def start_keyword(self, keyword):
503 """Called when keyword starts. Default implementation does nothing.
505 :param keyword: Keyword to process.
506 :type keyword: Keyword
510 if keyword.type == "setup":
511 self.visit_setup_kw(keyword)
512 elif keyword.type == "teardown":
513 self._lookup_kw_nr = 0
514 self.visit_teardown_kw(keyword)
516 self._lookup_kw_nr = 0
517 self.visit_test_kw(keyword)
518 except AttributeError:
521 def end_keyword(self, keyword):
522 """Called when keyword ends. Default implementation does nothing.
524 :param keyword: Keyword to process.
525 :type keyword: Keyword
530 def visit_test_kw(self, test_kw):
531 """Implements traversing through the test keyword and its child
534 :param test_kw: Keyword to process.
535 :type test_kw: Keyword
538 for keyword in test_kw.keywords:
539 if self.start_test_kw(keyword) is not False:
540 self.visit_test_kw(keyword)
541 self.end_test_kw(keyword)
543 def start_test_kw(self, test_kw):
544 """Called when test keyword starts. Default implementation does
547 :param test_kw: Keyword to process.
548 :type test_kw: Keyword
551 if test_kw.name.count("Show Runtime Counters On All Duts"):
552 self._lookup_kw_nr += 1
553 self._show_run_lookup_nr = 0
554 self._msg_type = "test-show-runtime"
555 test_kw.messages.visit(self)
557 def end_test_kw(self, test_kw):
558 """Called when keyword ends. Default implementation does nothing.
560 :param test_kw: Keyword to process.
561 :type test_kw: Keyword
566 def visit_setup_kw(self, setup_kw):
567 """Implements traversing through the teardown keyword and its child
570 :param setup_kw: Keyword to process.
571 :type setup_kw: Keyword
574 for keyword in setup_kw.keywords:
575 if self.start_setup_kw(keyword) is not False:
576 self.visit_setup_kw(keyword)
577 self.end_setup_kw(keyword)
579 def start_setup_kw(self, setup_kw):
580 """Called when teardown keyword starts. Default implementation does
583 :param setup_kw: Keyword to process.
584 :type setup_kw: Keyword
587 if setup_kw.name.count("Vpp Show Version Verbose") \
588 and not self._version:
589 self._msg_type = "setup-version"
590 setup_kw.messages.visit(self)
592 def end_setup_kw(self, setup_kw):
593 """Called when keyword ends. Default implementation does nothing.
595 :param setup_kw: Keyword to process.
596 :type setup_kw: Keyword
601 def visit_teardown_kw(self, teardown_kw):
602 """Implements traversing through the teardown keyword and its child
605 :param teardown_kw: Keyword to process.
606 :type teardown_kw: Keyword
609 for keyword in teardown_kw.keywords:
610 if self.start_teardown_kw(keyword) is not False:
611 self.visit_teardown_kw(keyword)
612 self.end_teardown_kw(keyword)
614 def start_teardown_kw(self, teardown_kw):
615 """Called when teardown keyword starts. Default implementation does
618 :param teardown_kw: Keyword to process.
619 :type teardown_kw: Keyword
623 if teardown_kw.name.count("Show Vat History On All Duts"):
624 self._vat_history_lookup_nr = 0
625 self._msg_type = "teardown-vat-history"
626 teardown_kw.messages.visit(self)
628 def end_teardown_kw(self, teardown_kw):
629 """Called when keyword ends. Default implementation does nothing.
631 :param teardown_kw: Keyword to process.
632 :type teardown_kw: Keyword
637 def visit_message(self, msg):
638 """Implements visiting the message.
640 :param msg: Message to process.
644 if self.start_message(msg) is not False:
645 self.end_message(msg)
647 def start_message(self, msg):
648 """Called when message starts. Get required information from messages:
651 :param msg: Message to process.
657 self.parse_msg[self._msg_type](msg)
659 def end_message(self, msg):
660 """Called when message ends. Default implementation does nothing.
662 :param msg: Message to process.
669 class InputData(object):
672 The data is extracted from output.xml files generated by Jenkins jobs and
673 stored in pandas' DataFrames.
684 - ID: test data (as described in ExecutionChecker documentation)
687 def __init__(self, spec):
690 :param spec: Specification.
691 :type spec: Specification
698 self._input_data = None
702 """Getter - Input data.
705 :rtype: pandas.Series
707 return self._input_data
709 def metadata(self, job, build):
712 :param job: Job which metadata we want.
713 :param build: Build which metadata we want.
717 :rtype: pandas.Series
720 return self.data[job][build]["metadata"]
722 def suites(self, job, build):
725 :param job: Job which suites we want.
726 :param build: Build which suites we want.
730 :rtype: pandas.Series
733 return self.data[job][str(build)]["suites"]
735 def tests(self, job, build):
738 :param job: Job which tests we want.
739 :param build: Build which tests we want.
743 :rtype: pandas.Series
746 return self.data[job][build]["tests"]
749 def _parse_tests(job, build):
750 """Process data from robot output.xml file and return JSON structured
753 :param job: The name of job which build output data will be processed.
754 :param build: The build which output data will be processed.
757 :returns: JSON data structure.
761 with open(build["file-name"], 'r') as data_file:
763 result = ExecutionResult(data_file)
764 except errors.DataError as err:
765 logging.error("Error occurred while parsing output.xml: {0}".
768 checker = ExecutionChecker(job=job, build=build)
769 result.visit(checker)
774 """Parse input data from input files and store in pandas' Series.
777 logging.info("Parsing input files ...")
780 for job, builds in self._cfg.builds.items():
781 logging.info(" Extracting data from the job '{0}' ...'".
785 if build["status"] == "failed" \
786 or build["status"] == "not found":
788 logging.info(" Extracting data from the build '{0}'".
789 format(build["build"]))
790 logging.info(" Processing the file '{0}'".
791 format(build["file-name"]))
792 data = InputData._parse_tests(job, build)
794 logging.error("Input data file from the job '{job}', build "
795 "'{build}' is damaged. Skipped.".
796 format(job=job, build=build["build"]))
799 build_data = pd.Series({
800 "metadata": pd.Series(data["metadata"].values(),
801 index=data["metadata"].keys()),
802 "suites": pd.Series(data["suites"].values(),
803 index=data["suites"].keys()),
804 "tests": pd.Series(data["tests"].values(),
805 index=data["tests"].keys()),
807 builds_data[str(build["build"])] = build_data
808 logging.info(" Done.")
810 job_data[job] = pd.Series(builds_data.values(),
811 index=builds_data.keys())
812 logging.info(" Done.")
814 self._input_data = pd.Series(job_data.values(), index=job_data.keys())
815 logging.info("Done.")
818 def _end_of_tag(tag_filter, start=0, closer="'"):
819 """Return the index of character in the string which is the end of tag.
821 :param tag_filter: The string where the end of tag is being searched.
822 :param start: The index where the searching is stated.
823 :param closer: The character which is the tag closer.
824 :type tag_filter: str
827 :returns: The index of the tag closer.
832 idx_opener = tag_filter.index(closer, start)
833 return tag_filter.index(closer, idx_opener + 1)
838 def _condition(tag_filter):
839 """Create a conditional statement from the given tag filter.
841 :param tag_filter: Filter based on tags from the element specification.
842 :type tag_filter: str
843 :returns: Conditional statement which can be evaluated.
849 index = InputData._end_of_tag(tag_filter, index)
853 tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
855 def filter_data(self, element, params=None, data_set="tests",
856 continue_on_error=False):
857 """Filter required data from the given jobs and builds.
859 The output data structure is:
876 :param element: Element which will use the filtered data.
877 :param params: Parameters which will be included in the output. If None,
878 all parameters are included.
879 :param data_set: The set of data to be filtered: tests, suites,
881 :param continue_on_error: Continue if there is error while reading the
882 data. The Item will be empty then
883 :type element: pandas.Series
886 :type continue_on_error: bool
887 :returns: Filtered data.
891 logging.info(" Creating the data set for the {0} '{1}'.".
892 format(element.get("type", ""), element.get("title", "")))
895 if element["filter"] in ("all", "template"):
898 cond = InputData._condition(element["filter"])
899 logging.debug(" Filter: {0}".format(cond))
901 logging.error(" No filter defined.")
905 params = element.get("parameters", None)
909 for job, builds in element["data"].items():
910 data[job] = pd.Series()
912 data[job][str(build)] = pd.Series()
914 data_iter = self.data[job][str(build)][data_set].\
917 if continue_on_error:
921 for test_ID, test_data in data_iter:
922 if eval(cond, {"tags": test_data.get("tags", "")}):
923 data[job][str(build)][test_ID] = pd.Series()
925 for param, val in test_data.items():
926 data[job][str(build)][test_ID][param] = val
930 data[job][str(build)][test_ID][param] =\
933 data[job][str(build)][test_ID][param] =\
937 except (KeyError, IndexError, ValueError) as err:
938 logging.error(" Missing mandatory parameter in the element "
939 "specification: {0}".format(err))
941 except AttributeError:
944 logging.error(" The filter '{0}' is not correct. Check if all "
945 "tags are enclosed by apostrophes.".format(cond))
949 def merge_data(data):
950 """Merge data from more jobs and builds to a simple data structure.
952 The output data structure is:
963 :param data: Data to merge.
964 :type data: pandas.Series
965 :returns: Merged data.
966 :rtype: pandas.Series
969 logging.info(" Merging data ...")
971 merged_data = pd.Series()
972 for _, builds in data.iteritems():
973 for _, item in builds.iteritems():
974 for ID, item_data in item.iteritems():
975 merged_data[ID] = item_data