1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
25 from robot.api import ExecutionResult, ResultVisitor
26 from collections import OrderedDict
27 from string import replace
30 class ExecutionChecker(ResultVisitor):
31 """Class to traverse through the test suite structure.
33 The functionality implemented in this class generates a json structure:
38 "metadata": { # Optional
39 "version": "VPP version",
40 "job": "Jenkins job name",
41 "build": "Information about the build"
45 "doc": "Suite 1 documentation",
46 "parent": "Suite 1 parent",
47 "level": "Level of the suite in the suite hierarchy"
50 "doc": "Suite N documentation",
51 "parent": "Suite 2 parent",
52 "level": "Level of the suite in the suite hierarchy"
58 "parent": "Name of the parent of the test",
59 "doc": "Test documentation"
61 "tags": ["tag 1", "tag 2", "tag n"],
62 "type": "PDR" | "NDR",
65 "unit": "pps" | "bps" | "percentage"
74 "50": { # Only for NDR
79 "10": { # Only for NDR
91 "50": { # Only for NDR
96 "10": { # Only for NDR
103 "lossTolerance": "lossTolerance", # Only for PDR
104 "vat-history": "DUT1 and DUT2 VAT History"
106 "show-run": "Show Run"
118 "metadata": { # Optional
119 "version": "VPP version",
120 "job": "Jenkins job name",
121 "build": "Information about the build"
125 "doc": "Suite 1 documentation",
126 "parent": "Suite 1 parent",
127 "level": "Level of the suite in the suite hierarchy"
130 "doc": "Suite N documentation",
131 "parent": "Suite 2 parent",
132 "level": "Level of the suite in the suite hierarchy"
138 "parent": "Name of the parent of the test",
139 "doc": "Test documentation"
140 "msg": "Test message"
141 "tags": ["tag 1", "tag 2", "tag n"],
142 "vat-history": "DUT1 and DUT2 VAT History"
143 "show-run": "Show Run"
144 "status": "PASS" | "FAIL"
152 .. note:: ID is the lowercase full path to the test.
155 REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
157 REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
158 r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+/-?\d+)\','
159 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
160 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
161 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
162 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
163 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
165 REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
166 r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
167 r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
169 REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
172 REGEX_VERSION = re.compile(r"(stdout: 'vat# vat# Version:)(\s*)(.*)")
174 def __init__(self, **metadata):
177 :param metadata: Key-value pairs to be included in "metadata" part of
182 # Type of message to parse out from the test messages
183 self._msg_type = None
188 # Number of VAT History messages found:
190 # 1 - VAT History of DUT1
191 # 2 - VAT History of DUT2
192 self._lookup_kw_nr = 0
193 self._vat_history_lookup_nr = 0
195 # Number of Show Running messages found
197 # 1 - Show run message found
198 self._show_run_lookup_nr = 0
200 # Test ID of currently processed test- the lowercase full path to the
204 # The main data structure
206 "metadata": OrderedDict(),
207 "suites": OrderedDict(),
208 "tests": OrderedDict()
211 # Save the provided metadata
212 for key, val in metadata.items():
213 self._data["metadata"][key] = val
215 # Dictionary defining the methods used to parse different types of
218 "setup-version": self._get_version,
219 "teardown-vat-history": self._get_vat_history,
220 "teardown-show-runtime": self._get_show_run
225 """Getter - Data parsed from the XML file.
227 :returns: Data parsed from the XML file.
232 def _get_version(self, msg):
233 """Called when extraction of VPP version is required.
235 :param msg: Message to process.
240 if msg.message.count("stdout: 'vat# vat# Version:"):
241 self._version = str(re.search(self.REGEX_VERSION, msg.message).
243 self._data["metadata"]["version"] = self._version
244 self._msg_type = None
246 logging.debug(" VPP version: {0}".format(self._version))
248 def _get_vat_history(self, msg):
249 """Called when extraction of VAT command history is required.
251 :param msg: Message to process.
255 if msg.message.count("VAT command history:"):
256 self._vat_history_lookup_nr += 1
257 if self._vat_history_lookup_nr == 1:
258 self._data["tests"][self._test_ID]["vat-history"] = str()
260 self._msg_type = None
261 text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
262 "VAT command history:", "", msg.message, count=1). \
263 replace("\n\n", "\n").replace('\n', ' |br| ').\
264 replace('\r', '').replace('"', "'")
266 self._data["tests"][self._test_ID]["vat-history"] += " |br| "
267 self._data["tests"][self._test_ID]["vat-history"] += \
268 "**DUT" + str(self._vat_history_lookup_nr) + ":** " + text
270 def _get_show_run(self, msg):
271 """Called when extraction of VPP operational data (output of CLI command
272 Show Runtime) is required.
274 :param msg: Message to process.
278 if msg.message.count("vat# Thread "):
279 self._show_run_lookup_nr += 1
280 if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
281 self._data["tests"][self._test_ID]["show-run"] = str()
282 if self._lookup_kw_nr > 1:
283 self._msg_type = None
284 if self._show_run_lookup_nr == 1:
285 text = msg.message.replace("vat# ", "").\
286 replace("return STDOUT ", "").replace("\n\n", "\n").\
287 replace('\n', ' |br| ').\
288 replace('\r', '').replace('"', "'")
290 self._data["tests"][self._test_ID]["show-run"] += " |br| "
291 self._data["tests"][self._test_ID]["show-run"] += \
292 "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
296 def _get_latency(self, msg, test_type):
297 """Get the latency data from the test message.
299 :param msg: Message to be parsed.
300 :param test_type: Type of the test - NDR or PDR.
303 :returns: Latencies parsed from the message.
307 if test_type == "NDR":
308 groups = re.search(self.REGEX_LAT_NDR, msg)
309 groups_range = range(1, 7)
310 elif test_type == "PDR":
311 groups = re.search(self.REGEX_LAT_PDR, msg)
312 groups_range = range(1, 3)
317 for idx in groups_range:
319 lat = [int(item) for item in str(groups.group(idx)).split('/')]
320 except (AttributeError, ValueError):
322 latencies.append(lat)
324 keys = ("min", "avg", "max")
332 latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
333 latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
334 if test_type == "NDR":
335 latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
336 latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
337 latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
338 latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
342 def visit_suite(self, suite):
343 """Implements traversing through the suite and its direct children.
345 :param suite: Suite to process.
349 if self.start_suite(suite) is not False:
350 suite.suites.visit(self)
351 suite.tests.visit(self)
352 self.end_suite(suite)
354 def start_suite(self, suite):
355 """Called when suite starts.
357 :param suite: Suite to process.
363 parent_name = suite.parent.name
364 except AttributeError:
367 doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
368 replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
369 doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
371 self._data["suites"][suite.longname.lower().replace('"', "'").
372 replace(" ", "_")] = {
373 "name": suite.name.lower(),
375 "parent": parent_name,
376 "level": len(suite.longname.split("."))
379 suite.keywords.visit(self)
381 def end_suite(self, suite):
382 """Called when suite ends.
384 :param suite: Suite to process.
390 def visit_test(self, test):
391 """Implements traversing through the test.
393 :param test: Test to process.
397 if self.start_test(test) is not False:
398 test.keywords.visit(self)
401 def start_test(self, test):
402 """Called when test starts.
404 :param test: Test to process.
409 tags = [str(tag) for tag in test.tags]
411 test_result["name"] = test.name.lower()
412 test_result["parent"] = test.parent.name.lower()
413 test_result["tags"] = tags
414 doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
415 replace('\r', '').replace('[', ' |br| [')
416 test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
417 test_result["msg"] = test.message.replace('\n', ' |br| '). \
418 replace('\r', '').replace('"', "'")
419 if test.status == "PASS" and "NDRPDRDISC" in tags:
421 if "NDRDISC" in tags:
423 elif "PDRDISC" in tags:
429 rate_value = str(re.search(
430 self.REGEX_RATE, test.message).group(1))
431 except AttributeError:
434 rate_unit = str(re.search(
435 self.REGEX_RATE, test.message).group(2))
436 except AttributeError:
439 test_result["type"] = test_type
440 test_result["throughput"] = dict()
441 test_result["throughput"]["value"] = int(rate_value.split('.')[0])
442 test_result["throughput"]["unit"] = rate_unit
443 test_result["latency"] = self._get_latency(test.message, test_type)
444 if test_type == "PDR":
445 test_result["lossTolerance"] = str(re.search(
446 self.REGEX_TOLERANCE, test.message).group(1))
448 test_result["status"] = test.status
450 self._test_ID = test.longname.lower()
451 self._data["tests"][self._test_ID] = test_result
453 def end_test(self, test):
454 """Called when test ends.
456 :param test: Test to process.
462 def visit_keyword(self, keyword):
463 """Implements traversing through the keyword and its child keywords.
465 :param keyword: Keyword to process.
466 :type keyword: Keyword
469 if self.start_keyword(keyword) is not False:
470 self.end_keyword(keyword)
472 def start_keyword(self, keyword):
473 """Called when keyword starts. Default implementation does nothing.
475 :param keyword: Keyword to process.
476 :type keyword: Keyword
480 if keyword.type == "setup":
481 self.visit_setup_kw(keyword)
482 elif keyword.type == "teardown":
483 self._lookup_kw_nr = 0
484 self.visit_teardown_kw(keyword)
485 except AttributeError:
488 def end_keyword(self, keyword):
489 """Called when keyword ends. Default implementation does nothing.
491 :param keyword: Keyword to process.
492 :type keyword: Keyword
497 def visit_setup_kw(self, setup_kw):
498 """Implements traversing through the teardown keyword and its child
501 :param setup_kw: Keyword to process.
502 :type setup_kw: Keyword
505 for keyword in setup_kw.keywords:
506 if self.start_setup_kw(keyword) is not False:
507 self.visit_setup_kw(keyword)
508 self.end_setup_kw(keyword)
510 def start_setup_kw(self, setup_kw):
511 """Called when teardown keyword starts. Default implementation does
514 :param setup_kw: Keyword to process.
515 :type setup_kw: Keyword
518 if setup_kw.name.count("Vpp Show Version Verbose") \
519 and not self._version:
520 self._msg_type = "setup-version"
521 setup_kw.messages.visit(self)
523 def end_setup_kw(self, setup_kw):
524 """Called when keyword ends. Default implementation does nothing.
526 :param setup_kw: Keyword to process.
527 :type setup_kw: Keyword
532 def visit_teardown_kw(self, teardown_kw):
533 """Implements traversing through the teardown keyword and its child
536 :param teardown_kw: Keyword to process.
537 :type teardown_kw: Keyword
540 for keyword in teardown_kw.keywords:
541 if self.start_teardown_kw(keyword) is not False:
542 self.visit_teardown_kw(keyword)
543 self.end_teardown_kw(keyword)
545 def start_teardown_kw(self, teardown_kw):
546 """Called when teardown keyword starts. Default implementation does
549 :param teardown_kw: Keyword to process.
550 :type teardown_kw: Keyword
554 if teardown_kw.name.count("Show Vat History On All Duts"):
555 self._vat_history_lookup_nr = 0
556 self._msg_type = "teardown-vat-history"
557 elif teardown_kw.name.count("Vpp Show Runtime"):
558 self._lookup_kw_nr += 1
559 self._show_run_lookup_nr = 0
560 self._msg_type = "teardown-show-runtime"
563 teardown_kw.messages.visit(self)
565 def end_teardown_kw(self, teardown_kw):
566 """Called when keyword ends. Default implementation does nothing.
568 :param teardown_kw: Keyword to process.
569 :type teardown_kw: Keyword
574 def visit_message(self, msg):
575 """Implements visiting the message.
577 :param msg: Message to process.
581 if self.start_message(msg) is not False:
582 self.end_message(msg)
584 def start_message(self, msg):
585 """Called when message starts. Get required information from messages:
588 :param msg: Message to process.
594 self.parse_msg[self._msg_type](msg)
596 def end_message(self, msg):
597 """Called when message ends. Default implementation does nothing.
599 :param msg: Message to process.
606 class InputData(object):
609 The data is extracted from output.xml files generated by Jenkins jobs and
610 stored in pandas' DataFrames.
621 - ID: test data (as described in ExecutionChecker documentation)
624 def __init__(self, spec):
627 :param spec: Specification.
628 :type spec: Specification
635 self._input_data = None
639 """Getter - Input data.
642 :rtype: pandas.Series
644 return self._input_data
646 def metadata(self, job, build):
649 :param job: Job which metadata we want.
650 :param build: Build which metadata we want.
654 :rtype: pandas.Series
657 return self.data[job][build]["metadata"]
659 def suites(self, job, build):
662 :param job: Job which suites we want.
663 :param build: Build which suites we want.
667 :rtype: pandas.Series
670 return self.data[job][str(build)]["suites"]
672 def tests(self, job, build):
675 :param job: Job which tests we want.
676 :param build: Build which tests we want.
680 :rtype: pandas.Series
683 return self.data[job][build]["tests"]
686 def _parse_tests(job, build):
687 """Process data from robot output.xml file and return JSON structured
690 :param job: The name of job which build output data will be processed.
691 :param build: The build which output data will be processed.
694 :returns: JSON data structure.
698 with open(build["file-name"], 'r') as data_file:
699 result = ExecutionResult(data_file)
700 checker = ExecutionChecker(job=job, build=build)
701 result.visit(checker)
706 """Parse input data from input files and store in pandas' Series.
709 logging.info("Parsing input files ...")
712 for job, builds in self._cfg.builds.items():
713 logging.info(" Extracting data from the job '{0}' ...'".
717 if build["status"] == "failed" \
718 or build["status"] == "not found":
720 logging.info(" Extracting data from the build '{0}'".
721 format(build["build"]))
722 logging.info(" Processing the file '{0}'".
723 format(build["file-name"]))
724 data = InputData._parse_tests(job, build)
726 build_data = pd.Series({
727 "metadata": pd.Series(data["metadata"].values(),
728 index=data["metadata"].keys()),
729 "suites": pd.Series(data["suites"].values(),
730 index=data["suites"].keys()),
731 "tests": pd.Series(data["tests"].values(),
732 index=data["tests"].keys()),
734 builds_data[str(build["build"])] = build_data
735 logging.info(" Done.")
737 job_data[job] = pd.Series(builds_data.values(),
738 index=builds_data.keys())
739 logging.info(" Done.")
741 self._input_data = pd.Series(job_data.values(), index=job_data.keys())
742 logging.info("Done.")
745 def _end_of_tag(tag_filter, start=0, closer="'"):
746 """Return the index of character in the string which is the end of tag.
748 :param tag_filter: The string where the end of tag is being searched.
749 :param start: The index where the searching is stated.
750 :param closer: The character which is the tag closer.
751 :type tag_filter: str
754 :returns: The index of the tag closer.
759 idx_opener = tag_filter.index(closer, start)
760 return tag_filter.index(closer, idx_opener + 1)
765 def _condition(tag_filter):
766 """Create a conditional statement from the given tag filter.
768 :param tag_filter: Filter based on tags from the element specification.
769 :type tag_filter: str
770 :returns: Conditional statement which can be evaluated.
776 index = InputData._end_of_tag(tag_filter, index)
780 tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
782 def filter_data(self, element, params=None, data_set="tests"):
783 """Filter required data from the given jobs and builds.
785 The output data structure is:
802 :param element: Element which will use the filtered data.
803 :param params: Parameters which will be included in the output. If None,
804 all parameters are included.
805 :param data_set: The set of data to be filtered: tests, suites,
807 :type element: pandas.Series
810 :returns: Filtered data.
814 logging.info(" Creating the data set for the {0} '{1}'.".
815 format(element["type"], element.get("title", "")))
818 if element["filter"] in ("all", "template"):
821 cond = InputData._condition(element["filter"])
822 logging.debug(" Filter: {0}".format(cond))
824 logging.error(" No filter defined.")
828 params = element.get("parameters", None)
832 for job, builds in element["data"].items():
833 data[job] = pd.Series()
835 data[job][str(build)] = pd.Series()
836 for test_ID, test_data in \
837 self.data[job][str(build)][data_set].iteritems():
838 if eval(cond, {"tags": test_data.get("tags", "")}):
839 data[job][str(build)][test_ID] = pd.Series()
841 for param, val in test_data.items():
842 data[job][str(build)][test_ID][param] = val
846 data[job][str(build)][test_ID][param] =\
849 data[job][str(build)][test_ID][param] =\
853 except (KeyError, IndexError, ValueError) as err:
854 logging.error(" Missing mandatory parameter in the element "
855 "specification.", err)
858 logging.error(" The filter '{0}' is not correct. Check if all "
859 "tags are enclosed by apostrophes.".format(cond))