1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
25 from robot.api import ExecutionResult, ResultVisitor
26 from collections import OrderedDict
27 from string import replace
30 class ExecutionChecker(ResultVisitor):
31 """Class to traverse through the test suite structure.
33 The functionality implemented in this class generates a json structure:
38 "metadata": { # Optional
39 "version": "VPP version",
40 "job": "Jenkins job name",
41 "build": "Information about the build"
45 "doc": "Suite 1 documentation",
46 "parent": "Suite 1 parent",
47 "level": "Level of the suite in the suite hierarchy"
50 "doc": "Suite N documentation",
51 "parent": "Suite 2 parent",
52 "level": "Level of the suite in the suite hierarchy"
58 "parent": "Name of the parent of the test",
59 "doc": "Test documentation"
61 "tags": ["tag 1", "tag 2", "tag n"],
62 "type": "PDR" | "NDR",
65 "unit": "pps" | "bps" | "percentage"
74 "50": { # Only for NDR
79 "10": { # Only for NDR
91 "50": { # Only for NDR
96 "10": { # Only for NDR
103 "lossTolerance": "lossTolerance", # Only for PDR
104 "vat-history": "DUT1 and DUT2 VAT History"
106 "show-run": "Show Run"
118 "metadata": { # Optional
119 "version": "VPP version",
120 "job": "Jenkins job name",
121 "build": "Information about the build"
125 "doc": "Suite 1 documentation",
126 "parent": "Suite 1 parent",
127 "level": "Level of the suite in the suite hierarchy"
130 "doc": "Suite N documentation",
131 "parent": "Suite 2 parent",
132 "level": "Level of the suite in the suite hierarchy"
138 "parent": "Name of the parent of the test",
139 "doc": "Test documentation"
140 "msg": "Test message"
141 "tags": ["tag 1", "tag 2", "tag n"],
142 "vat-history": "DUT1 and DUT2 VAT History"
143 "show-run": "Show Run"
144 "status": "PASS" | "FAIL"
152 .. note:: ID is the lowercase full path to the test.
155 REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
157 REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
158 r'LAT_\d+%NDR:\s\[\'(-?\d+\/-?\d+/-?\d+)\','
159 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
160 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
161 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
162 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
163 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
165 REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
166 r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
167 r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
169 REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
172 REGEX_VERSION = re.compile(r"(stdout: 'vat# vat# Version:)(\s*)(.*)")
174 REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
176 def __init__(self, **metadata):
179 :param metadata: Key-value pairs to be included in "metadata" part of
184 # Type of message to parse out from the test messages
185 self._msg_type = None
190 # Number of VAT History messages found:
192 # 1 - VAT History of DUT1
193 # 2 - VAT History of DUT2
194 self._lookup_kw_nr = 0
195 self._vat_history_lookup_nr = 0
197 # Number of Show Running messages found
199 # 1 - Show run message found
200 self._show_run_lookup_nr = 0
202 # Test ID of currently processed test- the lowercase full path to the
206 # The main data structure
208 "metadata": OrderedDict(),
209 "suites": OrderedDict(),
210 "tests": OrderedDict()
213 # Save the provided metadata
214 for key, val in metadata.items():
215 self._data["metadata"][key] = val
217 # Dictionary defining the methods used to parse different types of
220 "setup-version": self._get_version,
221 "teardown-vat-history": self._get_vat_history,
222 "teardown-show-runtime": self._get_show_run
227 """Getter - Data parsed from the XML file.
229 :returns: Data parsed from the XML file.
234 def _get_version(self, msg):
235 """Called when extraction of VPP version is required.
237 :param msg: Message to process.
242 if msg.message.count("stdout: 'vat# vat# Version:"):
243 self._version = str(re.search(self.REGEX_VERSION, msg.message).
245 self._data["metadata"]["version"] = self._version
246 self._msg_type = None
248 logging.debug(" VPP version: {0}".format(self._version))
250 def _get_vat_history(self, msg):
251 """Called when extraction of VAT command history is required.
253 :param msg: Message to process.
257 if msg.message.count("VAT command history:"):
258 self._vat_history_lookup_nr += 1
259 if self._vat_history_lookup_nr == 1:
260 self._data["tests"][self._test_ID]["vat-history"] = str()
262 self._msg_type = None
263 text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
264 "VAT command history:", "", msg.message, count=1). \
265 replace("\n\n", "\n").replace('\n', ' |br| ').\
266 replace('\r', '').replace('"', "'")
268 self._data["tests"][self._test_ID]["vat-history"] += " |br| "
269 self._data["tests"][self._test_ID]["vat-history"] += \
270 "**DUT" + str(self._vat_history_lookup_nr) + ":** " + text
272 def _get_show_run(self, msg):
273 """Called when extraction of VPP operational data (output of CLI command
274 Show Runtime) is required.
276 :param msg: Message to process.
280 if msg.message.count("return STDOUT Thread "):
281 self._show_run_lookup_nr += 1
282 if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
283 self._data["tests"][self._test_ID]["show-run"] = str()
284 if self._lookup_kw_nr > 1:
285 self._msg_type = None
286 if self._show_run_lookup_nr == 1:
287 text = msg.message.replace("vat# ", "").\
288 replace("return STDOUT ", "").replace("\n\n", "\n").\
289 replace('\n', ' |br| ').\
290 replace('\r', '').replace('"', "'")
292 self._data["tests"][self._test_ID]["show-run"] += " |br| "
293 self._data["tests"][self._test_ID]["show-run"] += \
294 "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
298 def _get_latency(self, msg, test_type):
299 """Get the latency data from the test message.
301 :param msg: Message to be parsed.
302 :param test_type: Type of the test - NDR or PDR.
305 :returns: Latencies parsed from the message.
309 if test_type == "NDR":
310 groups = re.search(self.REGEX_LAT_NDR, msg)
311 groups_range = range(1, 7)
312 elif test_type == "PDR":
313 groups = re.search(self.REGEX_LAT_PDR, msg)
314 groups_range = range(1, 3)
319 for idx in groups_range:
321 lat = [int(item) for item in str(groups.group(idx)).split('/')]
322 except (AttributeError, ValueError):
324 latencies.append(lat)
326 keys = ("min", "avg", "max")
334 latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
335 latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
336 if test_type == "NDR":
337 latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
338 latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
339 latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
340 latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
344 def visit_suite(self, suite):
345 """Implements traversing through the suite and its direct children.
347 :param suite: Suite to process.
351 if self.start_suite(suite) is not False:
352 suite.suites.visit(self)
353 suite.tests.visit(self)
354 self.end_suite(suite)
356 def start_suite(self, suite):
357 """Called when suite starts.
359 :param suite: Suite to process.
365 parent_name = suite.parent.name
366 except AttributeError:
369 doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
370 replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
371 doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
373 self._data["suites"][suite.longname.lower().replace('"', "'").
374 replace(" ", "_")] = {
375 "name": suite.name.lower(),
377 "parent": parent_name,
378 "level": len(suite.longname.split("."))
381 suite.keywords.visit(self)
383 def end_suite(self, suite):
384 """Called when suite ends.
386 :param suite: Suite to process.
392 def visit_test(self, test):
393 """Implements traversing through the test.
395 :param test: Test to process.
399 if self.start_test(test) is not False:
400 test.keywords.visit(self)
403 def start_test(self, test):
404 """Called when test starts.
406 :param test: Test to process.
411 tags = [str(tag) for tag in test.tags]
413 test_result["name"] = test.name.lower()
414 test_result["parent"] = test.parent.name.lower()
415 test_result["tags"] = tags
416 doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
417 replace('\r', '').replace('[', ' |br| [')
418 test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
419 test_result["msg"] = test.message.replace('\n', ' |br| '). \
420 replace('\r', '').replace('"', "'")
421 if test.status == "PASS" and ("NDRPDRDISC" in tags or "TCP" in tags):
423 if "NDRDISC" in tags:
425 elif "PDRDISC" in tags:
427 elif "TCP" in tags: # Change to wrk?
432 test_result["type"] = test_type
434 if test_type in ("NDR", "PDR"):
436 rate_value = str(re.search(
437 self.REGEX_RATE, test.message).group(1))
438 except AttributeError:
441 rate_unit = str(re.search(
442 self.REGEX_RATE, test.message).group(2))
443 except AttributeError:
446 test_result["throughput"] = dict()
447 test_result["throughput"]["value"] = \
448 int(rate_value.split('.')[0])
449 test_result["throughput"]["unit"] = rate_unit
450 test_result["latency"] = \
451 self._get_latency(test.message, test_type)
452 if test_type == "PDR":
453 test_result["lossTolerance"] = str(re.search(
454 self.REGEX_TOLERANCE, test.message).group(1))
456 elif test_type in ("TCP", ):
457 groups = re.search(self.REGEX_TCP, test.message)
458 test_result["result"] = dict()
459 test_result["result"]["value"] = int(groups.group(2))
460 test_result["result"]["unit"] = groups.group(1)
462 test_result["status"] = test.status
464 self._test_ID = test.longname.lower()
465 self._data["tests"][self._test_ID] = test_result
467 def end_test(self, test):
468 """Called when test ends.
470 :param test: Test to process.
476 def visit_keyword(self, keyword):
477 """Implements traversing through the keyword and its child keywords.
479 :param keyword: Keyword to process.
480 :type keyword: Keyword
483 if self.start_keyword(keyword) is not False:
484 self.end_keyword(keyword)
486 def start_keyword(self, keyword):
487 """Called when keyword starts. Default implementation does nothing.
489 :param keyword: Keyword to process.
490 :type keyword: Keyword
494 if keyword.type == "setup":
495 self.visit_setup_kw(keyword)
496 elif keyword.type == "teardown":
497 self._lookup_kw_nr = 0
498 self.visit_teardown_kw(keyword)
499 except AttributeError:
502 def end_keyword(self, keyword):
503 """Called when keyword ends. Default implementation does nothing.
505 :param keyword: Keyword to process.
506 :type keyword: Keyword
511 def visit_setup_kw(self, setup_kw):
512 """Implements traversing through the teardown keyword and its child
515 :param setup_kw: Keyword to process.
516 :type setup_kw: Keyword
519 for keyword in setup_kw.keywords:
520 if self.start_setup_kw(keyword) is not False:
521 self.visit_setup_kw(keyword)
522 self.end_setup_kw(keyword)
524 def start_setup_kw(self, setup_kw):
525 """Called when teardown keyword starts. Default implementation does
528 :param setup_kw: Keyword to process.
529 :type setup_kw: Keyword
532 if setup_kw.name.count("Vpp Show Version Verbose") \
533 and not self._version:
534 self._msg_type = "setup-version"
535 setup_kw.messages.visit(self)
537 def end_setup_kw(self, setup_kw):
538 """Called when keyword ends. Default implementation does nothing.
540 :param setup_kw: Keyword to process.
541 :type setup_kw: Keyword
546 def visit_teardown_kw(self, teardown_kw):
547 """Implements traversing through the teardown keyword and its child
550 :param teardown_kw: Keyword to process.
551 :type teardown_kw: Keyword
554 for keyword in teardown_kw.keywords:
555 if self.start_teardown_kw(keyword) is not False:
556 self.visit_teardown_kw(keyword)
557 self.end_teardown_kw(keyword)
559 def start_teardown_kw(self, teardown_kw):
560 """Called when teardown keyword starts. Default implementation does
563 :param teardown_kw: Keyword to process.
564 :type teardown_kw: Keyword
568 if teardown_kw.name.count("Show Vat History On All Duts"):
569 self._vat_history_lookup_nr = 0
570 self._msg_type = "teardown-vat-history"
571 elif teardown_kw.name.count("Show Statistics On All Duts"):
572 self._lookup_kw_nr += 1
573 self._show_run_lookup_nr = 0
574 self._msg_type = "teardown-show-runtime"
577 teardown_kw.messages.visit(self)
579 def end_teardown_kw(self, teardown_kw):
580 """Called when keyword ends. Default implementation does nothing.
582 :param teardown_kw: Keyword to process.
583 :type teardown_kw: Keyword
588 def visit_message(self, msg):
589 """Implements visiting the message.
591 :param msg: Message to process.
595 if self.start_message(msg) is not False:
596 self.end_message(msg)
598 def start_message(self, msg):
599 """Called when message starts. Get required information from messages:
602 :param msg: Message to process.
608 self.parse_msg[self._msg_type](msg)
610 def end_message(self, msg):
611 """Called when message ends. Default implementation does nothing.
613 :param msg: Message to process.
620 class InputData(object):
623 The data is extracted from output.xml files generated by Jenkins jobs and
624 stored in pandas' DataFrames.
635 - ID: test data (as described in ExecutionChecker documentation)
638 def __init__(self, spec):
641 :param spec: Specification.
642 :type spec: Specification
649 self._input_data = None
653 """Getter - Input data.
656 :rtype: pandas.Series
658 return self._input_data
660 def metadata(self, job, build):
663 :param job: Job which metadata we want.
664 :param build: Build which metadata we want.
668 :rtype: pandas.Series
671 return self.data[job][build]["metadata"]
673 def suites(self, job, build):
676 :param job: Job which suites we want.
677 :param build: Build which suites we want.
681 :rtype: pandas.Series
684 return self.data[job][str(build)]["suites"]
686 def tests(self, job, build):
689 :param job: Job which tests we want.
690 :param build: Build which tests we want.
694 :rtype: pandas.Series
697 return self.data[job][build]["tests"]
700 def _parse_tests(job, build):
701 """Process data from robot output.xml file and return JSON structured
704 :param job: The name of job which build output data will be processed.
705 :param build: The build which output data will be processed.
708 :returns: JSON data structure.
712 with open(build["file-name"], 'r') as data_file:
713 result = ExecutionResult(data_file)
714 checker = ExecutionChecker(job=job, build=build)
715 result.visit(checker)
720 """Parse input data from input files and store in pandas' Series.
723 logging.info("Parsing input files ...")
726 for job, builds in self._cfg.builds.items():
727 logging.info(" Extracting data from the job '{0}' ...'".
731 if build["status"] == "failed" \
732 or build["status"] == "not found":
734 logging.info(" Extracting data from the build '{0}'".
735 format(build["build"]))
736 logging.info(" Processing the file '{0}'".
737 format(build["file-name"]))
738 data = InputData._parse_tests(job, build)
740 build_data = pd.Series({
741 "metadata": pd.Series(data["metadata"].values(),
742 index=data["metadata"].keys()),
743 "suites": pd.Series(data["suites"].values(),
744 index=data["suites"].keys()),
745 "tests": pd.Series(data["tests"].values(),
746 index=data["tests"].keys()),
748 builds_data[str(build["build"])] = build_data
749 logging.info(" Done.")
751 job_data[job] = pd.Series(builds_data.values(),
752 index=builds_data.keys())
753 logging.info(" Done.")
755 self._input_data = pd.Series(job_data.values(), index=job_data.keys())
756 logging.info("Done.")
759 def _end_of_tag(tag_filter, start=0, closer="'"):
760 """Return the index of character in the string which is the end of tag.
762 :param tag_filter: The string where the end of tag is being searched.
763 :param start: The index where the searching is stated.
764 :param closer: The character which is the tag closer.
765 :type tag_filter: str
768 :returns: The index of the tag closer.
773 idx_opener = tag_filter.index(closer, start)
774 return tag_filter.index(closer, idx_opener + 1)
779 def _condition(tag_filter):
780 """Create a conditional statement from the given tag filter.
782 :param tag_filter: Filter based on tags from the element specification.
783 :type tag_filter: str
784 :returns: Conditional statement which can be evaluated.
790 index = InputData._end_of_tag(tag_filter, index)
794 tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
796 def filter_data(self, element, params=None, data_set="tests"):
797 """Filter required data from the given jobs and builds.
799 The output data structure is:
816 :param element: Element which will use the filtered data.
817 :param params: Parameters which will be included in the output. If None,
818 all parameters are included.
819 :param data_set: The set of data to be filtered: tests, suites,
821 :type element: pandas.Series
824 :returns: Filtered data.
828 logging.info(" Creating the data set for the {0} '{1}'.".
829 format(element["type"], element.get("title", "")))
832 if element["filter"] in ("all", "template"):
835 cond = InputData._condition(element["filter"])
836 logging.debug(" Filter: {0}".format(cond))
838 logging.error(" No filter defined.")
842 params = element.get("parameters", None)
846 for job, builds in element["data"].items():
847 data[job] = pd.Series()
849 data[job][str(build)] = pd.Series()
850 for test_ID, test_data in \
851 self.data[job][str(build)][data_set].iteritems():
852 if eval(cond, {"tags": test_data.get("tags", "")}):
853 data[job][str(build)][test_ID] = pd.Series()
855 for param, val in test_data.items():
856 data[job][str(build)][test_ID][param] = val
860 data[job][str(build)][test_ID][param] =\
863 data[job][str(build)][test_ID][param] =\
867 except (KeyError, IndexError, ValueError) as err:
868 logging.error(" Missing mandatory parameter in the element "
869 "specification: {0}".format(err))
871 except AttributeError:
874 logging.error(" The filter '{0}' is not correct. Check if all "
875 "tags are enclosed by apostrophes.".format(cond))
879 def merge_data(data):
880 """Merge data from more jobs and builds to a simple data structure.
882 The output data structure is:
893 :param data: Data to merge.
894 :type data: pandas.Series
895 :returns: Merged data.
896 :rtype: pandas.Series
899 logging.info(" Merging data ...")
901 merged_data = pd.Series()
902 for _, builds in data.iteritems():
903 for _, item in builds.iteritems():
904 for ID, item_data in item.iteritems():
905 merged_data[ID] = item_data