1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
22 import multiprocessing
28 from robot.api import ExecutionResult, ResultVisitor
29 from robot import errors
30 from collections import OrderedDict
31 from string import replace
33 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
35 from input_data_files import download_and_unzip_data_file
36 from utils import Worker
39 class ExecutionChecker(ResultVisitor):
40 """Class to traverse through the test suite structure.
42 The functionality implemented in this class generates a json structure:
48 "generated": "Timestamp",
49 "version": "SUT version",
50 "job": "Jenkins job name",
51 "build": "Information about the build"
54 "Suite long name 1": {
56 "doc": "Suite 1 documentation",
57 "parent": "Suite 1 parent",
58 "level": "Level of the suite in the suite hierarchy"
60 "Suite long name N": {
62 "doc": "Suite N documentation",
63 "parent": "Suite 2 parent",
64 "level": "Level of the suite in the suite hierarchy"
71 "parent": "Name of the parent of the test",
72 "doc": "Test documentation",
73 "msg": "Test message",
74 "vat-history": "DUT1 and DUT2 VAT History",
75 "show-run": "Show Run",
76 "tags": ["tag 1", "tag 2", "tag n"],
78 "status": "PASS" | "FAIL",
120 "parent": "Name of the parent of the test",
121 "doc": "Test documentation",
122 "msg": "Test message",
123 "tags": ["tag 1", "tag 2", "tag n"],
125 "status": "PASS" | "FAIL",
132 "parent": "Name of the parent of the test",
133 "doc": "Test documentation",
134 "msg": "Test message",
135 "tags": ["tag 1", "tag 2", "tag n"],
136 "type": "MRR" | "BMRR",
137 "status": "PASS" | "FAIL",
139 "receive-rate": AvgStdevMetadata,
143 # TODO: Remove when definitely no NDRPDRDISC tests are used:
147 "parent": "Name of the parent of the test",
148 "doc": "Test documentation",
149 "msg": "Test message",
150 "tags": ["tag 1", "tag 2", "tag n"],
151 "type": "PDR" | "NDR",
152 "status": "PASS" | "FAIL",
153 "throughput": { # Only type: "PDR" | "NDR"
155 "unit": "pps" | "bps" | "percentage"
157 "latency": { # Only type: "PDR" | "NDR"
164 "50": { # Only for NDR
169 "10": { # Only for NDR
181 "50": { # Only for NDR
186 "10": { # Only for NDR
193 "lossTolerance": "lossTolerance", # Only type: "PDR"
194 "vat-history": "DUT1 and DUT2 VAT History"
195 "show-run": "Show Run"
207 "metadata": { # Optional
208 "version": "VPP version",
209 "job": "Jenkins job name",
210 "build": "Information about the build"
214 "doc": "Suite 1 documentation",
215 "parent": "Suite 1 parent",
216 "level": "Level of the suite in the suite hierarchy"
219 "doc": "Suite N documentation",
220 "parent": "Suite 2 parent",
221 "level": "Level of the suite in the suite hierarchy"
227 "parent": "Name of the parent of the test",
228 "doc": "Test documentation"
229 "msg": "Test message"
230 "tags": ["tag 1", "tag 2", "tag n"],
231 "vat-history": "DUT1 and DUT2 VAT History"
232 "show-run": "Show Run"
233 "status": "PASS" | "FAIL"
241 .. note:: ID is the lowercase full path to the test.
244 # TODO: Remove when definitely no NDRPDRDISC tests are used:
245 REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
247 REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
248 r'NDR_UPPER:\s(\d+.\d+).*\n'
249 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
250 r'PDR_UPPER:\s(\d+.\d+)')
252 # TODO: Remove when definitely no NDRPDRDISC tests are used:
253 REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
254 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
255 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
256 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
257 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
258 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
259 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
261 REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
262 r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
263 r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
265 REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
266 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
268 REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
271 REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*)(.*)")
273 REGEX_VERSION_DPDK = re.compile(r"(return STDOUT testpmd)([\d\D\n]*)"
274 r"(RTE Version: 'DPDK )(.*)(')")
276 REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
278 REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
279 r'tx\s(\d*),\srx\s(\d*)')
281 REGEX_BMRR = re.compile(r'Maximum Receive Rate trial results'
282 r' in packets per second: \[(.*)\]')
284 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
286 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
288 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
290 REGEX_TC_NUMBER = re.compile(r'tc[0-9]{2}-')
292 def __init__(self, metadata):
295 :param metadata: Key-value pairs to be included in "metadata" part of
300 # Type of message to parse out from the test messages
301 self._msg_type = None
307 self._timestamp = None
309 # Number of VAT History messages found:
311 # 1 - VAT History of DUT1
312 # 2 - VAT History of DUT2
313 self._lookup_kw_nr = 0
314 self._vat_history_lookup_nr = 0
316 # Number of Show Running messages found
318 # 1 - Show run message found
319 self._show_run_lookup_nr = 0
321 # Test ID of currently processed test- the lowercase full path to the
325 # The main data structure
327 "metadata": OrderedDict(),
328 "suites": OrderedDict(),
329 "tests": OrderedDict()
332 # Save the provided metadata
333 for key, val in metadata.items():
334 self._data["metadata"][key] = val
336 # Dictionary defining the methods used to parse different types of
339 "timestamp": self._get_timestamp,
340 "vpp-version": self._get_vpp_version,
341 "dpdk-version": self._get_dpdk_version,
342 "teardown-vat-history": self._get_vat_history,
343 "test-show-runtime": self._get_show_run
348 """Getter - Data parsed from the XML file.
350 :returns: Data parsed from the XML file.
355 def _get_vpp_version(self, msg):
356 """Called when extraction of VPP version is required.
358 :param msg: Message to process.
363 if msg.message.count("return STDOUT Version:"):
364 self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
366 self._data["metadata"]["version"] = self._version
367 self._msg_type = None
369 def _get_dpdk_version(self, msg):
370 """Called when extraction of DPDK version is required.
372 :param msg: Message to process.
377 if msg.message.count("return STDOUT testpmd"):
379 self._version = str(re.search(
380 self.REGEX_VERSION_DPDK, msg.message). group(4))
381 self._data["metadata"]["version"] = self._version
385 self._msg_type = None
387 def _get_timestamp(self, msg):
388 """Called when extraction of timestamp is required.
390 :param msg: Message to process.
395 self._timestamp = msg.timestamp[:14]
396 self._data["metadata"]["generated"] = self._timestamp
397 self._msg_type = None
399 def _get_vat_history(self, msg):
400 """Called when extraction of VAT command history is required.
402 :param msg: Message to process.
406 if msg.message.count("VAT command history:"):
407 self._vat_history_lookup_nr += 1
408 if self._vat_history_lookup_nr == 1:
409 self._data["tests"][self._test_ID]["vat-history"] = str()
411 self._msg_type = None
412 text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
413 "VAT command history:", "", msg.message, count=1). \
414 replace("\n\n", "\n").replace('\n', ' |br| ').\
415 replace('\r', '').replace('"', "'")
417 self._data["tests"][self._test_ID]["vat-history"] += " |br| "
418 self._data["tests"][self._test_ID]["vat-history"] += \
419 "**DUT" + str(self._vat_history_lookup_nr) + ":** " + text
421 def _get_show_run(self, msg):
422 """Called when extraction of VPP operational data (output of CLI command
423 Show Runtime) is required.
425 :param msg: Message to process.
429 if msg.message.count("return STDOUT Thread "):
430 self._show_run_lookup_nr += 1
431 if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
432 self._data["tests"][self._test_ID]["show-run"] = str()
433 if self._lookup_kw_nr > 1:
434 self._msg_type = None
435 if self._show_run_lookup_nr == 1:
436 text = msg.message.replace("vat# ", "").\
437 replace("return STDOUT ", "").replace("\n\n", "\n").\
438 replace('\n', ' |br| ').\
439 replace('\r', '').replace('"', "'")
441 self._data["tests"][self._test_ID]["show-run"] += " |br| "
442 self._data["tests"][self._test_ID]["show-run"] += \
443 "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
447 # TODO: Remove when definitely no NDRPDRDISC tests are used:
448 def _get_latency(self, msg, test_type):
449 """Get the latency data from the test message.
451 :param msg: Message to be parsed.
452 :param test_type: Type of the test - NDR or PDR.
455 :returns: Latencies parsed from the message.
459 if test_type == "NDR":
460 groups = re.search(self.REGEX_LAT_NDR, msg)
461 groups_range = range(1, 7)
462 elif test_type == "PDR":
463 groups = re.search(self.REGEX_LAT_PDR, msg)
464 groups_range = range(1, 3)
469 for idx in groups_range:
471 lat = [int(item) for item in str(groups.group(idx)).split('/')]
472 except (AttributeError, ValueError):
474 latencies.append(lat)
476 keys = ("min", "avg", "max")
484 latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
485 latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
486 if test_type == "NDR":
487 latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
488 latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
489 latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
490 latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
494 def _get_ndrpdr_throughput(self, msg):
495 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
498 :param msg: The test message to be parsed.
500 :returns: Parsed data as a dict and the status (PASS/FAIL).
501 :rtype: tuple(dict, str)
505 "NDR": {"LOWER": -1.0, "UPPER": -1.0},
506 "PDR": {"LOWER": -1.0, "UPPER": -1.0}
509 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
511 if groups is not None:
513 throughput["NDR"]["LOWER"] = float(groups.group(1))
514 throughput["NDR"]["UPPER"] = float(groups.group(2))
515 throughput["PDR"]["LOWER"] = float(groups.group(3))
516 throughput["PDR"]["UPPER"] = float(groups.group(4))
518 except (IndexError, ValueError):
521 return throughput, status
523 def _get_ndrpdr_latency(self, msg):
524 """Get LATENCY from the test message.
526 :param msg: The test message to be parsed.
528 :returns: Parsed data as a dict and the status (PASS/FAIL).
529 :rtype: tuple(dict, str)
534 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
535 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
538 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
539 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
543 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
545 if groups is not None:
546 keys = ("min", "avg", "max")
548 latency["NDR"]["direction1"] = dict(
549 zip(keys, [float(l) for l in groups.group(1).split('/')]))
550 latency["NDR"]["direction2"] = dict(
551 zip(keys, [float(l) for l in groups.group(2).split('/')]))
552 latency["PDR"]["direction1"] = dict(
553 zip(keys, [float(l) for l in groups.group(3).split('/')]))
554 latency["PDR"]["direction2"] = dict(
555 zip(keys, [float(l) for l in groups.group(4).split('/')]))
557 except (IndexError, ValueError):
560 return latency, status
562 def visit_suite(self, suite):
563 """Implements traversing through the suite and its direct children.
565 :param suite: Suite to process.
569 if self.start_suite(suite) is not False:
570 suite.suites.visit(self)
571 suite.tests.visit(self)
572 self.end_suite(suite)
574 def start_suite(self, suite):
575 """Called when suite starts.
577 :param suite: Suite to process.
583 parent_name = suite.parent.name
584 except AttributeError:
587 doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
588 replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
589 doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
591 self._data["suites"][suite.longname.lower().replace('"', "'").
592 replace(" ", "_")] = {
593 "name": suite.name.lower(),
595 "parent": parent_name,
596 "level": len(suite.longname.split("."))
599 suite.keywords.visit(self)
601 def end_suite(self, suite):
602 """Called when suite ends.
604 :param suite: Suite to process.
610 def visit_test(self, test):
611 """Implements traversing through the test.
613 :param test: Test to process.
617 if self.start_test(test) is not False:
618 test.keywords.visit(self)
621 def start_test(self, test):
622 """Called when test starts.
624 :param test: Test to process.
629 tags = [str(tag) for tag in test.tags]
631 test_result["name"] = test.name.lower()
632 # Remove TC number from the TC name (not needed):
633 test_result["name"] = re.sub(self.REGEX_TC_NUMBER, "",
635 test_result["parent"] = test.parent.name.lower()
636 test_result["tags"] = tags
637 doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
638 replace('\r', '').replace('[', ' |br| [')
639 test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
640 test_result["msg"] = test.message.replace('\n', ' |br| '). \
641 replace('\r', '').replace('"', "'")
642 test_result["type"] = "FUNC"
643 test_result["status"] = test.status
644 # Remove TC number from the TC long name (backward compatibility):
645 self._test_ID = re.sub(self.REGEX_TC_NUMBER, "", test.longname.lower())
647 if "PERFTEST" in tags:
648 # Replace info about cores (e.g. -1c-) with the info about threads
649 # and cores (e.g. -1t1c-) in the long test case names and in the
650 # test case names if necessary.
651 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_ID)
654 for tag in test_result["tags"]:
655 groups = re.search(self.REGEX_TC_TAG, tag)
661 self._test_ID = re.sub(self.REGEX_TC_NAME_NEW,
662 "-{0}-".format(tag_tc.lower()),
665 test_result["name"] = re.sub(self.REGEX_TC_NAME_NEW,
666 "-{0}-".format(tag_tc.lower()),
670 test_result["status"] = "FAIL"
671 self._data["tests"][self._test_ID] = test_result
672 logging.error("The test '{0}' has no or more than one "
673 "multi-threading tags.".format(self._test_ID))
676 if test.status == "PASS" and ("NDRPDRDISC" in tags or
681 # TODO: Remove when definitely no NDRPDRDISC tests are used:
682 if "NDRDISC" in tags:
683 test_result["type"] = "NDR"
684 # TODO: Remove when definitely no NDRPDRDISC tests are used:
685 elif "PDRDISC" in tags:
686 test_result["type"] = "PDR"
687 elif "NDRPDR" in tags:
688 test_result["type"] = "NDRPDR"
690 test_result["type"] = "TCP"
692 test_result["type"] = "MRR"
693 elif "FRMOBL" in tags or "BMRR" in tags:
694 test_result["type"] = "BMRR"
696 test_result["status"] = "FAIL"
697 self._data["tests"][self._test_ID] = test_result
700 # TODO: Remove when definitely no NDRPDRDISC tests are used:
701 if test_result["type"] in ("NDR", "PDR"):
703 rate_value = str(re.search(
704 self.REGEX_RATE, test.message).group(1))
705 except AttributeError:
708 rate_unit = str(re.search(
709 self.REGEX_RATE, test.message).group(2))
710 except AttributeError:
713 test_result["throughput"] = dict()
714 test_result["throughput"]["value"] = \
715 int(rate_value.split('.')[0])
716 test_result["throughput"]["unit"] = rate_unit
717 test_result["latency"] = \
718 self._get_latency(test.message, test_result["type"])
719 if test_result["type"] == "PDR":
720 test_result["lossTolerance"] = str(re.search(
721 self.REGEX_TOLERANCE, test.message).group(1))
723 elif test_result["type"] in ("NDRPDR", ):
724 test_result["throughput"], test_result["status"] = \
725 self._get_ndrpdr_throughput(test.message)
726 test_result["latency"], test_result["status"] = \
727 self._get_ndrpdr_latency(test.message)
729 elif test_result["type"] in ("TCP", ):
730 groups = re.search(self.REGEX_TCP, test.message)
731 test_result["result"] = int(groups.group(2))
733 elif test_result["type"] in ("MRR", "BMRR"):
734 test_result["result"] = dict()
735 groups = re.search(self.REGEX_BMRR, test.message)
736 if groups is not None:
737 items_str = groups.group(1)
738 items_float = [float(item.strip()) for item
739 in items_str.split(",")]
740 test_result["result"]["receive-rate"] = \
741 AvgStdevMetadataFactory.from_data(items_float)
743 groups = re.search(self.REGEX_MRR, test.message)
744 test_result["result"]["receive-rate"] = \
745 AvgStdevMetadataFactory.from_data([
746 float(groups.group(3)) / float(groups.group(1)), ])
748 self._data["tests"][self._test_ID] = test_result
750 def end_test(self, test):
751 """Called when test ends.
753 :param test: Test to process.
759 def visit_keyword(self, keyword):
760 """Implements traversing through the keyword and its child keywords.
762 :param keyword: Keyword to process.
763 :type keyword: Keyword
766 if self.start_keyword(keyword) is not False:
767 self.end_keyword(keyword)
769 def start_keyword(self, keyword):
770 """Called when keyword starts. Default implementation does nothing.
772 :param keyword: Keyword to process.
773 :type keyword: Keyword
777 if keyword.type == "setup":
778 self.visit_setup_kw(keyword)
779 elif keyword.type == "teardown":
780 self._lookup_kw_nr = 0
781 self.visit_teardown_kw(keyword)
783 self._lookup_kw_nr = 0
784 self.visit_test_kw(keyword)
785 except AttributeError:
788 def end_keyword(self, keyword):
789 """Called when keyword ends. Default implementation does nothing.
791 :param keyword: Keyword to process.
792 :type keyword: Keyword
797 def visit_test_kw(self, test_kw):
798 """Implements traversing through the test keyword and its child
801 :param test_kw: Keyword to process.
802 :type test_kw: Keyword
805 for keyword in test_kw.keywords:
806 if self.start_test_kw(keyword) is not False:
807 self.visit_test_kw(keyword)
808 self.end_test_kw(keyword)
810 def start_test_kw(self, test_kw):
811 """Called when test keyword starts. Default implementation does
814 :param test_kw: Keyword to process.
815 :type test_kw: Keyword
818 if test_kw.name.count("Show Runtime Counters On All Duts"):
819 self._lookup_kw_nr += 1
820 self._show_run_lookup_nr = 0
821 self._msg_type = "test-show-runtime"
822 elif test_kw.name.count("Start The L2fwd Test") and not self._version:
823 self._msg_type = "dpdk-version"
826 test_kw.messages.visit(self)
828 def end_test_kw(self, test_kw):
829 """Called when keyword ends. Default implementation does nothing.
831 :param test_kw: Keyword to process.
832 :type test_kw: Keyword
837 def visit_setup_kw(self, setup_kw):
838 """Implements traversing through the teardown keyword and its child
841 :param setup_kw: Keyword to process.
842 :type setup_kw: Keyword
845 for keyword in setup_kw.keywords:
846 if self.start_setup_kw(keyword) is not False:
847 self.visit_setup_kw(keyword)
848 self.end_setup_kw(keyword)
850 def start_setup_kw(self, setup_kw):
851 """Called when teardown keyword starts. Default implementation does
854 :param setup_kw: Keyword to process.
855 :type setup_kw: Keyword
858 if setup_kw.name.count("Show Vpp Version On All Duts") \
859 and not self._version:
860 self._msg_type = "vpp-version"
862 elif setup_kw.name.count("Setup performance global Variables") \
863 and not self._timestamp:
864 self._msg_type = "timestamp"
867 setup_kw.messages.visit(self)
869 def end_setup_kw(self, setup_kw):
870 """Called when keyword ends. Default implementation does nothing.
872 :param setup_kw: Keyword to process.
873 :type setup_kw: Keyword
878 def visit_teardown_kw(self, teardown_kw):
879 """Implements traversing through the teardown keyword and its child
882 :param teardown_kw: Keyword to process.
883 :type teardown_kw: Keyword
886 for keyword in teardown_kw.keywords:
887 if self.start_teardown_kw(keyword) is not False:
888 self.visit_teardown_kw(keyword)
889 self.end_teardown_kw(keyword)
891 def start_teardown_kw(self, teardown_kw):
892 """Called when teardown keyword starts. Default implementation does
895 :param teardown_kw: Keyword to process.
896 :type teardown_kw: Keyword
900 if teardown_kw.name.count("Show Vat History On All Duts"):
901 self._vat_history_lookup_nr = 0
902 self._msg_type = "teardown-vat-history"
903 teardown_kw.messages.visit(self)
905 def end_teardown_kw(self, teardown_kw):
906 """Called when keyword ends. Default implementation does nothing.
908 :param teardown_kw: Keyword to process.
909 :type teardown_kw: Keyword
914 def visit_message(self, msg):
915 """Implements visiting the message.
917 :param msg: Message to process.
921 if self.start_message(msg) is not False:
922 self.end_message(msg)
924 def start_message(self, msg):
925 """Called when message starts. Get required information from messages:
928 :param msg: Message to process.
934 self.parse_msg[self._msg_type](msg)
936 def end_message(self, msg):
937 """Called when message ends. Default implementation does nothing.
939 :param msg: Message to process.
946 class InputData(object):
949 The data is extracted from output.xml files generated by Jenkins jobs and
950 stored in pandas' DataFrames.
956 (as described in ExecutionChecker documentation)
958 (as described in ExecutionChecker documentation)
960 (as described in ExecutionChecker documentation)
963 def __init__(self, spec):
966 :param spec: Specification.
967 :type spec: Specification
974 self._input_data = pd.Series()
978 """Getter - Input data.
981 :rtype: pandas.Series
983 return self._input_data
985 def metadata(self, job, build):
988 :param job: Job which metadata we want.
989 :param build: Build which metadata we want.
993 :rtype: pandas.Series
996 return self.data[job][build]["metadata"]
998 def suites(self, job, build):
1001 :param job: Job which suites we want.
1002 :param build: Build which suites we want.
1006 :rtype: pandas.Series
1009 return self.data[job][str(build)]["suites"]
1011 def tests(self, job, build):
1014 :param job: Job which tests we want.
1015 :param build: Build which tests we want.
1019 :rtype: pandas.Series
1022 return self.data[job][build]["tests"]
1025 def _parse_tests(job, build, log):
1026 """Process data from robot output.xml file and return JSON structured
1029 :param job: The name of job which build output data will be processed.
1030 :param build: The build which output data will be processed.
1031 :param log: List of log messages.
1034 :type log: list of tuples (severity, msg)
1035 :returns: JSON data structure.
1044 with open(build["file-name"], 'r') as data_file:
1046 result = ExecutionResult(data_file)
1047 except errors.DataError as err:
1048 log.append(("ERROR", "Error occurred while parsing output.xml: "
1051 checker = ExecutionChecker(metadata)
1052 result.visit(checker)
1056 def _download_and_parse_build(self, pid, data_queue, job, build, repeat):
1057 """Download and parse the input data file.
1059 :param pid: PID of the process executing this method.
1060 :param data_queue: Shared memory between processes. Queue which keeps
1061 the result data. This data is then read by the main process and used
1062 in further processing.
1063 :param job: Name of the Jenkins job which generated the processed input
1065 :param build: Information about the Jenkins build which generated the
1066 processed input file.
1067 :param repeat: Repeat the download specified number of times if not
1070 :type data_queue: multiprocessing.Manager().Queue()
1078 logging.info(" Processing the job/build: {0}: {1}".
1079 format(job, build["build"]))
1081 logs.append(("INFO", " Processing the job/build: {0}: {1}".
1082 format(job, build["build"])))
1089 success = download_and_unzip_data_file(self._cfg, job, build, pid,
1095 logs.append(("ERROR", "It is not possible to download the input "
1096 "data file from the job '{job}', build "
1097 "'{build}', or it is damaged. Skipped.".
1098 format(job=job, build=build["build"])))
1100 logs.append(("INFO", " Processing data from the build '{0}' ...".
1101 format(build["build"])))
1102 data = InputData._parse_tests(job, build, logs)
1104 logs.append(("ERROR", "Input data file from the job '{job}', "
1105 "build '{build}' is damaged. Skipped.".
1106 format(job=job, build=build["build"])))
1111 remove(build["file-name"])
1112 except OSError as err:
1113 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
1114 format(build["file-name"], err)))
1115 logs.append(("INFO", " Done."))
1124 data_queue.put(result)
1126 def download_and_parse_data(self, repeat=1):
1127 """Download the input data files, parse input data from input files and
1128 store in pandas' Series.
1130 :param repeat: Repeat the download specified number of times if not
1135 logging.info("Downloading and parsing input files ...")
1137 work_queue = multiprocessing.JoinableQueue()
1138 manager = multiprocessing.Manager()
1139 data_queue = manager.Queue()
1140 cpus = multiprocessing.cpu_count()
1143 for cpu in range(cpus):
1144 worker = Worker(work_queue,
1146 self._download_and_parse_build)
1147 worker.daemon = True
1149 workers.append(worker)
1150 os.system("taskset -p -c {0} {1} > /dev/null 2>&1".
1151 format(cpu, worker.pid))
1153 for job, builds in self._cfg.builds.items():
1154 for build in builds:
1155 work_queue.put((job, build, repeat))
1159 logging.info("Done.")
1161 while not data_queue.empty():
1162 result = data_queue.get()
1165 build_nr = result["build"]["build"]
1168 data = result["data"]
1169 build_data = pd.Series({
1170 "metadata": pd.Series(data["metadata"].values(),
1171 index=data["metadata"].keys()),
1172 "suites": pd.Series(data["suites"].values(),
1173 index=data["suites"].keys()),
1174 "tests": pd.Series(data["tests"].values(),
1175 index=data["tests"].keys())})
1177 if self._input_data.get(job, None) is None:
1178 self._input_data[job] = pd.Series()
1179 self._input_data[job][str(build_nr)] = build_data
1181 self._cfg.set_input_file_name(job, build_nr,
1182 result["build"]["file-name"])
1184 self._cfg.set_input_state(job, build_nr, result["state"])
1186 for item in result["logs"]:
1187 if item[0] == "INFO":
1188 logging.info(item[1])
1189 elif item[0] == "ERROR":
1190 logging.error(item[1])
1191 elif item[0] == "DEBUG":
1192 logging.debug(item[1])
1193 elif item[0] == "CRITICAL":
1194 logging.critical(item[1])
1195 elif item[0] == "WARNING":
1196 logging.warning(item[1])
1200 # Terminate all workers
1201 for worker in workers:
1205 logging.info("Done.")
1208 def _end_of_tag(tag_filter, start=0, closer="'"):
1209 """Return the index of character in the string which is the end of tag.
1211 :param tag_filter: The string where the end of tag is being searched.
1212 :param start: The index where the searching is stated.
1213 :param closer: The character which is the tag closer.
1214 :type tag_filter: str
1217 :returns: The index of the tag closer.
1222 idx_opener = tag_filter.index(closer, start)
1223 return tag_filter.index(closer, idx_opener + 1)
1228 def _condition(tag_filter):
1229 """Create a conditional statement from the given tag filter.
1231 :param tag_filter: Filter based on tags from the element specification.
1232 :type tag_filter: str
1233 :returns: Conditional statement which can be evaluated.
1239 index = InputData._end_of_tag(tag_filter, index)
1243 tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
1245 def filter_data(self, element, params=None, data_set="tests",
1246 continue_on_error=False):
1247 """Filter required data from the given jobs and builds.
1249 The output data structure is:
1253 - test (or suite) 1 ID:
1259 - test (or suite) n ID:
1266 :param element: Element which will use the filtered data.
1267 :param params: Parameters which will be included in the output. If None,
1268 all parameters are included.
1269 :param data_set: The set of data to be filtered: tests, suites,
1271 :param continue_on_error: Continue if there is error while reading the
1272 data. The Item will be empty then
1273 :type element: pandas.Series
1276 :type continue_on_error: bool
1277 :returns: Filtered data.
1278 :rtype pandas.Series
1282 if element["filter"] in ("all", "template"):
1285 cond = InputData._condition(element["filter"])
1286 logging.debug(" Filter: {0}".format(cond))
1288 logging.error(" No filter defined.")
1292 params = element.get("parameters", None)
1294 params.append("type")
1298 for job, builds in element["data"].items():
1299 data[job] = pd.Series()
1300 for build in builds:
1301 data[job][str(build)] = pd.Series()
1303 data_iter = self.data[job][str(build)][data_set].\
1306 if continue_on_error:
1310 for test_ID, test_data in data_iter:
1311 if eval(cond, {"tags": test_data.get("tags", "")}):
1312 data[job][str(build)][test_ID] = pd.Series()
1314 for param, val in test_data.items():
1315 data[job][str(build)][test_ID][param] = val
1317 for param in params:
1319 data[job][str(build)][test_ID][param] =\
1322 data[job][str(build)][test_ID][param] =\
1326 except (KeyError, IndexError, ValueError) as err:
1327 logging.error(" Missing mandatory parameter in the element "
1328 "specification: {0}".format(err))
1330 except AttributeError:
1333 logging.error(" The filter '{0}' is not correct. Check if all "
1334 "tags are enclosed by apostrophes.".format(cond))
1338 def merge_data(data):
1339 """Merge data from more jobs and builds to a simple data structure.
1341 The output data structure is:
1343 - test (suite) 1 ID:
1349 - test (suite) n ID:
1352 :param data: Data to merge.
1353 :type data: pandas.Series
1354 :returns: Merged data.
1355 :rtype: pandas.Series
1358 logging.info(" Merging data ...")
1360 merged_data = pd.Series()
1361 for _, builds in data.iteritems():
1362 for _, item in builds.iteritems():
1363 for ID, item_data in item.iteritems():
1364 merged_data[ID] = item_data