1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
22 import multiprocessing
28 from robot.api import ExecutionResult, ResultVisitor
29 from robot import errors
30 from collections import OrderedDict
31 from string import replace
33 from os.path import join
34 from datetime import datetime as dt
35 from datetime import timedelta
36 from json import loads
37 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
39 from input_data_files import download_and_unzip_data_file
40 from utils import Worker
43 # Separator used in file names
47 class ExecutionChecker(ResultVisitor):
48 """Class to traverse through the test suite structure.
50 The functionality implemented in this class generates a json structure:
56 "generated": "Timestamp",
57 "version": "SUT version",
58 "job": "Jenkins job name",
59 "build": "Information about the build"
62 "Suite long name 1": {
64 "doc": "Suite 1 documentation",
65 "parent": "Suite 1 parent",
66 "level": "Level of the suite in the suite hierarchy"
68 "Suite long name N": {
70 "doc": "Suite N documentation",
71 "parent": "Suite 2 parent",
72 "level": "Level of the suite in the suite hierarchy"
79 "parent": "Name of the parent of the test",
80 "doc": "Test documentation",
81 "msg": "Test message",
82 "vat-history": "DUT1 and DUT2 VAT History",
83 "show-run": "Show Run",
84 "tags": ["tag 1", "tag 2", "tag n"],
86 "status": "PASS" | "FAIL",
128 "parent": "Name of the parent of the test",
129 "doc": "Test documentation",
130 "msg": "Test message",
131 "tags": ["tag 1", "tag 2", "tag n"],
133 "status": "PASS" | "FAIL",
140 "parent": "Name of the parent of the test",
141 "doc": "Test documentation",
142 "msg": "Test message",
143 "tags": ["tag 1", "tag 2", "tag n"],
144 "type": "MRR" | "BMRR",
145 "status": "PASS" | "FAIL",
147 "receive-rate": AvgStdevMetadata,
151 # TODO: Remove when definitely no NDRPDRDISC tests are used:
155 "parent": "Name of the parent of the test",
156 "doc": "Test documentation",
157 "msg": "Test message",
158 "tags": ["tag 1", "tag 2", "tag n"],
159 "type": "PDR" | "NDR",
160 "status": "PASS" | "FAIL",
161 "throughput": { # Only type: "PDR" | "NDR"
163 "unit": "pps" | "bps" | "percentage"
165 "latency": { # Only type: "PDR" | "NDR"
172 "50": { # Only for NDR
177 "10": { # Only for NDR
189 "50": { # Only for NDR
194 "10": { # Only for NDR
201 "lossTolerance": "lossTolerance", # Only type: "PDR"
202 "vat-history": "DUT1 and DUT2 VAT History"
203 "show-run": "Show Run"
215 "metadata": { # Optional
216 "version": "VPP version",
217 "job": "Jenkins job name",
218 "build": "Information about the build"
222 "doc": "Suite 1 documentation",
223 "parent": "Suite 1 parent",
224 "level": "Level of the suite in the suite hierarchy"
227 "doc": "Suite N documentation",
228 "parent": "Suite 2 parent",
229 "level": "Level of the suite in the suite hierarchy"
235 "parent": "Name of the parent of the test",
236 "doc": "Test documentation"
237 "msg": "Test message"
238 "tags": ["tag 1", "tag 2", "tag n"],
239 "vat-history": "DUT1 and DUT2 VAT History"
240 "show-run": "Show Run"
241 "status": "PASS" | "FAIL"
249 .. note:: ID is the lowercase full path to the test.
252 # TODO: Remove when definitely no NDRPDRDISC tests are used:
253 REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
255 REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
256 r'NDR_UPPER:\s(\d+.\d+).*\n'
257 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
258 r'PDR_UPPER:\s(\d+.\d+)')
260 # TODO: Remove when definitely no NDRPDRDISC tests are used:
261 REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
262 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
263 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
264 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
265 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
266 r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
267 r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
269 REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
270 r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
271 r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
273 REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
274 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
276 REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
279 REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|"
280 r"VPP Version:\s*)(.*)")
282 REGEX_VERSION_DPDK = re.compile(r"(return STDOUT testpmd)([\d\D\n]*)"
283 r"(RTE Version: 'DPDK )(.*)(')")
285 REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
287 REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
288 r'tx\s(\d*),\srx\s(\d*)')
290 REGEX_BMRR = re.compile(r'Maximum Receive Rate trial results'
291 r' in packets per second: \[(.*)\]')
293 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
295 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
297 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
299 REGEX_TC_NUMBER = re.compile(r'tc[0-9]{2}-')
301 def __init__(self, metadata, mapping, ignore):
304 :param metadata: Key-value pairs to be included in "metadata" part of
306 :param mapping: Mapping of the old names of test cases to the new
308 :param ignore: List of TCs to be ignored.
314 # Type of message to parse out from the test messages
315 self._msg_type = None
321 self._timestamp = None
323 # Testbed. The testbed is identified by TG node IP address.
326 # Mapping of TCs long names
327 self._mapping = mapping
330 self._ignore = ignore
332 # Number of VAT History messages found:
334 # 1 - VAT History of DUT1
335 # 2 - VAT History of DUT2
336 self._lookup_kw_nr = 0
337 self._vat_history_lookup_nr = 0
339 # Number of Show Running messages found
341 # 1 - Show run message found
342 self._show_run_lookup_nr = 0
344 # Test ID of currently processed test- the lowercase full path to the
348 # The main data structure
350 "metadata": OrderedDict(),
351 "suites": OrderedDict(),
352 "tests": OrderedDict()
355 # Save the provided metadata
356 for key, val in metadata.items():
357 self._data["metadata"][key] = val
359 # Dictionary defining the methods used to parse different types of
362 "timestamp": self._get_timestamp,
363 "vpp-version": self._get_vpp_version,
364 "dpdk-version": self._get_dpdk_version,
365 "teardown-vat-history": self._get_vat_history,
366 "test-show-runtime": self._get_show_run,
367 "testbed": self._get_testbed
372 """Getter - Data parsed from the XML file.
374 :returns: Data parsed from the XML file.
379 def _get_testbed(self, msg):
380 """Called when extraction of testbed IP is required.
381 The testbed is identified by TG node IP address.
383 :param msg: Message to process.
388 if msg.message.count("Arguments:"):
389 message = str(msg.message).replace(' ', '').replace('\n', '').\
390 replace("'", '"').replace('b"', '"').\
391 replace("honeycom", "honeycomb")
392 message = loads(message[11:-1])
394 self._testbed = message["TG"]["host"]
395 except (KeyError, ValueError):
398 self._data["metadata"]["testbed"] = self._testbed
399 self._msg_type = None
401 def _get_vpp_version(self, msg):
402 """Called when extraction of VPP version is required.
404 :param msg: Message to process.
409 if msg.message.count("return STDOUT Version:") or \
410 msg.message.count("VPP Version:"):
411 self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
413 self._data["metadata"]["version"] = self._version
414 self._msg_type = None
416 def _get_dpdk_version(self, msg):
417 """Called when extraction of DPDK version is required.
419 :param msg: Message to process.
424 if msg.message.count("return STDOUT testpmd"):
426 self._version = str(re.search(
427 self.REGEX_VERSION_DPDK, msg.message). group(4))
428 self._data["metadata"]["version"] = self._version
432 self._msg_type = None
434 def _get_timestamp(self, msg):
435 """Called when extraction of timestamp is required.
437 :param msg: Message to process.
442 self._timestamp = msg.timestamp[:14]
443 self._data["metadata"]["generated"] = self._timestamp
444 self._msg_type = None
446 def _get_vat_history(self, msg):
447 """Called when extraction of VAT command history is required.
449 :param msg: Message to process.
453 if msg.message.count("VAT command history:"):
454 self._vat_history_lookup_nr += 1
455 if self._vat_history_lookup_nr == 1:
456 self._data["tests"][self._test_ID]["vat-history"] = str()
458 self._msg_type = None
459 text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
460 "VAT command history:", "", msg.message, count=1). \
461 replace("\n\n", "\n").replace('\n', ' |br| ').\
462 replace('\r', '').replace('"', "'")
464 self._data["tests"][self._test_ID]["vat-history"] += " |br| "
465 self._data["tests"][self._test_ID]["vat-history"] += \
466 "**DUT" + str(self._vat_history_lookup_nr) + ":** " + text
468 def _get_show_run(self, msg):
469 """Called when extraction of VPP operational data (output of CLI command
470 Show Runtime) is required.
472 :param msg: Message to process.
476 if msg.message.count("return STDOUT Thread "):
477 self._show_run_lookup_nr += 1
478 if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
479 self._data["tests"][self._test_ID]["show-run"] = str()
480 if self._lookup_kw_nr > 1:
481 self._msg_type = None
482 if self._show_run_lookup_nr == 1:
483 text = msg.message.replace("vat# ", "").\
484 replace("return STDOUT ", "").replace("\n\n", "\n").\
485 replace('\n', ' |br| ').\
486 replace('\r', '').replace('"', "'")
488 self._data["tests"][self._test_ID]["show-run"] += " |br| "
489 self._data["tests"][self._test_ID]["show-run"] += \
490 "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
494 # TODO: Remove when definitely no NDRPDRDISC tests are used:
495 def _get_latency(self, msg, test_type):
496 """Get the latency data from the test message.
498 :param msg: Message to be parsed.
499 :param test_type: Type of the test - NDR or PDR.
502 :returns: Latencies parsed from the message.
506 if test_type == "NDR":
507 groups = re.search(self.REGEX_LAT_NDR, msg)
508 groups_range = range(1, 7)
509 elif test_type == "PDR":
510 groups = re.search(self.REGEX_LAT_PDR, msg)
511 groups_range = range(1, 3)
516 for idx in groups_range:
518 lat = [int(item) for item in str(groups.group(idx)).split('/')]
519 except (AttributeError, ValueError):
521 latencies.append(lat)
523 keys = ("min", "avg", "max")
531 latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
532 latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
533 if test_type == "NDR":
534 latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
535 latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
536 latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
537 latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
541 def _get_ndrpdr_throughput(self, msg):
542 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
545 :param msg: The test message to be parsed.
547 :returns: Parsed data as a dict and the status (PASS/FAIL).
548 :rtype: tuple(dict, str)
552 "NDR": {"LOWER": -1.0, "UPPER": -1.0},
553 "PDR": {"LOWER": -1.0, "UPPER": -1.0}
556 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
558 if groups is not None:
560 throughput["NDR"]["LOWER"] = float(groups.group(1))
561 throughput["NDR"]["UPPER"] = float(groups.group(2))
562 throughput["PDR"]["LOWER"] = float(groups.group(3))
563 throughput["PDR"]["UPPER"] = float(groups.group(4))
565 except (IndexError, ValueError):
568 return throughput, status
570 def _get_ndrpdr_latency(self, msg):
571 """Get LATENCY from the test message.
573 :param msg: The test message to be parsed.
575 :returns: Parsed data as a dict and the status (PASS/FAIL).
576 :rtype: tuple(dict, str)
581 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
582 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
585 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
586 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
590 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
592 if groups is not None:
593 keys = ("min", "avg", "max")
595 latency["NDR"]["direction1"] = dict(
596 zip(keys, [float(l) for l in groups.group(1).split('/')]))
597 latency["NDR"]["direction2"] = dict(
598 zip(keys, [float(l) for l in groups.group(2).split('/')]))
599 latency["PDR"]["direction1"] = dict(
600 zip(keys, [float(l) for l in groups.group(3).split('/')]))
601 latency["PDR"]["direction2"] = dict(
602 zip(keys, [float(l) for l in groups.group(4).split('/')]))
604 except (IndexError, ValueError):
607 return latency, status
609 def visit_suite(self, suite):
610 """Implements traversing through the suite and its direct children.
612 :param suite: Suite to process.
616 if self.start_suite(suite) is not False:
617 suite.suites.visit(self)
618 suite.tests.visit(self)
619 self.end_suite(suite)
621 def start_suite(self, suite):
622 """Called when suite starts.
624 :param suite: Suite to process.
630 parent_name = suite.parent.name
631 except AttributeError:
634 doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
635 replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
636 doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
638 self._data["suites"][suite.longname.lower().replace('"', "'").
639 replace(" ", "_")] = {
640 "name": suite.name.lower(),
642 "parent": parent_name,
643 "level": len(suite.longname.split("."))
646 suite.keywords.visit(self)
648 def end_suite(self, suite):
649 """Called when suite ends.
651 :param suite: Suite to process.
657 def visit_test(self, test):
658 """Implements traversing through the test.
660 :param test: Test to process.
664 if self.start_test(test) is not False:
665 test.keywords.visit(self)
668 def start_test(self, test):
669 """Called when test starts.
671 :param test: Test to process.
676 longname_orig = test.longname.lower()
678 # Check the ignore list
679 if longname_orig in self._ignore:
682 tags = [str(tag) for tag in test.tags]
685 # Change the TC long name and name if defined in the mapping table
686 longname = self._mapping.get(longname_orig, None)
687 if longname is not None:
688 name = longname.split('.')[-1]
689 logging.debug("{0}\n{1}\n{2}\n{3}".format(
690 self._data["metadata"], longname_orig, longname, name))
692 longname = longname_orig
693 name = test.name.lower()
695 # Remove TC number from the TC long name (backward compatibility):
696 self._test_ID = re.sub(self.REGEX_TC_NUMBER, "", longname)
697 # Remove TC number from the TC name (not needed):
698 test_result["name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
700 test_result["parent"] = test.parent.name.lower()
701 test_result["tags"] = tags
702 doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
703 replace('\r', '').replace('[', ' |br| [')
704 test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
705 test_result["msg"] = test.message.replace('\n', ' |br| '). \
706 replace('\r', '').replace('"', "'")
707 test_result["type"] = "FUNC"
708 test_result["status"] = test.status
710 if "PERFTEST" in tags:
711 # Replace info about cores (e.g. -1c-) with the info about threads
712 # and cores (e.g. -1t1c-) in the long test case names and in the
713 # test case names if necessary.
714 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_ID)
717 for tag in test_result["tags"]:
718 groups = re.search(self.REGEX_TC_TAG, tag)
724 self._test_ID = re.sub(self.REGEX_TC_NAME_NEW,
725 "-{0}-".format(tag_tc.lower()),
728 test_result["name"] = re.sub(self.REGEX_TC_NAME_NEW,
729 "-{0}-".format(tag_tc.lower()),
733 test_result["status"] = "FAIL"
734 self._data["tests"][self._test_ID] = test_result
735 logging.debug("The test '{0}' has no or more than one "
736 "multi-threading tags.".format(self._test_ID))
737 logging.debug("Tags: {0}".format(test_result["tags"]))
740 if test.status == "PASS" and ("NDRPDRDISC" in tags or
745 # TODO: Remove when definitely no NDRPDRDISC tests are used:
746 if "NDRDISC" in tags:
747 test_result["type"] = "NDR"
748 # TODO: Remove when definitely no NDRPDRDISC tests are used:
749 elif "PDRDISC" in tags:
750 test_result["type"] = "PDR"
751 elif "NDRPDR" in tags:
752 test_result["type"] = "NDRPDR"
754 test_result["type"] = "TCP"
756 test_result["type"] = "MRR"
757 elif "FRMOBL" in tags or "BMRR" in tags:
758 test_result["type"] = "BMRR"
760 test_result["status"] = "FAIL"
761 self._data["tests"][self._test_ID] = test_result
764 # TODO: Remove when definitely no NDRPDRDISC tests are used:
765 if test_result["type"] in ("NDR", "PDR"):
767 rate_value = str(re.search(
768 self.REGEX_RATE, test.message).group(1))
769 except AttributeError:
772 rate_unit = str(re.search(
773 self.REGEX_RATE, test.message).group(2))
774 except AttributeError:
777 test_result["throughput"] = dict()
778 test_result["throughput"]["value"] = \
779 int(rate_value.split('.')[0])
780 test_result["throughput"]["unit"] = rate_unit
781 test_result["latency"] = \
782 self._get_latency(test.message, test_result["type"])
783 if test_result["type"] == "PDR":
784 test_result["lossTolerance"] = str(re.search(
785 self.REGEX_TOLERANCE, test.message).group(1))
787 elif test_result["type"] in ("NDRPDR", ):
788 test_result["throughput"], test_result["status"] = \
789 self._get_ndrpdr_throughput(test.message)
790 test_result["latency"], test_result["status"] = \
791 self._get_ndrpdr_latency(test.message)
793 elif test_result["type"] in ("TCP", ):
794 groups = re.search(self.REGEX_TCP, test.message)
795 test_result["result"] = int(groups.group(2))
797 elif test_result["type"] in ("MRR", "BMRR"):
798 test_result["result"] = dict()
799 groups = re.search(self.REGEX_BMRR, test.message)
800 if groups is not None:
801 items_str = groups.group(1)
802 items_float = [float(item.strip()) for item
803 in items_str.split(",")]
804 metadata = AvgStdevMetadataFactory.from_data(items_float)
805 # Next two lines have been introduced in CSIT-1179,
806 # to be removed in CSIT-1180.
809 test_result["result"]["receive-rate"] = metadata
811 groups = re.search(self.REGEX_MRR, test.message)
812 test_result["result"]["receive-rate"] = \
813 AvgStdevMetadataFactory.from_data([
814 float(groups.group(3)) / float(groups.group(1)), ])
816 self._data["tests"][self._test_ID] = test_result
818 def end_test(self, test):
819 """Called when test ends.
821 :param test: Test to process.
827 def visit_keyword(self, keyword):
828 """Implements traversing through the keyword and its child keywords.
830 :param keyword: Keyword to process.
831 :type keyword: Keyword
834 if self.start_keyword(keyword) is not False:
835 self.end_keyword(keyword)
837 def start_keyword(self, keyword):
838 """Called when keyword starts. Default implementation does nothing.
840 :param keyword: Keyword to process.
841 :type keyword: Keyword
845 if keyword.type == "setup":
846 self.visit_setup_kw(keyword)
847 elif keyword.type == "teardown":
848 self._lookup_kw_nr = 0
849 self.visit_teardown_kw(keyword)
851 self._lookup_kw_nr = 0
852 self.visit_test_kw(keyword)
853 except AttributeError:
856 def end_keyword(self, keyword):
857 """Called when keyword ends. Default implementation does nothing.
859 :param keyword: Keyword to process.
860 :type keyword: Keyword
865 def visit_test_kw(self, test_kw):
866 """Implements traversing through the test keyword and its child
869 :param test_kw: Keyword to process.
870 :type test_kw: Keyword
873 for keyword in test_kw.keywords:
874 if self.start_test_kw(keyword) is not False:
875 self.visit_test_kw(keyword)
876 self.end_test_kw(keyword)
878 def start_test_kw(self, test_kw):
879 """Called when test keyword starts. Default implementation does
882 :param test_kw: Keyword to process.
883 :type test_kw: Keyword
886 if test_kw.name.count("Show Runtime Counters On All Duts"):
887 self._lookup_kw_nr += 1
888 self._show_run_lookup_nr = 0
889 self._msg_type = "test-show-runtime"
890 elif test_kw.name.count("Start The L2fwd Test") and not self._version:
891 self._msg_type = "dpdk-version"
894 test_kw.messages.visit(self)
896 def end_test_kw(self, test_kw):
897 """Called when keyword ends. Default implementation does nothing.
899 :param test_kw: Keyword to process.
900 :type test_kw: Keyword
905 def visit_setup_kw(self, setup_kw):
906 """Implements traversing through the teardown keyword and its child
909 :param setup_kw: Keyword to process.
910 :type setup_kw: Keyword
913 for keyword in setup_kw.keywords:
914 if self.start_setup_kw(keyword) is not False:
915 self.visit_setup_kw(keyword)
916 self.end_setup_kw(keyword)
918 def start_setup_kw(self, setup_kw):
919 """Called when teardown keyword starts. Default implementation does
922 :param setup_kw: Keyword to process.
923 :type setup_kw: Keyword
926 if setup_kw.name.count("Show Vpp Version On All Duts") \
927 and not self._version:
928 self._msg_type = "vpp-version"
930 elif setup_kw.name.count("Setup performance global Variables") \
931 and not self._timestamp:
932 self._msg_type = "timestamp"
933 elif setup_kw.name.count("Setup Framework") and not self._testbed:
934 self._msg_type = "testbed"
937 setup_kw.messages.visit(self)
939 def end_setup_kw(self, setup_kw):
940 """Called when keyword ends. Default implementation does nothing.
942 :param setup_kw: Keyword to process.
943 :type setup_kw: Keyword
948 def visit_teardown_kw(self, teardown_kw):
949 """Implements traversing through the teardown keyword and its child
952 :param teardown_kw: Keyword to process.
953 :type teardown_kw: Keyword
956 for keyword in teardown_kw.keywords:
957 if self.start_teardown_kw(keyword) is not False:
958 self.visit_teardown_kw(keyword)
959 self.end_teardown_kw(keyword)
961 def start_teardown_kw(self, teardown_kw):
962 """Called when teardown keyword starts. Default implementation does
965 :param teardown_kw: Keyword to process.
966 :type teardown_kw: Keyword
970 if teardown_kw.name.count("Show Vat History On All Duts"):
971 self._vat_history_lookup_nr = 0
972 self._msg_type = "teardown-vat-history"
973 teardown_kw.messages.visit(self)
975 def end_teardown_kw(self, teardown_kw):
976 """Called when keyword ends. Default implementation does nothing.
978 :param teardown_kw: Keyword to process.
979 :type teardown_kw: Keyword
984 def visit_message(self, msg):
985 """Implements visiting the message.
987 :param msg: Message to process.
991 if self.start_message(msg) is not False:
992 self.end_message(msg)
994 def start_message(self, msg):
995 """Called when message starts. Get required information from messages:
998 :param msg: Message to process.
1004 self.parse_msg[self._msg_type](msg)
1006 def end_message(self, msg):
1007 """Called when message ends. Default implementation does nothing.
1009 :param msg: Message to process.
1016 class InputData(object):
1019 The data is extracted from output.xml files generated by Jenkins jobs and
1020 stored in pandas' DataFrames.
1026 (as described in ExecutionChecker documentation)
1028 (as described in ExecutionChecker documentation)
1030 (as described in ExecutionChecker documentation)
1033 def __init__(self, spec):
1036 :param spec: Specification.
1037 :type spec: Specification
1044 self._input_data = pd.Series()
1048 """Getter - Input data.
1050 :returns: Input data
1051 :rtype: pandas.Series
1053 return self._input_data
1055 def metadata(self, job, build):
1056 """Getter - metadata
1058 :param job: Job which metadata we want.
1059 :param build: Build which metadata we want.
1063 :rtype: pandas.Series
1066 return self.data[job][build]["metadata"]
1068 def suites(self, job, build):
1071 :param job: Job which suites we want.
1072 :param build: Build which suites we want.
1076 :rtype: pandas.Series
1079 return self.data[job][str(build)]["suites"]
1081 def tests(self, job, build):
1084 :param job: Job which tests we want.
1085 :param build: Build which tests we want.
1089 :rtype: pandas.Series
1092 return self.data[job][build]["tests"]
1094 def _parse_tests(self, job, build, log):
1095 """Process data from robot output.xml file and return JSON structured
1098 :param job: The name of job which build output data will be processed.
1099 :param build: The build which output data will be processed.
1100 :param log: List of log messages.
1103 :type log: list of tuples (severity, msg)
1104 :returns: JSON data structure.
1113 with open(build["file-name"], 'r') as data_file:
1115 result = ExecutionResult(data_file)
1116 except errors.DataError as err:
1117 log.append(("ERROR", "Error occurred while parsing output.xml: "
1120 checker = ExecutionChecker(metadata, self._cfg.mapping,
1122 result.visit(checker)
1126 def _download_and_parse_build(self, pid, data_queue, job, build, repeat):
1127 """Download and parse the input data file.
1129 :param pid: PID of the process executing this method.
1130 :param data_queue: Shared memory between processes. Queue which keeps
1131 the result data. This data is then read by the main process and used
1132 in further processing.
1133 :param job: Name of the Jenkins job which generated the processed input
1135 :param build: Information about the Jenkins build which generated the
1136 processed input file.
1137 :param repeat: Repeat the download specified number of times if not
1140 :type data_queue: multiprocessing.Manager().Queue()
1148 logging.info(" Processing the job/build: {0}: {1}".
1149 format(job, build["build"]))
1151 logs.append(("INFO", " Processing the job/build: {0}: {1}".
1152 format(job, build["build"])))
1159 success = download_and_unzip_data_file(self._cfg, job, build, pid,
1165 logs.append(("ERROR", "It is not possible to download the input "
1166 "data file from the job '{job}', build "
1167 "'{build}', or it is damaged. Skipped.".
1168 format(job=job, build=build["build"])))
1170 logs.append(("INFO", " Processing data from the build '{0}' ...".
1171 format(build["build"])))
1172 data = self._parse_tests(job, build, logs)
1174 logs.append(("ERROR", "Input data file from the job '{job}', "
1175 "build '{build}' is damaged. Skipped.".
1176 format(job=job, build=build["build"])))
1181 remove(build["file-name"])
1182 except OSError as err:
1183 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
1184 format(build["file-name"], repr(err))))
1186 # If the time-period is defined in the specification file, remove all
1187 # files which are outside the time period.
1188 timeperiod = self._cfg.input.get("time-period", None)
1189 if timeperiod and data:
1191 timeperiod = timedelta(int(timeperiod))
1192 metadata = data.get("metadata", None)
1194 generated = metadata.get("generated", None)
1196 generated = dt.strptime(generated, "%Y%m%d %H:%M")
1197 if (now - generated) > timeperiod:
1198 # Remove the data and the file:
1203 " The build {job}/{build} is outdated, will be "
1204 "removed".format(job=job, build=build["build"])))
1205 file_name = self._cfg.input["file-name"]
1207 self._cfg.environment["paths"]["DIR[WORKING,DATA]"],
1208 "{job}{sep}{build}{sep}{name}".
1211 build=build["build"],
1215 logs.append(("INFO",
1216 " The file {name} has been removed".
1217 format(name=full_name)))
1218 except OSError as err:
1219 logs.append(("ERROR",
1220 "Cannot remove the file '{0}': {1}".
1221 format(full_name, repr(err))))
1223 logs.append(("INFO", " Done."))
1232 data_queue.put(result)
1234 def download_and_parse_data(self, repeat=1):
1235 """Download the input data files, parse input data from input files and
1236 store in pandas' Series.
1238 :param repeat: Repeat the download specified number of times if not
1243 logging.info("Downloading and parsing input files ...")
1245 work_queue = multiprocessing.JoinableQueue()
1246 manager = multiprocessing.Manager()
1247 data_queue = manager.Queue()
1248 cpus = multiprocessing.cpu_count()
1251 for cpu in range(cpus):
1252 worker = Worker(work_queue,
1254 self._download_and_parse_build)
1255 worker.daemon = True
1257 workers.append(worker)
1258 os.system("taskset -p -c {0} {1} > /dev/null 2>&1".
1259 format(cpu, worker.pid))
1261 for job, builds in self._cfg.builds.items():
1262 for build in builds:
1263 work_queue.put((job, build, repeat))
1267 logging.info("Done.")
1269 while not data_queue.empty():
1270 result = data_queue.get()
1273 build_nr = result["build"]["build"]
1276 data = result["data"]
1277 build_data = pd.Series({
1278 "metadata": pd.Series(data["metadata"].values(),
1279 index=data["metadata"].keys()),
1280 "suites": pd.Series(data["suites"].values(),
1281 index=data["suites"].keys()),
1282 "tests": pd.Series(data["tests"].values(),
1283 index=data["tests"].keys())})
1285 if self._input_data.get(job, None) is None:
1286 self._input_data[job] = pd.Series()
1287 self._input_data[job][str(build_nr)] = build_data
1289 self._cfg.set_input_file_name(job, build_nr,
1290 result["build"]["file-name"])
1292 self._cfg.set_input_state(job, build_nr, result["state"])
1294 for item in result["logs"]:
1295 if item[0] == "INFO":
1296 logging.info(item[1])
1297 elif item[0] == "ERROR":
1298 logging.error(item[1])
1299 elif item[0] == "DEBUG":
1300 logging.debug(item[1])
1301 elif item[0] == "CRITICAL":
1302 logging.critical(item[1])
1303 elif item[0] == "WARNING":
1304 logging.warning(item[1])
1308 # Terminate all workers
1309 for worker in workers:
1313 logging.info("Done.")
1316 def _end_of_tag(tag_filter, start=0, closer="'"):
1317 """Return the index of character in the string which is the end of tag.
1319 :param tag_filter: The string where the end of tag is being searched.
1320 :param start: The index where the searching is stated.
1321 :param closer: The character which is the tag closer.
1322 :type tag_filter: str
1325 :returns: The index of the tag closer.
1330 idx_opener = tag_filter.index(closer, start)
1331 return tag_filter.index(closer, idx_opener + 1)
1336 def _condition(tag_filter):
1337 """Create a conditional statement from the given tag filter.
1339 :param tag_filter: Filter based on tags from the element specification.
1340 :type tag_filter: str
1341 :returns: Conditional statement which can be evaluated.
1347 index = InputData._end_of_tag(tag_filter, index)
1351 tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
1353 def filter_data(self, element, params=None, data_set="tests",
1354 continue_on_error=False):
1355 """Filter required data from the given jobs and builds.
1357 The output data structure is:
1361 - test (or suite) 1 ID:
1367 - test (or suite) n ID:
1374 :param element: Element which will use the filtered data.
1375 :param params: Parameters which will be included in the output. If None,
1376 all parameters are included.
1377 :param data_set: The set of data to be filtered: tests, suites,
1379 :param continue_on_error: Continue if there is error while reading the
1380 data. The Item will be empty then
1381 :type element: pandas.Series
1384 :type continue_on_error: bool
1385 :returns: Filtered data.
1386 :rtype pandas.Series
1390 if element["filter"] in ("all", "template"):
1393 cond = InputData._condition(element["filter"])
1394 logging.debug(" Filter: {0}".format(cond))
1396 logging.error(" No filter defined.")
1400 params = element.get("parameters", None)
1402 params.append("type")
1406 for job, builds in element["data"].items():
1407 data[job] = pd.Series()
1408 for build in builds:
1409 data[job][str(build)] = pd.Series()
1411 data_iter = self.data[job][str(build)][data_set].\
1414 if continue_on_error:
1418 for test_ID, test_data in data_iter:
1419 if eval(cond, {"tags": test_data.get("tags", "")}):
1420 data[job][str(build)][test_ID] = pd.Series()
1422 for param, val in test_data.items():
1423 data[job][str(build)][test_ID][param] = val
1425 for param in params:
1427 data[job][str(build)][test_ID][param] =\
1430 data[job][str(build)][test_ID][param] =\
1434 except (KeyError, IndexError, ValueError) as err:
1435 logging.error(" Missing mandatory parameter in the element "
1436 "specification: {0}".format(err))
1438 except AttributeError:
1441 logging.error(" The filter '{0}' is not correct. Check if all "
1442 "tags are enclosed by apostrophes.".format(cond))
1446 def merge_data(data):
1447 """Merge data from more jobs and builds to a simple data structure.
1449 The output data structure is:
1451 - test (suite) 1 ID:
1457 - test (suite) n ID:
1460 :param data: Data to merge.
1461 :type data: pandas.Series
1462 :returns: Merged data.
1463 :rtype: pandas.Series
1466 logging.info(" Merging data ...")
1468 merged_data = pd.Series()
1469 for _, builds in data.iteritems():
1470 for _, item in builds.iteritems():
1471 for ID, item_data in item.iteritems():
1472 merged_data[ID] = item_data