1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
29 from datetime import datetime as dt
30 from datetime import timedelta
31 from json import loads
36 from robot.api import ExecutionResult, ResultVisitor
37 from robot import errors
39 from resources.libraries.python import jumpavg
40 from input_data_files import download_and_unzip_data_file
43 # Separator used in file names
47 class ExecutionChecker(ResultVisitor):
48 """Class to traverse through the test suite structure.
50 The functionality implemented in this class generates a json structure:
56 "generated": "Timestamp",
57 "version": "SUT version",
58 "job": "Jenkins job name",
59 "build": "Information about the build"
62 "Suite long name 1": {
64 "doc": "Suite 1 documentation",
65 "parent": "Suite 1 parent",
66 "level": "Level of the suite in the suite hierarchy"
68 "Suite long name N": {
70 "doc": "Suite N documentation",
71 "parent": "Suite 2 parent",
72 "level": "Level of the suite in the suite hierarchy"
79 "parent": "Name of the parent of the test",
80 "doc": "Test documentation",
81 "msg": "Test message",
82 "conf-history": "DUT1 and DUT2 VAT History",
83 "show-run": "Show Run",
84 "tags": ["tag 1", "tag 2", "tag n"],
86 "status": "PASS" | "FAIL",
132 "parent": "Name of the parent of the test",
133 "doc": "Test documentation",
134 "msg": "Test message",
135 "tags": ["tag 1", "tag 2", "tag n"],
137 "status": "PASS" | "FAIL",
144 "parent": "Name of the parent of the test",
145 "doc": "Test documentation",
146 "msg": "Test message",
147 "tags": ["tag 1", "tag 2", "tag n"],
148 "type": "MRR" | "BMRR",
149 "status": "PASS" | "FAIL",
151 "receive-rate": float,
152 # Average of a list, computed using AvgStdevStats.
153 # In CSIT-1180, replace with List[float].
167 "metadata": { # Optional
168 "version": "VPP version",
169 "job": "Jenkins job name",
170 "build": "Information about the build"
174 "doc": "Suite 1 documentation",
175 "parent": "Suite 1 parent",
176 "level": "Level of the suite in the suite hierarchy"
179 "doc": "Suite N documentation",
180 "parent": "Suite 2 parent",
181 "level": "Level of the suite in the suite hierarchy"
187 "parent": "Name of the parent of the test",
188 "doc": "Test documentation"
189 "msg": "Test message"
190 "tags": ["tag 1", "tag 2", "tag n"],
191 "conf-history": "DUT1 and DUT2 VAT History"
192 "show-run": "Show Run"
193 "status": "PASS" | "FAIL"
201 .. note:: ID is the lowercase full path to the test.
204 REGEX_PLR_RATE = re.compile(r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
205 r'PLRsearch upper bound::?\s(\d+.\d+)')
207 REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
208 r'NDR_UPPER:\s(\d+.\d+).*\n'
209 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
210 r'PDR_UPPER:\s(\d+.\d+)')
212 REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
213 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
215 REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
218 REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|"
219 r"VPP Version:\s*|VPP version:\s*)(.*)")
221 REGEX_VERSION_DPDK = re.compile(r"(DPDK version:\s*|DPDK Version:\s*)(.*)")
223 REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s(\d*).*$')
225 REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
226 r'tx\s(\d*),\srx\s(\d*)')
228 REGEX_BMRR = re.compile(r'Maximum Receive Rate trial results'
229 r' in packets per second: \[(.*)\]')
231 REGEX_RECONF_LOSS = re.compile(r'Packets lost due to reconfig: (\d*)')
232 REGEX_RECONF_TIME = re.compile(r'Implied time lost: (\d*.[\de-]*)')
234 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
236 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
238 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
240 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
242 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
244 def __init__(self, metadata, mapping, ignore):
247 :param metadata: Key-value pairs to be included in "metadata" part of
249 :param mapping: Mapping of the old names of test cases to the new
251 :param ignore: List of TCs to be ignored.
257 # Type of message to parse out from the test messages
258 self._msg_type = None
264 self._timestamp = None
266 # Testbed. The testbed is identified by TG node IP address.
269 # Mapping of TCs long names
270 self._mapping = mapping
273 self._ignore = ignore
275 # Number of PAPI History messages found:
277 # 1 - PAPI History of DUT1
278 # 2 - PAPI History of DUT2
279 self._lookup_kw_nr = 0
280 self._conf_history_lookup_nr = 0
282 # Number of Show Running messages found
284 # 1 - Show run message found
285 self._show_run_lookup_nr = 0
287 # Test ID of currently processed test- the lowercase full path to the
291 # The main data structure
293 u"metadata": OrderedDict(),
294 u"suites": OrderedDict(),
295 u"tests": OrderedDict()
298 # Save the provided metadata
299 for key, val in metadata.items():
300 self._data[u"metadata"][key] = val
302 # Dictionary defining the methods used to parse different types of
305 u"timestamp": self._get_timestamp,
306 u"vpp-version": self._get_vpp_version,
307 u"dpdk-version": self._get_dpdk_version,
308 # TODO: Remove when not needed:
309 u"teardown-vat-history": self._get_vat_history,
310 u"teardown-papi-history": self._get_papi_history,
311 u"test-show-runtime": self._get_show_run,
312 u"testbed": self._get_testbed
317 """Getter - Data parsed from the XML file.
319 :returns: Data parsed from the XML file.
324 def _get_testbed(self, msg):
325 """Called when extraction of testbed IP is required.
326 The testbed is identified by TG node IP address.
328 :param msg: Message to process.
333 if msg.message.count(u"Setup of TG node") or \
334 msg.message.count(u"Setup of node TG host"):
335 reg_tg_ip = re.compile(
336 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
338 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
339 except (KeyError, ValueError, IndexError, AttributeError):
342 self._data[u"metadata"][u"testbed"] = self._testbed
343 self._msg_type = None
345 def _get_vpp_version(self, msg):
346 """Called when extraction of VPP version is required.
348 :param msg: Message to process.
353 if msg.message.count(u"return STDOUT Version:") or \
354 msg.message.count(u"VPP Version:") or \
355 msg.message.count(u"VPP version:"):
356 self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
358 self._data[u"metadata"][u"version"] = self._version
359 self._msg_type = None
361 def _get_dpdk_version(self, msg):
362 """Called when extraction of DPDK version is required.
364 :param msg: Message to process.
369 if msg.message.count(u"DPDK Version:"):
371 self._version = str(re.search(
372 self.REGEX_VERSION_DPDK, msg.message).group(2))
373 self._data[u"metadata"][u"version"] = self._version
377 self._msg_type = None
379 def _get_timestamp(self, msg):
380 """Called when extraction of timestamp is required.
382 :param msg: Message to process.
387 self._timestamp = msg.timestamp[:14]
388 self._data[u"metadata"][u"generated"] = self._timestamp
389 self._msg_type = None
391 def _get_vat_history(self, msg):
392 """Called when extraction of VAT command history is required.
394 TODO: Remove when not needed.
396 :param msg: Message to process.
400 if msg.message.count(u"VAT command history:"):
401 self._conf_history_lookup_nr += 1
402 if self._conf_history_lookup_nr == 1:
403 self._data[u"tests"][self._test_id][u"conf-history"] = str()
405 self._msg_type = None
406 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
407 r"VAT command history:", u"",
408 msg.message, count=1).replace(u'\n', u' |br| ').\
411 self._data[u"tests"][self._test_id][u"conf-history"] += (
412 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
415 def _get_papi_history(self, msg):
416 """Called when extraction of PAPI command history is required.
418 :param msg: Message to process.
422 if msg.message.count(u"PAPI command history:"):
423 self._conf_history_lookup_nr += 1
424 if self._conf_history_lookup_nr == 1:
425 self._data[u"tests"][self._test_id][u"conf-history"] = str()
427 self._msg_type = None
428 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
429 r"PAPI command history:", u"",
430 msg.message, count=1).replace(u'\n', u' |br| ').\
433 self._data[u"tests"][self._test_id][u"conf-history"] += (
434 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
437 def _get_show_run(self, msg):
438 """Called when extraction of VPP operational data (output of CLI command
439 Show Runtime) is required.
441 :param msg: Message to process.
446 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
447 self._data[u"tests"][self._test_id][u"show-run"] = str()
449 if msg.message.count(u"stats runtime") or \
450 msg.message.count(u"Runtime"):
452 host = str(re.search(self.REGEX_TC_PAPI_CLI, msg.message).
454 except (AttributeError, IndexError):
457 socket = str(re.search(self.REGEX_TC_PAPI_CLI, msg.message).
459 except (AttributeError, IndexError):
466 replace(u'b"', u'"').
467 replace(u'u"', u'"').
471 threads_nr = len(runtime[0][u"clocks"])
472 except (IndexError, KeyError):
482 table = [[tbl_hdr, ] for _ in range(threads_nr)]
484 for idx in range(threads_nr):
485 name = format(item[u"name"])
486 calls = format(item[u"calls"][idx])
487 vectors = format(item[u"vectors"][idx])
488 suspends = format(item[u"suspends"][idx])
489 if item[u"vectors"][idx] > 0:
491 item[u"clocks"][idx]/item[u"vectors"][idx], u".2e")
492 elif item[u"calls"][idx] > 0:
494 item[u"clocks"][idx]/item[u"calls"][idx], u".2e")
495 elif item[u"suspends"][idx] > 0:
497 item[u"clocks"][idx]/item[u"suspends"][idx], u".2e")
500 if item[u"calls"][idx] > 0:
501 vectors_call = format(
502 item[u"vectors"][idx]/item[u"calls"][idx], u".2f")
504 vectors_call = format(0, u".2f")
505 if int(calls) + int(vectors) + int(suspends):
507 name, calls, vectors, suspends, clocks, vectors_call
510 for idx in range(threads_nr):
511 text += f"Thread {idx} "
512 text += u"vpp_main\n" if idx == 0 else f"vpp_wk_{idx-1}\n"
514 for row in table[idx]:
515 if txt_table is None:
516 txt_table = prettytable.PrettyTable(row)
519 txt_table.add_row(row)
520 txt_table.set_style(prettytable.MSWORD_FRIENDLY)
521 txt_table.align[u"Name"] = u"l"
522 txt_table.align[u"Calls"] = u"r"
523 txt_table.align[u"Vectors"] = u"r"
524 txt_table.align[u"Suspends"] = u"r"
525 txt_table.align[u"Clocks"] = u"r"
526 txt_table.align[u"Vectors/Calls"] = u"r"
528 text += txt_table.get_string(sortby=u"Name") + u'\n'
529 text = f" \n **DUT: {host}/{socket}** \n {text}".\
530 replace(u'\n', u' |br| ').\
531 replace(u'\r', u'').\
533 self._data[u"tests"][self._test_id][u"show-run"] += text
535 def _get_ndrpdr_throughput(self, msg):
536 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
539 :param msg: The test message to be parsed.
541 :returns: Parsed data as a dict and the status (PASS/FAIL).
542 :rtype: tuple(dict, str)
546 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
547 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
550 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
552 if groups is not None:
554 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
555 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
556 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
557 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
559 except (IndexError, ValueError):
562 return throughput, status
564 def _get_plr_throughput(self, msg):
565 """Get PLRsearch lower bound and PLRsearch upper bound from the test
568 :param msg: The test message to be parsed.
570 :returns: Parsed data as a dict and the status (PASS/FAIL).
571 :rtype: tuple(dict, str)
579 groups = re.search(self.REGEX_PLR_RATE, msg)
581 if groups is not None:
583 throughput[u"LOWER"] = float(groups.group(1))
584 throughput[u"UPPER"] = float(groups.group(2))
586 except (IndexError, ValueError):
589 return throughput, status
591 def _get_ndrpdr_latency(self, msg):
592 """Get LATENCY from the test message.
594 :param msg: The test message to be parsed.
596 :returns: Parsed data as a dict and the status (PASS/FAIL).
597 :rtype: tuple(dict, str)
607 u"direction1": copy.copy(latency_default),
608 u"direction2": copy.copy(latency_default)
611 u"direction1": copy.copy(latency_default),
612 u"direction2": copy.copy(latency_default)
616 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
618 def process_latency(in_str):
619 """Return object with parsed latency values.
621 TODO: Define class for the return type.
623 :param in_str: Input string, min/avg/max/hdrh format.
625 :returns: Dict with corresponding keys, except hdrh float values.
627 :throws IndexError: If in_str does not have enough substrings.
628 :throws ValueError: If a substring does not convert to float.
630 in_list = in_str.split('/', 3)
633 u"min": float(in_list[0]),
634 u"avg": float(in_list[1]),
635 u"max": float(in_list[2]),
639 if len(in_list) == 4:
640 rval[u"hdrh"] = str(in_list[3])
644 if groups is not None:
646 latency[u"NDR"][u"direction1"] = \
647 process_latency(groups.group(1))
648 latency[u"NDR"][u"direction2"] = \
649 process_latency(groups.group(2))
650 latency[u"PDR"][u"direction1"] = \
651 process_latency(groups.group(3))
652 latency[u"PDR"][u"direction2"] = \
653 process_latency(groups.group(4))
655 except (IndexError, ValueError):
658 return latency, status
660 def visit_suite(self, suite):
661 """Implements traversing through the suite and its direct children.
663 :param suite: Suite to process.
667 if self.start_suite(suite) is not False:
668 suite.suites.visit(self)
669 suite.tests.visit(self)
670 self.end_suite(suite)
672 def start_suite(self, suite):
673 """Called when suite starts.
675 :param suite: Suite to process.
681 parent_name = suite.parent.name
682 except AttributeError:
685 doc_str = suite.doc.\
686 replace(u'"', u"'").\
687 replace(u'\n', u' ').\
688 replace(u'\r', u'').\
689 replace(u'*[', u' |br| *[').\
690 replace(u"*", u"**").\
691 replace(u' |br| *[', u'*[', 1)
693 self._data[u"suites"][suite.longname.lower().
695 replace(u" ", u"_")] = {
696 u"name": suite.name.lower(),
698 u"parent": parent_name,
699 u"level": len(suite.longname.split(u"."))
702 suite.keywords.visit(self)
704 def end_suite(self, suite):
705 """Called when suite ends.
707 :param suite: Suite to process.
712 def visit_test(self, test):
713 """Implements traversing through the test.
715 :param test: Test to process.
719 if self.start_test(test) is not False:
720 test.keywords.visit(self)
723 def start_test(self, test):
724 """Called when test starts.
726 :param test: Test to process.
731 longname_orig = test.longname.lower()
733 # Check the ignore list
734 if longname_orig in self._ignore:
737 tags = [str(tag) for tag in test.tags]
740 # Change the TC long name and name if defined in the mapping table
741 longname = self._mapping.get(longname_orig, None)
742 if longname is not None:
743 name = longname.split(u'.')[-1]
745 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
749 longname = longname_orig
750 name = test.name.lower()
752 # Remove TC number from the TC long name (backward compatibility):
753 self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
754 # Remove TC number from the TC name (not needed):
755 test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
757 test_result[u"parent"] = test.parent.name.lower()
758 test_result[u"tags"] = tags
759 test_result["doc"] = test.doc.\
760 replace(u'"', u"'").\
761 replace(u'\n', u' ').\
762 replace(u'\r', u'').\
763 replace(u'[', u' |br| [').\
764 replace(u' |br| [', u'[', 1)
765 test_result[u"msg"] = test.message.\
766 replace(u'\n', u' |br| ').\
767 replace(u'\r', u'').\
769 test_result[u"type"] = u"FUNC"
770 test_result[u"status"] = test.status
772 if u"PERFTEST" in tags:
773 # Replace info about cores (e.g. -1c-) with the info about threads
774 # and cores (e.g. -1t1c-) in the long test case names and in the
775 # test case names if necessary.
776 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
780 for tag in test_result[u"tags"]:
781 groups = re.search(self.REGEX_TC_TAG, tag)
787 self._test_id = re.sub(self.REGEX_TC_NAME_NEW,
788 f"-{tag_tc.lower()}-",
791 test_result[u"name"] = re.sub(self.REGEX_TC_NAME_NEW,
792 f"-{tag_tc.lower()}-",
796 test_result[u"status"] = u"FAIL"
797 self._data[u"tests"][self._test_id] = test_result
799 f"The test {self._test_id} has no or more than one "
800 f"multi-threading tags.\n"
801 f"Tags: {test_result[u'tags']}"
805 if test.status == u"PASS":
806 if u"NDRPDR" in tags:
807 test_result[u"type"] = u"NDRPDR"
808 test_result[u"throughput"], test_result[u"status"] = \
809 self._get_ndrpdr_throughput(test.message)
810 test_result[u"latency"], test_result[u"status"] = \
811 self._get_ndrpdr_latency(test.message)
812 elif u"SOAK" in tags:
813 test_result[u"type"] = u"SOAK"
814 test_result[u"throughput"], test_result[u"status"] = \
815 self._get_plr_throughput(test.message)
817 test_result[u"type"] = u"TCP"
818 groups = re.search(self.REGEX_TCP, test.message)
819 test_result[u"result"] = int(groups.group(2))
820 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
822 test_result[u"type"] = u"MRR"
824 test_result[u"type"] = u"BMRR"
826 test_result[u"result"] = dict()
827 groups = re.search(self.REGEX_BMRR, test.message)
828 if groups is not None:
829 items_str = groups.group(1)
830 items_float = [float(item.strip()) for item
831 in items_str.split(",")]
832 # Use whole list in CSIT-1180.
833 stats = jumpavg.AvgStdevStats.for_runs(items_float)
834 test_result[u"result"][u"receive-rate"] = stats.avg
836 groups = re.search(self.REGEX_MRR, test.message)
837 test_result[u"result"][u"receive-rate"] = \
838 float(groups.group(3)) / float(groups.group(1))
839 elif u"RECONF" in tags:
840 test_result[u"type"] = u"RECONF"
841 test_result[u"result"] = None
843 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
844 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
845 test_result[u"result"] = {
846 u"loss": int(grps_loss.group(1)),
847 u"time": float(grps_time.group(1))
849 except (AttributeError, IndexError, ValueError, TypeError):
850 test_result[u"status"] = u"FAIL"
852 test_result[u"status"] = u"FAIL"
853 self._data[u"tests"][self._test_id] = test_result
856 self._data[u"tests"][self._test_id] = test_result
858 def end_test(self, test):
859 """Called when test ends.
861 :param test: Test to process.
866 def visit_keyword(self, keyword):
867 """Implements traversing through the keyword and its child keywords.
869 :param keyword: Keyword to process.
870 :type keyword: Keyword
873 if self.start_keyword(keyword) is not False:
874 self.end_keyword(keyword)
876 def start_keyword(self, keyword):
877 """Called when keyword starts. Default implementation does nothing.
879 :param keyword: Keyword to process.
880 :type keyword: Keyword
884 if keyword.type == u"setup":
885 self.visit_setup_kw(keyword)
886 elif keyword.type == u"teardown":
887 self._lookup_kw_nr = 0
888 self.visit_teardown_kw(keyword)
890 self._lookup_kw_nr = 0
891 self.visit_test_kw(keyword)
892 except AttributeError:
895 def end_keyword(self, keyword):
896 """Called when keyword ends. Default implementation does nothing.
898 :param keyword: Keyword to process.
899 :type keyword: Keyword
903 def visit_test_kw(self, test_kw):
904 """Implements traversing through the test keyword and its child
907 :param test_kw: Keyword to process.
908 :type test_kw: Keyword
911 for keyword in test_kw.keywords:
912 if self.start_test_kw(keyword) is not False:
913 self.visit_test_kw(keyword)
914 self.end_test_kw(keyword)
916 def start_test_kw(self, test_kw):
917 """Called when test keyword starts. Default implementation does
920 :param test_kw: Keyword to process.
921 :type test_kw: Keyword
924 if test_kw.name.count(u"Show Runtime Counters On All Duts"):
925 self._lookup_kw_nr += 1
926 self._show_run_lookup_nr = 0
927 self._msg_type = u"test-show-runtime"
928 elif test_kw.name.count(u"Install Dpdk Test") and not self._version:
929 self._msg_type = u"dpdk-version"
932 test_kw.messages.visit(self)
934 def end_test_kw(self, test_kw):
935 """Called when keyword ends. Default implementation does nothing.
937 :param test_kw: Keyword to process.
938 :type test_kw: Keyword
942 def visit_setup_kw(self, setup_kw):
943 """Implements traversing through the teardown keyword and its child
946 :param setup_kw: Keyword to process.
947 :type setup_kw: Keyword
950 for keyword in setup_kw.keywords:
951 if self.start_setup_kw(keyword) is not False:
952 self.visit_setup_kw(keyword)
953 self.end_setup_kw(keyword)
955 def start_setup_kw(self, setup_kw):
956 """Called when teardown keyword starts. Default implementation does
959 :param setup_kw: Keyword to process.
960 :type setup_kw: Keyword
963 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
964 and not self._version:
965 self._msg_type = u"vpp-version"
966 elif setup_kw.name.count(u"Set Global Variable") \
967 and not self._timestamp:
968 self._msg_type = u"timestamp"
969 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
970 self._msg_type = u"testbed"
973 setup_kw.messages.visit(self)
975 def end_setup_kw(self, setup_kw):
976 """Called when keyword ends. Default implementation does nothing.
978 :param setup_kw: Keyword to process.
979 :type setup_kw: Keyword
983 def visit_teardown_kw(self, teardown_kw):
984 """Implements traversing through the teardown keyword and its child
987 :param teardown_kw: Keyword to process.
988 :type teardown_kw: Keyword
991 for keyword in teardown_kw.keywords:
992 if self.start_teardown_kw(keyword) is not False:
993 self.visit_teardown_kw(keyword)
994 self.end_teardown_kw(keyword)
996 def start_teardown_kw(self, teardown_kw):
997 """Called when teardown keyword starts
999 :param teardown_kw: Keyword to process.
1000 :type teardown_kw: Keyword
1004 if teardown_kw.name.count(u"Show Vat History On All Duts"):
1005 # TODO: Remove when not needed:
1006 self._conf_history_lookup_nr = 0
1007 self._msg_type = u"teardown-vat-history"
1008 teardown_kw.messages.visit(self)
1009 elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1010 self._conf_history_lookup_nr = 0
1011 self._msg_type = u"teardown-papi-history"
1012 teardown_kw.messages.visit(self)
1014 def end_teardown_kw(self, teardown_kw):
1015 """Called when keyword ends. Default implementation does nothing.
1017 :param teardown_kw: Keyword to process.
1018 :type teardown_kw: Keyword
1022 def visit_message(self, msg):
1023 """Implements visiting the message.
1025 :param msg: Message to process.
1029 if self.start_message(msg) is not False:
1030 self.end_message(msg)
1032 def start_message(self, msg):
1033 """Called when message starts. Get required information from messages:
1036 :param msg: Message to process.
1042 self.parse_msg[self._msg_type](msg)
1044 def end_message(self, msg):
1045 """Called when message ends. Default implementation does nothing.
1047 :param msg: Message to process.
1056 The data is extracted from output.xml files generated by Jenkins jobs and
1057 stored in pandas' DataFrames.
1063 (as described in ExecutionChecker documentation)
1065 (as described in ExecutionChecker documentation)
1067 (as described in ExecutionChecker documentation)
1070 def __init__(self, spec):
1073 :param spec: Specification.
1074 :type spec: Specification
1081 self._input_data = pd.Series()
1085 """Getter - Input data.
1087 :returns: Input data
1088 :rtype: pandas.Series
1090 return self._input_data
1092 def metadata(self, job, build):
1093 """Getter - metadata
1095 :param job: Job which metadata we want.
1096 :param build: Build which metadata we want.
1100 :rtype: pandas.Series
1103 return self.data[job][build][u"metadata"]
1105 def suites(self, job, build):
1108 :param job: Job which suites we want.
1109 :param build: Build which suites we want.
1113 :rtype: pandas.Series
1116 return self.data[job][str(build)][u"suites"]
1118 def tests(self, job, build):
1121 :param job: Job which tests we want.
1122 :param build: Build which tests we want.
1126 :rtype: pandas.Series
1129 return self.data[job][build][u"tests"]
1131 def _parse_tests(self, job, build, log):
1132 """Process data from robot output.xml file and return JSON structured
1135 :param job: The name of job which build output data will be processed.
1136 :param build: The build which output data will be processed.
1137 :param log: List of log messages.
1140 :type log: list of tuples (severity, msg)
1141 :returns: JSON data structure.
1150 with open(build[u"file-name"], u'r') as data_file:
1152 result = ExecutionResult(data_file)
1153 except errors.DataError as err:
1155 (u"ERROR", f"Error occurred while parsing output.xml: "
1159 checker = ExecutionChecker(metadata, self._cfg.mapping,
1161 result.visit(checker)
1165 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1166 """Download and parse the input data file.
1168 :param pid: PID of the process executing this method.
1169 :param job: Name of the Jenkins job which generated the processed input
1171 :param build: Information about the Jenkins build which generated the
1172 processed input file.
1173 :param repeat: Repeat the download specified number of times if not
1184 (u"INFO", f" Processing the job/build: {job}: {build[u'build']}")
1192 success = download_and_unzip_data_file(self._cfg, job, build, pid,
1200 f"It is not possible to download the input data file from the "
1201 f"job {job}, build {build[u'build']}, or it is damaged. "
1207 f" Processing data from the build {build[u'build']} ...")
1209 data = self._parse_tests(job, build, logs)
1213 f"Input data file from the job {job}, build "
1214 f"{build[u'build']} is damaged. Skipped.")
1217 state = u"processed"
1220 remove(build[u"file-name"])
1221 except OSError as err:
1223 ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
1227 # If the time-period is defined in the specification file, remove all
1228 # files which are outside the time period.
1229 timeperiod = self._cfg.input.get(u"time-period", None)
1230 if timeperiod and data:
1232 timeperiod = timedelta(int(timeperiod))
1233 metadata = data.get(u"metadata", None)
1235 generated = metadata.get(u"generated", None)
1237 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1238 if (now - generated) > timeperiod:
1239 # Remove the data and the file:
1244 f" The build {job}/{build[u'build']} is "
1245 f"outdated, will be removed.")
1247 logs.append((u"INFO", u" Done."))
1249 for level, line in logs:
1250 if level == u"INFO":
1252 elif level == u"ERROR":
1254 elif level == u"DEBUG":
1256 elif level == u"CRITICAL":
1257 logging.critical(line)
1258 elif level == u"WARNING":
1259 logging.warning(line)
1261 return {u"data": data, u"state": state, u"job": job, u"build": build}
1263 def download_and_parse_data(self, repeat=1):
1264 """Download the input data files, parse input data from input files and
1265 store in pandas' Series.
1267 :param repeat: Repeat the download specified number of times if not
1272 logging.info(u"Downloading and parsing input files ...")
1274 for job, builds in self._cfg.builds.items():
1275 for build in builds:
1277 result = self._download_and_parse_build(job, build, repeat)
1278 build_nr = result[u"build"][u"build"]
1281 data = result[u"data"]
1282 build_data = pd.Series({
1283 u"metadata": pd.Series(
1284 list(data[u"metadata"].values()),
1285 index=list(data[u"metadata"].keys())
1287 u"suites": pd.Series(
1288 list(data[u"suites"].values()),
1289 index=list(data[u"suites"].keys())
1291 u"tests": pd.Series(
1292 list(data[u"tests"].values()),
1293 index=list(data[u"tests"].keys())
1297 if self._input_data.get(job, None) is None:
1298 self._input_data[job] = pd.Series()
1299 self._input_data[job][str(build_nr)] = build_data
1301 self._cfg.set_input_file_name(
1302 job, build_nr, result[u"build"][u"file-name"])
1304 self._cfg.set_input_state(job, build_nr, result[u"state"])
1307 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1308 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1310 logging.info(u"Done.")
1313 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1314 """Return the index of character in the string which is the end of tag.
1316 :param tag_filter: The string where the end of tag is being searched.
1317 :param start: The index where the searching is stated.
1318 :param closer: The character which is the tag closer.
1319 :type tag_filter: str
1322 :returns: The index of the tag closer.
1327 idx_opener = tag_filter.index(closer, start)
1328 return tag_filter.index(closer, idx_opener + 1)
1333 def _condition(tag_filter):
1334 """Create a conditional statement from the given tag filter.
1336 :param tag_filter: Filter based on tags from the element specification.
1337 :type tag_filter: str
1338 :returns: Conditional statement which can be evaluated.
1344 index = InputData._end_of_tag(tag_filter, index)
1348 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1350 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1351 continue_on_error=False):
1352 """Filter required data from the given jobs and builds.
1354 The output data structure is:
1358 - test (or suite) 1 ID:
1364 - test (or suite) n ID:
1371 :param element: Element which will use the filtered data.
1372 :param params: Parameters which will be included in the output. If None,
1373 all parameters are included.
1374 :param data: If not None, this data is used instead of data specified
1376 :param data_set: The set of data to be filtered: tests, suites,
1378 :param continue_on_error: Continue if there is error while reading the
1379 data. The Item will be empty then
1380 :type element: pandas.Series
1384 :type continue_on_error: bool
1385 :returns: Filtered data.
1386 :rtype pandas.Series
1390 if element[u"filter"] in (u"all", u"template"):
1393 cond = InputData._condition(element[u"filter"])
1394 logging.debug(f" Filter: {cond}")
1396 logging.error(u" No filter defined.")
1400 params = element.get(u"parameters", None)
1402 params.append(u"type")
1404 data_to_filter = data if data else element[u"data"]
1407 for job, builds in data_to_filter.items():
1408 data[job] = pd.Series()
1409 for build in builds:
1410 data[job][str(build)] = pd.Series()
1413 self.data[job][str(build)][data_set].items())
1415 if continue_on_error:
1419 for test_id, test_data in data_dict.items():
1420 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1421 data[job][str(build)][test_id] = pd.Series()
1423 for param, val in test_data.items():
1424 data[job][str(build)][test_id][param] = val
1426 for param in params:
1428 data[job][str(build)][test_id][param] =\
1431 data[job][str(build)][test_id][param] =\
1435 except (KeyError, IndexError, ValueError) as err:
1437 f"Missing mandatory parameter in the element specification: "
1441 except AttributeError as err:
1442 logging.error(repr(err))
1444 except SyntaxError as err:
1446 f"The filter {cond} is not correct. Check if all tags are "
1447 f"enclosed by apostrophes.\n{repr(err)}"
1451 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1452 continue_on_error=False):
1453 """Filter required data from the given jobs and builds.
1455 The output data structure is:
1459 - test (or suite) 1 ID:
1465 - test (or suite) n ID:
1472 :param element: Element which will use the filtered data.
1473 :param params: Parameters which will be included in the output. If None,
1474 all parameters are included.
1475 :param data_set: The set of data to be filtered: tests, suites,
1477 :param continue_on_error: Continue if there is error while reading the
1478 data. The Item will be empty then
1479 :type element: pandas.Series
1482 :type continue_on_error: bool
1483 :returns: Filtered data.
1484 :rtype pandas.Series
1487 include = element.get(u"include", None)
1489 logging.warning(u"No tests to include, skipping the element.")
1493 params = element.get(u"parameters", None)
1495 params.append(u"type")
1499 for job, builds in element[u"data"].items():
1500 data[job] = pd.Series()
1501 for build in builds:
1502 data[job][str(build)] = pd.Series()
1503 for test in include:
1505 reg_ex = re.compile(str(test).lower())
1506 for test_id in self.data[job][
1507 str(build)][data_set].keys():
1508 if re.match(reg_ex, str(test_id).lower()):
1509 test_data = self.data[job][
1510 str(build)][data_set][test_id]
1511 data[job][str(build)][test_id] = pd.Series()
1513 for param, val in test_data.items():
1514 data[job][str(build)][test_id]\
1517 for param in params:
1519 data[job][str(build)][
1523 data[job][str(build)][
1524 test_id][param] = u"No Data"
1525 except KeyError as err:
1526 logging.error(repr(err))
1527 if continue_on_error:
1532 except (KeyError, IndexError, ValueError) as err:
1534 f"Missing mandatory parameter in the element "
1535 f"specification: {repr(err)}"
1538 except AttributeError as err:
1539 logging.error(repr(err))
1543 def merge_data(data):
1544 """Merge data from more jobs and builds to a simple data structure.
1546 The output data structure is:
1548 - test (suite) 1 ID:
1554 - test (suite) n ID:
1557 :param data: Data to merge.
1558 :type data: pandas.Series
1559 :returns: Merged data.
1560 :rtype: pandas.Series
1563 logging.info(u" Merging data ...")
1565 merged_data = pd.Series()
1566 for builds in data.values:
1567 for item in builds.values:
1568 for item_id, item_data in item.items():
1569 merged_data[item_id] = item_data