1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
29 from datetime import datetime as dt
30 from datetime import timedelta
31 from json import loads
36 from robot.api import ExecutionResult, ResultVisitor
37 from robot import errors
39 from resources.libraries.python import jumpavg
40 from input_data_files import download_and_unzip_data_file
43 # Separator used in file names
47 class ExecutionChecker(ResultVisitor):
48 """Class to traverse through the test suite structure.
50 The functionality implemented in this class generates a json structure:
56 "generated": "Timestamp",
57 "version": "SUT version",
58 "job": "Jenkins job name",
59 "build": "Information about the build"
62 "Suite long name 1": {
64 "doc": "Suite 1 documentation",
65 "parent": "Suite 1 parent",
66 "level": "Level of the suite in the suite hierarchy"
68 "Suite long name N": {
70 "doc": "Suite N documentation",
71 "parent": "Suite 2 parent",
72 "level": "Level of the suite in the suite hierarchy"
79 "parent": "Name of the parent of the test",
80 "doc": "Test documentation",
81 "msg": "Test message",
82 "conf-history": "DUT1 and DUT2 VAT History",
83 "show-run": "Show Run",
84 "tags": ["tag 1", "tag 2", "tag n"],
86 "status": "PASS" | "FAIL",
132 "parent": "Name of the parent of the test",
133 "doc": "Test documentation",
134 "msg": "Test message",
135 "tags": ["tag 1", "tag 2", "tag n"],
137 "status": "PASS" | "FAIL",
144 "parent": "Name of the parent of the test",
145 "doc": "Test documentation",
146 "msg": "Test message",
147 "tags": ["tag 1", "tag 2", "tag n"],
148 "type": "MRR" | "BMRR",
149 "status": "PASS" | "FAIL",
151 "receive-rate": float,
152 # Average of a list, computed using AvgStdevStats.
153 # In CSIT-1180, replace with List[float].
167 "metadata": { # Optional
168 "version": "VPP version",
169 "job": "Jenkins job name",
170 "build": "Information about the build"
174 "doc": "Suite 1 documentation",
175 "parent": "Suite 1 parent",
176 "level": "Level of the suite in the suite hierarchy"
179 "doc": "Suite N documentation",
180 "parent": "Suite 2 parent",
181 "level": "Level of the suite in the suite hierarchy"
187 "parent": "Name of the parent of the test",
188 "doc": "Test documentation"
189 "msg": "Test message"
190 "tags": ["tag 1", "tag 2", "tag n"],
191 "conf-history": "DUT1 and DUT2 VAT History"
192 "show-run": "Show Run"
193 "status": "PASS" | "FAIL"
201 .. note:: ID is the lowercase full path to the test.
204 REGEX_PLR_RATE = re.compile(r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
205 r'PLRsearch upper bound::?\s(\d+.\d+)')
207 REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
208 r'NDR_UPPER:\s(\d+.\d+).*\n'
209 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
210 r'PDR_UPPER:\s(\d+.\d+)')
212 REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
213 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
215 REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
218 REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|"
219 r"VPP Version:\s*|VPP version:\s*)(.*)")
221 REGEX_VERSION_DPDK = re.compile(r"(DPDK version:\s*|DPDK Version:\s*)(.*)")
223 REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s(\d*).*$')
225 REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
226 r'tx\s(\d*),\srx\s(\d*)')
228 REGEX_BMRR = re.compile(r'Maximum Receive Rate trial results'
229 r' in packets per second: \[(.*)\]')
231 REGEX_RECONF_LOSS = re.compile(r'Packets lost due to reconfig: (\d*)')
232 REGEX_RECONF_TIME = re.compile(r'Implied time lost: (\d*.[\de-]*)')
234 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
236 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
238 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
240 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
242 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
244 def __init__(self, metadata, mapping, ignore):
247 :param metadata: Key-value pairs to be included in "metadata" part of
249 :param mapping: Mapping of the old names of test cases to the new
251 :param ignore: List of TCs to be ignored.
257 # Type of message to parse out from the test messages
258 self._msg_type = None
264 self._timestamp = None
266 # Testbed. The testbed is identified by TG node IP address.
269 # Mapping of TCs long names
270 self._mapping = mapping
273 self._ignore = ignore
275 # Number of PAPI History messages found:
277 # 1 - PAPI History of DUT1
278 # 2 - PAPI History of DUT2
279 self._lookup_kw_nr = 0
280 self._conf_history_lookup_nr = 0
282 # Number of Show Running messages found
284 # 1 - Show run message found
285 self._show_run_lookup_nr = 0
287 # Test ID of currently processed test- the lowercase full path to the
291 # The main data structure
293 u"metadata": OrderedDict(),
294 u"suites": OrderedDict(),
295 u"tests": OrderedDict()
298 # Save the provided metadata
299 for key, val in metadata.items():
300 self._data[u"metadata"][key] = val
302 # Dictionary defining the methods used to parse different types of
305 u"timestamp": self._get_timestamp,
306 u"vpp-version": self._get_vpp_version,
307 u"dpdk-version": self._get_dpdk_version,
308 # TODO: Remove when not needed:
309 u"teardown-vat-history": self._get_vat_history,
310 u"teardown-papi-history": self._get_papi_history,
311 u"test-show-runtime": self._get_show_run,
312 u"testbed": self._get_testbed
317 """Getter - Data parsed from the XML file.
319 :returns: Data parsed from the XML file.
324 def _get_testbed(self, msg):
325 """Called when extraction of testbed IP is required.
326 The testbed is identified by TG node IP address.
328 :param msg: Message to process.
333 if msg.message.count(u"Setup of TG node") or \
334 msg.message.count(u"Setup of node TG host"):
335 reg_tg_ip = re.compile(
336 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
338 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
339 except (KeyError, ValueError, IndexError, AttributeError):
342 self._data[u"metadata"][u"testbed"] = self._testbed
343 self._msg_type = None
345 def _get_vpp_version(self, msg):
346 """Called when extraction of VPP version is required.
348 :param msg: Message to process.
353 if msg.message.count(u"return STDOUT Version:") or \
354 msg.message.count(u"VPP Version:") or \
355 msg.message.count(u"VPP version:"):
356 self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
358 self._data[u"metadata"][u"version"] = self._version
359 self._msg_type = None
361 def _get_dpdk_version(self, msg):
362 """Called when extraction of DPDK version is required.
364 :param msg: Message to process.
369 if msg.message.count(u"DPDK Version:"):
371 self._version = str(re.search(
372 self.REGEX_VERSION_DPDK, msg.message).group(2))
373 self._data[u"metadata"][u"version"] = self._version
377 self._msg_type = None
379 def _get_timestamp(self, msg):
380 """Called when extraction of timestamp is required.
382 :param msg: Message to process.
387 self._timestamp = msg.timestamp[:14]
388 self._data[u"metadata"][u"generated"] = self._timestamp
389 self._msg_type = None
391 def _get_vat_history(self, msg):
392 """Called when extraction of VAT command history is required.
394 TODO: Remove when not needed.
396 :param msg: Message to process.
400 if msg.message.count(u"VAT command history:"):
401 self._conf_history_lookup_nr += 1
402 if self._conf_history_lookup_nr == 1:
403 self._data[u"tests"][self._test_id][u"conf-history"] = str()
405 self._msg_type = None
406 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
407 r"VAT command history:", u"",
408 msg.message, count=1).replace(u'\n', u' |br| ').\
411 self._data[u"tests"][self._test_id][u"conf-history"] += (
412 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
415 def _get_papi_history(self, msg):
416 """Called when extraction of PAPI command history is required.
418 :param msg: Message to process.
422 if msg.message.count(u"PAPI command history:"):
423 self._conf_history_lookup_nr += 1
424 if self._conf_history_lookup_nr == 1:
425 self._data[u"tests"][self._test_id][u"conf-history"] = str()
427 self._msg_type = None
428 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
429 r"PAPI command history:", u"",
430 msg.message, count=1).replace(u'\n', u' |br| ').\
433 self._data[u"tests"][self._test_id][u"conf-history"] += (
434 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
437 def _get_show_run(self, msg):
438 """Called when extraction of VPP operational data (output of CLI command
439 Show Runtime) is required.
441 :param msg: Message to process.
445 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
446 self._data[u"tests"][self._test_id][u"show-run"] = str()
448 if msg.message.count(u"stats runtime") or \
449 msg.message.count(u"Runtime"):
450 host = str(re.search(self.REGEX_TC_PAPI_CLI, msg.message).
452 socket = str(re.search(self.REGEX_TC_PAPI_CLI, msg.message).
459 replace(u'b"', u'"').
460 replace(u'u"', u'"').
464 threads_nr = len(runtime[0][u"clocks"])
465 except (IndexError, KeyError):
475 table = [[tbl_hdr, ] for _ in range(threads_nr)]
477 for idx in range(threads_nr):
478 name = format(item[u"name"])
479 calls = format(item[u"calls"][idx])
480 vectors = format(item[u"vectors"][idx])
481 suspends = format(item[u"suspends"][idx])
482 if item[u"vectors"][idx] > 0:
484 item[u"clocks"][idx]/item[u"vectors"][idx], u".2e")
485 elif item[u"calls"][idx] > 0:
487 item[u"clocks"][idx]/item[u"calls"][idx], u".2e")
488 elif item[u"suspends"][idx] > 0:
490 item[u"clocks"][idx]/item[u"suspends"][idx], u".2e")
493 if item[u"calls"][idx] > 0:
494 vectors_call = format(
495 item[u"vectors"][idx]/item[u"calls"][idx], u".2f")
497 vectors_call = format(0, u".2f")
498 if int(calls) + int(vectors) + int(suspends):
500 name, calls, vectors, suspends, clocks, vectors_call
503 for idx in range(threads_nr):
504 text += f"Thread {idx} "
505 text += u"vpp_main\n" if idx == 0 else f"vpp_wk_{idx-1}\n"
507 for row in table[idx]:
508 if txt_table is None:
509 txt_table = prettytable.PrettyTable(row)
512 txt_table.add_row(row)
513 txt_table.set_style(prettytable.MSWORD_FRIENDLY)
514 txt_table.align[u"Name"] = u"l"
515 txt_table.align[u"Calls"] = u"r"
516 txt_table.align[u"Vectors"] = u"r"
517 txt_table.align[u"Suspends"] = u"r"
518 txt_table.align[u"Clocks"] = u"r"
519 txt_table.align[u"Vectors/Calls"] = u"r"
521 text += txt_table.get_string(sortby=u"Name") + u'\n'
522 text = f" \n **DUT: {host}/{socket}** \n {text}".\
523 replace(u'\n', u' |br| ').\
524 replace(u'\r', u'').\
526 self._data[u"tests"][self._test_id][u"show-run"] += text
528 def _get_ndrpdr_throughput(self, msg):
529 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
532 :param msg: The test message to be parsed.
534 :returns: Parsed data as a dict and the status (PASS/FAIL).
535 :rtype: tuple(dict, str)
539 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
540 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
543 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
545 if groups is not None:
547 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
548 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
549 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
550 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
552 except (IndexError, ValueError):
555 return throughput, status
557 def _get_plr_throughput(self, msg):
558 """Get PLRsearch lower bound and PLRsearch upper bound from the test
561 :param msg: The test message to be parsed.
563 :returns: Parsed data as a dict and the status (PASS/FAIL).
564 :rtype: tuple(dict, str)
572 groups = re.search(self.REGEX_PLR_RATE, msg)
574 if groups is not None:
576 throughput[u"LOWER"] = float(groups.group(1))
577 throughput[u"UPPER"] = float(groups.group(2))
579 except (IndexError, ValueError):
582 return throughput, status
584 def _get_ndrpdr_latency(self, msg):
585 """Get LATENCY from the test message.
587 :param msg: The test message to be parsed.
589 :returns: Parsed data as a dict and the status (PASS/FAIL).
590 :rtype: tuple(dict, str)
600 u"direction1": copy.copy(latency_default),
601 u"direction2": copy.copy(latency_default)
604 u"direction1": copy.copy(latency_default),
605 u"direction2": copy.copy(latency_default)
609 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
611 def process_latency(in_str):
612 """Return object with parsed latency values.
614 TODO: Define class for the return type.
616 :param in_str: Input string, min/avg/max/hdrh format.
618 :returns: Dict with corresponding keys, except hdrh float values.
620 :throws IndexError: If in_str does not have enough substrings.
621 :throws ValueError: If a substring does not convert to float.
623 in_list = in_str.split('/', 3)
626 u"min": float(in_list[0]),
627 u"avg": float(in_list[1]),
628 u"max": float(in_list[2]),
632 if len(in_list) == 4:
633 rval[u"hdrh"] = str(in_list[3])
637 if groups is not None:
639 latency[u"NDR"][u"direction1"] = \
640 process_latency(groups.group(1))
641 latency[u"NDR"][u"direction2"] = \
642 process_latency(groups.group(2))
643 latency[u"PDR"][u"direction1"] = \
644 process_latency(groups.group(3))
645 latency[u"PDR"][u"direction2"] = \
646 process_latency(groups.group(4))
648 except (IndexError, ValueError):
651 return latency, status
653 def visit_suite(self, suite):
654 """Implements traversing through the suite and its direct children.
656 :param suite: Suite to process.
660 if self.start_suite(suite) is not False:
661 suite.suites.visit(self)
662 suite.tests.visit(self)
663 self.end_suite(suite)
665 def start_suite(self, suite):
666 """Called when suite starts.
668 :param suite: Suite to process.
674 parent_name = suite.parent.name
675 except AttributeError:
678 doc_str = suite.doc.\
679 replace(u'"', u"'").\
680 replace(u'\n', u' ').\
681 replace(u'\r', u'').\
682 replace(u'*[', u' |br| *[').\
683 replace(u"*", u"**").\
684 replace(u' |br| *[', u'*[', 1)
686 self._data[u"suites"][suite.longname.lower().
688 replace(u" ", u"_")] = {
689 u"name": suite.name.lower(),
691 u"parent": parent_name,
692 u"level": len(suite.longname.split(u"."))
695 suite.keywords.visit(self)
697 def end_suite(self, suite):
698 """Called when suite ends.
700 :param suite: Suite to process.
705 def visit_test(self, test):
706 """Implements traversing through the test.
708 :param test: Test to process.
712 if self.start_test(test) is not False:
713 test.keywords.visit(self)
716 def start_test(self, test):
717 """Called when test starts.
719 :param test: Test to process.
724 longname_orig = test.longname.lower()
726 # Check the ignore list
727 if longname_orig in self._ignore:
730 tags = [str(tag) for tag in test.tags]
733 # Change the TC long name and name if defined in the mapping table
734 longname = self._mapping.get(longname_orig, None)
735 if longname is not None:
736 name = longname.split(u'.')[-1]
738 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
742 longname = longname_orig
743 name = test.name.lower()
745 # Remove TC number from the TC long name (backward compatibility):
746 self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
747 # Remove TC number from the TC name (not needed):
748 test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
750 test_result[u"parent"] = test.parent.name.lower()
751 test_result[u"tags"] = tags
752 test_result["doc"] = test.doc.\
753 replace(u'"', u"'").\
754 replace(u'\n', u' ').\
755 replace(u'\r', u'').\
756 replace(u'[', u' |br| [').\
757 replace(u' |br| [', u'[', 1)
758 test_result[u"msg"] = test.message.\
759 replace(u'\n', u' |br| ').\
760 replace(u'\r', u'').\
762 test_result[u"type"] = u"FUNC"
763 test_result[u"status"] = test.status
765 if u"PERFTEST" in tags:
766 # Replace info about cores (e.g. -1c-) with the info about threads
767 # and cores (e.g. -1t1c-) in the long test case names and in the
768 # test case names if necessary.
769 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
773 for tag in test_result[u"tags"]:
774 groups = re.search(self.REGEX_TC_TAG, tag)
780 self._test_id = re.sub(self.REGEX_TC_NAME_NEW,
781 f"-{tag_tc.lower()}-",
784 test_result[u"name"] = re.sub(self.REGEX_TC_NAME_NEW,
785 f"-{tag_tc.lower()}-",
789 test_result[u"status"] = u"FAIL"
790 self._data[u"tests"][self._test_id] = test_result
792 f"The test {self._test_id} has no or more than one "
793 f"multi-threading tags.\n"
794 f"Tags: {test_result[u'tags']}"
798 if test.status == u"PASS":
799 if u"NDRPDR" in tags:
800 test_result[u"type"] = u"NDRPDR"
801 test_result[u"throughput"], test_result[u"status"] = \
802 self._get_ndrpdr_throughput(test.message)
803 test_result[u"latency"], test_result[u"status"] = \
804 self._get_ndrpdr_latency(test.message)
805 elif u"SOAK" in tags:
806 test_result[u"type"] = u"SOAK"
807 test_result[u"throughput"], test_result[u"status"] = \
808 self._get_plr_throughput(test.message)
810 test_result[u"type"] = u"TCP"
811 groups = re.search(self.REGEX_TCP, test.message)
812 test_result[u"result"] = int(groups.group(2))
813 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
815 test_result[u"type"] = u"MRR"
817 test_result[u"type"] = u"BMRR"
819 test_result[u"result"] = dict()
820 groups = re.search(self.REGEX_BMRR, test.message)
821 if groups is not None:
822 items_str = groups.group(1)
823 items_float = [float(item.strip()) for item
824 in items_str.split(",")]
825 # Use whole list in CSIT-1180.
826 stats = jumpavg.AvgStdevStats.for_runs(items_float)
827 test_result[u"result"][u"receive-rate"] = stats.avg
829 groups = re.search(self.REGEX_MRR, test.message)
830 test_result[u"result"][u"receive-rate"] = \
831 float(groups.group(3)) / float(groups.group(1))
832 elif u"RECONF" in tags:
833 test_result[u"type"] = u"RECONF"
834 test_result[u"result"] = None
836 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
837 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
838 test_result[u"result"] = {
839 u"loss": int(grps_loss.group(1)),
840 u"time": float(grps_time.group(1))
842 except (AttributeError, IndexError, ValueError, TypeError):
843 test_result[u"status"] = u"FAIL"
845 test_result[u"status"] = u"FAIL"
846 self._data[u"tests"][self._test_id] = test_result
849 self._data[u"tests"][self._test_id] = test_result
851 def end_test(self, test):
852 """Called when test ends.
854 :param test: Test to process.
859 def visit_keyword(self, keyword):
860 """Implements traversing through the keyword and its child keywords.
862 :param keyword: Keyword to process.
863 :type keyword: Keyword
866 if self.start_keyword(keyword) is not False:
867 self.end_keyword(keyword)
869 def start_keyword(self, keyword):
870 """Called when keyword starts. Default implementation does nothing.
872 :param keyword: Keyword to process.
873 :type keyword: Keyword
877 if keyword.type == u"setup":
878 self.visit_setup_kw(keyword)
879 elif keyword.type == u"teardown":
880 self._lookup_kw_nr = 0
881 self.visit_teardown_kw(keyword)
883 self._lookup_kw_nr = 0
884 self.visit_test_kw(keyword)
885 except AttributeError:
888 def end_keyword(self, keyword):
889 """Called when keyword ends. Default implementation does nothing.
891 :param keyword: Keyword to process.
892 :type keyword: Keyword
896 def visit_test_kw(self, test_kw):
897 """Implements traversing through the test keyword and its child
900 :param test_kw: Keyword to process.
901 :type test_kw: Keyword
904 for keyword in test_kw.keywords:
905 if self.start_test_kw(keyword) is not False:
906 self.visit_test_kw(keyword)
907 self.end_test_kw(keyword)
909 def start_test_kw(self, test_kw):
910 """Called when test keyword starts. Default implementation does
913 :param test_kw: Keyword to process.
914 :type test_kw: Keyword
917 if test_kw.name.count(u"Show Runtime Counters On All Duts"):
918 self._lookup_kw_nr += 1
919 self._show_run_lookup_nr = 0
920 self._msg_type = u"test-show-runtime"
921 elif test_kw.name.count(u"Install Dpdk Test") and not self._version:
922 self._msg_type = u"dpdk-version"
925 test_kw.messages.visit(self)
927 def end_test_kw(self, test_kw):
928 """Called when keyword ends. Default implementation does nothing.
930 :param test_kw: Keyword to process.
931 :type test_kw: Keyword
935 def visit_setup_kw(self, setup_kw):
936 """Implements traversing through the teardown keyword and its child
939 :param setup_kw: Keyword to process.
940 :type setup_kw: Keyword
943 for keyword in setup_kw.keywords:
944 if self.start_setup_kw(keyword) is not False:
945 self.visit_setup_kw(keyword)
946 self.end_setup_kw(keyword)
948 def start_setup_kw(self, setup_kw):
949 """Called when teardown keyword starts. Default implementation does
952 :param setup_kw: Keyword to process.
953 :type setup_kw: Keyword
956 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
957 and not self._version:
958 self._msg_type = u"vpp-version"
959 elif setup_kw.name.count(u"Set Global Variable") \
960 and not self._timestamp:
961 self._msg_type = u"timestamp"
962 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
963 self._msg_type = u"testbed"
966 setup_kw.messages.visit(self)
968 def end_setup_kw(self, setup_kw):
969 """Called when keyword ends. Default implementation does nothing.
971 :param setup_kw: Keyword to process.
972 :type setup_kw: Keyword
976 def visit_teardown_kw(self, teardown_kw):
977 """Implements traversing through the teardown keyword and its child
980 :param teardown_kw: Keyword to process.
981 :type teardown_kw: Keyword
984 for keyword in teardown_kw.keywords:
985 if self.start_teardown_kw(keyword) is not False:
986 self.visit_teardown_kw(keyword)
987 self.end_teardown_kw(keyword)
989 def start_teardown_kw(self, teardown_kw):
990 """Called when teardown keyword starts
992 :param teardown_kw: Keyword to process.
993 :type teardown_kw: Keyword
997 if teardown_kw.name.count(u"Show Vat History On All Duts"):
998 # TODO: Remove when not needed:
999 self._conf_history_lookup_nr = 0
1000 self._msg_type = u"teardown-vat-history"
1001 teardown_kw.messages.visit(self)
1002 elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1003 self._conf_history_lookup_nr = 0
1004 self._msg_type = u"teardown-papi-history"
1005 teardown_kw.messages.visit(self)
1007 def end_teardown_kw(self, teardown_kw):
1008 """Called when keyword ends. Default implementation does nothing.
1010 :param teardown_kw: Keyword to process.
1011 :type teardown_kw: Keyword
1015 def visit_message(self, msg):
1016 """Implements visiting the message.
1018 :param msg: Message to process.
1022 if self.start_message(msg) is not False:
1023 self.end_message(msg)
1025 def start_message(self, msg):
1026 """Called when message starts. Get required information from messages:
1029 :param msg: Message to process.
1035 self.parse_msg[self._msg_type](msg)
1037 def end_message(self, msg):
1038 """Called when message ends. Default implementation does nothing.
1040 :param msg: Message to process.
1049 The data is extracted from output.xml files generated by Jenkins jobs and
1050 stored in pandas' DataFrames.
1056 (as described in ExecutionChecker documentation)
1058 (as described in ExecutionChecker documentation)
1060 (as described in ExecutionChecker documentation)
1063 def __init__(self, spec):
1066 :param spec: Specification.
1067 :type spec: Specification
1074 self._input_data = pd.Series()
1078 """Getter - Input data.
1080 :returns: Input data
1081 :rtype: pandas.Series
1083 return self._input_data
1085 def metadata(self, job, build):
1086 """Getter - metadata
1088 :param job: Job which metadata we want.
1089 :param build: Build which metadata we want.
1093 :rtype: pandas.Series
1096 return self.data[job][build][u"metadata"]
1098 def suites(self, job, build):
1101 :param job: Job which suites we want.
1102 :param build: Build which suites we want.
1106 :rtype: pandas.Series
1109 return self.data[job][str(build)][u"suites"]
1111 def tests(self, job, build):
1114 :param job: Job which tests we want.
1115 :param build: Build which tests we want.
1119 :rtype: pandas.Series
1122 return self.data[job][build][u"tests"]
1124 def _parse_tests(self, job, build, log):
1125 """Process data from robot output.xml file and return JSON structured
1128 :param job: The name of job which build output data will be processed.
1129 :param build: The build which output data will be processed.
1130 :param log: List of log messages.
1133 :type log: list of tuples (severity, msg)
1134 :returns: JSON data structure.
1143 with open(build[u"file-name"], u'r') as data_file:
1145 result = ExecutionResult(data_file)
1146 except errors.DataError as err:
1148 (u"ERROR", f"Error occurred while parsing output.xml: "
1152 checker = ExecutionChecker(metadata, self._cfg.mapping,
1154 result.visit(checker)
1158 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1159 """Download and parse the input data file.
1161 :param pid: PID of the process executing this method.
1162 :param job: Name of the Jenkins job which generated the processed input
1164 :param build: Information about the Jenkins build which generated the
1165 processed input file.
1166 :param repeat: Repeat the download specified number of times if not
1177 (u"INFO", f" Processing the job/build: {job}: {build[u'build']}")
1185 success = download_and_unzip_data_file(self._cfg, job, build, pid,
1193 f"It is not possible to download the input data file from the "
1194 f"job {job}, build {build[u'build']}, or it is damaged. "
1200 f" Processing data from the build {build[u'build']} ...")
1202 data = self._parse_tests(job, build, logs)
1206 f"Input data file from the job {job}, build "
1207 f"{build[u'build']} is damaged. Skipped.")
1210 state = u"processed"
1213 remove(build[u"file-name"])
1214 except OSError as err:
1216 ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
1220 # If the time-period is defined in the specification file, remove all
1221 # files which are outside the time period.
1222 timeperiod = self._cfg.input.get(u"time-period", None)
1223 if timeperiod and data:
1225 timeperiod = timedelta(int(timeperiod))
1226 metadata = data.get(u"metadata", None)
1228 generated = metadata.get(u"generated", None)
1230 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1231 if (now - generated) > timeperiod:
1232 # Remove the data and the file:
1237 f" The build {job}/{build[u'build']} is "
1238 f"outdated, will be removed.")
1240 logs.append((u"INFO", u" Done."))
1242 for level, line in logs:
1243 if level == u"INFO":
1245 elif level == u"ERROR":
1247 elif level == u"DEBUG":
1249 elif level == u"CRITICAL":
1250 logging.critical(line)
1251 elif level == u"WARNING":
1252 logging.warning(line)
1254 return {u"data": data, u"state": state, u"job": job, u"build": build}
1256 def download_and_parse_data(self, repeat=1):
1257 """Download the input data files, parse input data from input files and
1258 store in pandas' Series.
1260 :param repeat: Repeat the download specified number of times if not
1265 logging.info(u"Downloading and parsing input files ...")
1267 for job, builds in self._cfg.builds.items():
1268 for build in builds:
1270 result = self._download_and_parse_build(job, build, repeat)
1271 build_nr = result[u"build"][u"build"]
1274 data = result[u"data"]
1275 build_data = pd.Series({
1276 u"metadata": pd.Series(
1277 list(data[u"metadata"].values()),
1278 index=list(data[u"metadata"].keys())
1280 u"suites": pd.Series(
1281 list(data[u"suites"].values()),
1282 index=list(data[u"suites"].keys())
1284 u"tests": pd.Series(
1285 list(data[u"tests"].values()),
1286 index=list(data[u"tests"].keys())
1290 if self._input_data.get(job, None) is None:
1291 self._input_data[job] = pd.Series()
1292 self._input_data[job][str(build_nr)] = build_data
1294 self._cfg.set_input_file_name(
1295 job, build_nr, result[u"build"][u"file-name"])
1297 self._cfg.set_input_state(job, build_nr, result[u"state"])
1300 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1301 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1303 logging.info(u"Done.")
1306 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1307 """Return the index of character in the string which is the end of tag.
1309 :param tag_filter: The string where the end of tag is being searched.
1310 :param start: The index where the searching is stated.
1311 :param closer: The character which is the tag closer.
1312 :type tag_filter: str
1315 :returns: The index of the tag closer.
1320 idx_opener = tag_filter.index(closer, start)
1321 return tag_filter.index(closer, idx_opener + 1)
1326 def _condition(tag_filter):
1327 """Create a conditional statement from the given tag filter.
1329 :param tag_filter: Filter based on tags from the element specification.
1330 :type tag_filter: str
1331 :returns: Conditional statement which can be evaluated.
1337 index = InputData._end_of_tag(tag_filter, index)
1341 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1343 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1344 continue_on_error=False):
1345 """Filter required data from the given jobs and builds.
1347 The output data structure is:
1351 - test (or suite) 1 ID:
1357 - test (or suite) n ID:
1364 :param element: Element which will use the filtered data.
1365 :param params: Parameters which will be included in the output. If None,
1366 all parameters are included.
1367 :param data: If not None, this data is used instead of data specified
1369 :param data_set: The set of data to be filtered: tests, suites,
1371 :param continue_on_error: Continue if there is error while reading the
1372 data. The Item will be empty then
1373 :type element: pandas.Series
1377 :type continue_on_error: bool
1378 :returns: Filtered data.
1379 :rtype pandas.Series
1383 if element[u"filter"] in (u"all", u"template"):
1386 cond = InputData._condition(element[u"filter"])
1387 logging.debug(f" Filter: {cond}")
1389 logging.error(u" No filter defined.")
1393 params = element.get(u"parameters", None)
1395 params.append(u"type")
1397 data_to_filter = data if data else element[u"data"]
1400 for job, builds in data_to_filter.items():
1401 data[job] = pd.Series()
1402 for build in builds:
1403 data[job][str(build)] = pd.Series()
1406 self.data[job][str(build)][data_set].items())
1408 if continue_on_error:
1412 for test_id, test_data in data_dict.items():
1413 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1414 data[job][str(build)][test_id] = pd.Series()
1416 for param, val in test_data.items():
1417 data[job][str(build)][test_id][param] = val
1419 for param in params:
1421 data[job][str(build)][test_id][param] =\
1424 data[job][str(build)][test_id][param] =\
1428 except (KeyError, IndexError, ValueError) as err:
1430 f"Missing mandatory parameter in the element specification: "
1434 except AttributeError as err:
1435 logging.error(repr(err))
1437 except SyntaxError as err:
1439 f"The filter {cond} is not correct. Check if all tags are "
1440 f"enclosed by apostrophes.\n{repr(err)}"
1444 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1445 continue_on_error=False):
1446 """Filter required data from the given jobs and builds.
1448 The output data structure is:
1452 - test (or suite) 1 ID:
1458 - test (or suite) n ID:
1465 :param element: Element which will use the filtered data.
1466 :param params: Parameters which will be included in the output. If None,
1467 all parameters are included.
1468 :param data_set: The set of data to be filtered: tests, suites,
1470 :param continue_on_error: Continue if there is error while reading the
1471 data. The Item will be empty then
1472 :type element: pandas.Series
1475 :type continue_on_error: bool
1476 :returns: Filtered data.
1477 :rtype pandas.Series
1480 include = element.get(u"include", None)
1482 logging.warning(u"No tests to include, skipping the element.")
1486 params = element.get(u"parameters", None)
1488 params.append(u"type")
1492 for job, builds in element[u"data"].items():
1493 data[job] = pd.Series()
1494 for build in builds:
1495 data[job][str(build)] = pd.Series()
1496 for test in include:
1498 reg_ex = re.compile(str(test).lower())
1499 for test_id in self.data[job][
1500 str(build)][data_set].keys():
1501 if re.match(reg_ex, str(test_id).lower()):
1502 test_data = self.data[job][
1503 str(build)][data_set][test_id]
1504 data[job][str(build)][test_id] = pd.Series()
1506 for param, val in test_data.items():
1507 data[job][str(build)][test_id]\
1510 for param in params:
1512 data[job][str(build)][
1516 data[job][str(build)][
1517 test_id][param] = u"No Data"
1518 except KeyError as err:
1519 logging.error(repr(err))
1520 if continue_on_error:
1525 except (KeyError, IndexError, ValueError) as err:
1527 f"Missing mandatory parameter in the element "
1528 f"specification: {repr(err)}"
1531 except AttributeError as err:
1532 logging.error(repr(err))
1536 def merge_data(data):
1537 """Merge data from more jobs and builds to a simple data structure.
1539 The output data structure is:
1541 - test (suite) 1 ID:
1547 - test (suite) n ID:
1550 :param data: Data to merge.
1551 :type data: pandas.Series
1552 :returns: Merged data.
1553 :rtype: pandas.Series
1556 logging.info(u" Merging data ...")
1558 merged_data = pd.Series()
1559 for builds in data.values:
1560 for item in builds.values:
1561 for item_id, item_data in item.items():
1562 merged_data[item_id] = item_data