1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
28 from os import remove, walk, listdir
29 from os.path import isfile, isdir, join
30 from datetime import datetime as dt
31 from datetime import timedelta
32 from json import loads
33 from json.decoder import JSONDecodeError
40 from robot.api import ExecutionResult, ResultVisitor
41 from robot import errors
43 from resources.libraries.python import jumpavg
44 from input_data_files import download_and_unzip_data_file
45 from pal_errors import PresentationError
48 # Separator used in file names
52 class ExecutionChecker(ResultVisitor):
53 """Class to traverse through the test suite structure.
55 The functionality implemented in this class generates a json structure:
61 "generated": "Timestamp",
62 "version": "SUT version",
63 "job": "Jenkins job name",
64 "build": "Information about the build"
67 "Suite long name 1": {
69 "doc": "Suite 1 documentation",
70 "parent": "Suite 1 parent",
71 "level": "Level of the suite in the suite hierarchy"
73 "Suite long name N": {
75 "doc": "Suite N documentation",
76 "parent": "Suite 2 parent",
77 "level": "Level of the suite in the suite hierarchy"
84 "parent": "Name of the parent of the test",
85 "doc": "Test documentation",
86 "msg": "Test message",
87 "conf-history": "DUT1 and DUT2 VAT History",
88 "show-run": "Show Run",
89 "tags": ["tag 1", "tag 2", "tag n"],
91 "status": "PASS" | "FAIL",
137 "parent": "Name of the parent of the test",
138 "doc": "Test documentation",
139 "msg": "Test message",
140 "tags": ["tag 1", "tag 2", "tag n"],
142 "status": "PASS" | "FAIL",
149 "parent": "Name of the parent of the test",
150 "doc": "Test documentation",
151 "msg": "Test message",
152 "tags": ["tag 1", "tag 2", "tag n"],
153 "type": "MRR" | "BMRR",
154 "status": "PASS" | "FAIL",
156 "receive-rate": float,
157 # Average of a list, computed using AvgStdevStats.
158 # In CSIT-1180, replace with List[float].
172 "metadata": { # Optional
173 "version": "VPP version",
174 "job": "Jenkins job name",
175 "build": "Information about the build"
179 "doc": "Suite 1 documentation",
180 "parent": "Suite 1 parent",
181 "level": "Level of the suite in the suite hierarchy"
184 "doc": "Suite N documentation",
185 "parent": "Suite 2 parent",
186 "level": "Level of the suite in the suite hierarchy"
192 "parent": "Name of the parent of the test",
193 "doc": "Test documentation"
194 "msg": "Test message"
195 "tags": ["tag 1", "tag 2", "tag n"],
196 "conf-history": "DUT1 and DUT2 VAT History"
197 "show-run": "Show Run"
198 "status": "PASS" | "FAIL"
206 .. note:: ID is the lowercase full path to the test.
209 REGEX_PLR_RATE = re.compile(
210 r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
211 r'PLRsearch upper bound::?\s(\d+.\d+)'
213 REGEX_NDRPDR_RATE = re.compile(
214 r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
215 r'NDR_UPPER:\s(\d+.\d+).*\n'
216 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
217 r'PDR_UPPER:\s(\d+.\d+)'
219 REGEX_NDRPDR_GBPS = re.compile(
220 r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
221 r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
222 r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
223 r'PDR_UPPER:.*,\s(\d+.\d+)'
225 REGEX_PERF_MSG_INFO = re.compile(
226 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
227 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
228 r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
229 r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
230 r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
232 REGEX_CPS_MSG_INFO = re.compile(
233 r'NDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*\n'
234 r'PDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*'
236 REGEX_PPS_MSG_INFO = re.compile(
237 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
238 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*'
240 REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
242 REGEX_VSAP_MSG_INFO = re.compile(
243 r'Transfer Rate: (\d*.\d*).*\n'
244 r'Latency: (\d*.\d*).*\n'
245 r'Completed requests: (\d*).*\n'
246 r'Failed requests: (\d*).*\n'
247 r'Total data transferred: (\d*).*\n'
248 r'Connection [cr]ps rate:\s*(\d*.\d*)'
251 # Needed for CPS and PPS tests
252 REGEX_NDRPDR_LAT_BASE = re.compile(
253 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
254 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
256 REGEX_NDRPDR_LAT = re.compile(
257 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
258 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
259 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
260 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
261 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
262 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
265 REGEX_VERSION_VPP = re.compile(
266 r"(return STDOUT Version:\s*|"
267 r"VPP Version:\s*|VPP version:\s*)(.*)"
269 REGEX_VERSION_DPDK = re.compile(
270 r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
272 REGEX_TCP = re.compile(
273 r'Total\s(rps|cps|throughput):\s(\d*).*$'
275 REGEX_MRR = re.compile(
276 r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
277 r'tx\s(\d*),\srx\s(\d*)'
279 REGEX_BMRR = re.compile(
280 r'.*trial results.*: \[(.*)\]'
282 REGEX_RECONF_LOSS = re.compile(
283 r'Packets lost due to reconfig: (\d*)'
285 REGEX_RECONF_TIME = re.compile(
286 r'Implied time lost: (\d*.[\de-]*)'
288 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
290 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
292 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
294 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
296 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
298 def __init__(self, metadata, mapping, ignore, for_output):
301 :param metadata: Key-value pairs to be included in "metadata" part of
303 :param mapping: Mapping of the old names of test cases to the new
305 :param ignore: List of TCs to be ignored.
306 :param for_output: Output to be generated from downloaded data.
310 :type for_output: str
313 # Type of message to parse out from the test messages
314 self._msg_type = None
320 self._timestamp = None
322 # Testbed. The testbed is identified by TG node IP address.
325 # Mapping of TCs long names
326 self._mapping = mapping
329 self._ignore = ignore
331 self._for_output = for_output
333 # Number of PAPI History messages found:
335 # 1 - PAPI History of DUT1
336 # 2 - PAPI History of DUT2
337 self._conf_history_lookup_nr = 0
339 self._sh_run_counter = 0
340 self._telemetry_kw_counter = 0
341 self._telemetry_msg_counter = 0
343 # Test ID of currently processed test- the lowercase full path to the
347 # The main data structure
349 u"metadata": OrderedDict(),
350 u"suites": OrderedDict(),
351 u"tests": OrderedDict()
354 # Save the provided metadata
355 for key, val in metadata.items():
356 self._data[u"metadata"][key] = val
358 # Dictionary defining the methods used to parse different types of
361 u"timestamp": self._get_timestamp,
362 u"vpp-version": self._get_vpp_version,
363 u"dpdk-version": self._get_dpdk_version,
364 u"teardown-papi-history": self._get_papi_history,
365 u"test-show-runtime": self._get_show_run,
366 u"testbed": self._get_testbed,
367 u"test-telemetry": self._get_telemetry
372 """Getter - Data parsed from the XML file.
374 :returns: Data parsed from the XML file.
379 def _get_data_from_mrr_test_msg(self, msg):
380 """Get info from message of MRR performance tests.
382 :param msg: Message to be processed.
384 :returns: Processed message or original message if a problem occurs.
388 groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
389 if not groups or groups.lastindex != 1:
390 return u"Test Failed."
393 data = groups.group(1).split(u", ")
394 except (AttributeError, IndexError, ValueError, KeyError):
395 return u"Test Failed."
400 out_str += f"{(float(item) / 1e6):.2f}, "
401 return out_str[:-2] + u"]"
402 except (AttributeError, IndexError, ValueError, KeyError):
403 return u"Test Failed."
405 def _get_data_from_cps_test_msg(self, msg):
406 """Get info from message of NDRPDR CPS tests.
408 :param msg: Message to be processed.
410 :returns: Processed message or "Test Failed." if a problem occurs.
414 groups = re.search(self.REGEX_CPS_MSG_INFO, msg)
415 if not groups or groups.lastindex != 2:
416 return u"Test Failed."
420 f"1. {(float(groups.group(1)) / 1e6):5.2f}\n"
421 f"2. {(float(groups.group(2)) / 1e6):5.2f}"
423 except (AttributeError, IndexError, ValueError, KeyError):
424 return u"Test Failed."
426 def _get_data_from_pps_test_msg(self, msg):
427 """Get info from message of NDRPDR PPS tests.
429 :param msg: Message to be processed.
431 :returns: Processed message or "Test Failed." if a problem occurs.
435 groups = re.search(self.REGEX_PPS_MSG_INFO, msg)
436 if not groups or groups.lastindex != 4:
437 return u"Test Failed."
441 f"1. {(float(groups.group(1)) / 1e6):5.2f} "
442 f"{float(groups.group(2)):5.2f}\n"
443 f"2. {(float(groups.group(3)) / 1e6):5.2f} "
444 f"{float(groups.group(4)):5.2f}"
446 except (AttributeError, IndexError, ValueError, KeyError):
447 return u"Test Failed."
449 def _get_data_from_perf_test_msg(self, msg):
450 """Get info from message of NDRPDR performance tests.
452 :param msg: Message to be processed.
454 :returns: Processed message or "Test Failed." if a problem occurs.
458 groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
459 if not groups or groups.lastindex != 10:
460 return u"Test Failed."
464 u"ndr_low": float(groups.group(1)),
465 u"ndr_low_b": float(groups.group(2)),
466 u"pdr_low": float(groups.group(3)),
467 u"pdr_low_b": float(groups.group(4)),
468 u"pdr_lat_90_1": groups.group(5),
469 u"pdr_lat_90_2": groups.group(6),
470 u"pdr_lat_50_1": groups.group(7),
471 u"pdr_lat_50_2": groups.group(8),
472 u"pdr_lat_10_1": groups.group(9),
473 u"pdr_lat_10_2": groups.group(10),
475 except (AttributeError, IndexError, ValueError, KeyError):
476 return u"Test Failed."
478 def _process_lat(in_str_1, in_str_2):
479 """Extract min, avg, max values from latency string.
481 :param in_str_1: Latency string for one direction produced by robot
483 :param in_str_2: Latency string for second direction produced by
487 :returns: Processed latency string or None if a problem occurs.
490 in_list_1 = in_str_1.split('/', 3)
491 in_list_2 = in_str_2.split('/', 3)
493 if len(in_list_1) != 4 and len(in_list_2) != 4:
496 in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
498 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
499 except hdrh.codec.HdrLengthException:
502 in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
504 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
505 except hdrh.codec.HdrLengthException:
508 if hdr_lat_1 and hdr_lat_2:
510 hdr_lat_1.get_value_at_percentile(50.0),
511 hdr_lat_1.get_value_at_percentile(90.0),
512 hdr_lat_1.get_value_at_percentile(99.0),
513 hdr_lat_2.get_value_at_percentile(50.0),
514 hdr_lat_2.get_value_at_percentile(90.0),
515 hdr_lat_2.get_value_at_percentile(99.0)
525 f"1. {(data[u'ndr_low'] / 1e6):5.2f} "
526 f"{data[u'ndr_low_b']:5.2f}"
527 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f} "
528 f"{data[u'pdr_low_b']:5.2f}"
531 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
532 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
533 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
536 max_len = len(str(max((max(item) for item in latency))))
537 max_len = 4 if max_len < 4 else max_len
539 for idx, lat in enumerate(latency):
544 f"{lat[0]:{max_len}d} "
545 f"{lat[1]:{max_len}d} "
546 f"{lat[2]:{max_len}d} "
547 f"{lat[3]:{max_len}d} "
548 f"{lat[4]:{max_len}d} "
549 f"{lat[5]:{max_len}d} "
554 except (AttributeError, IndexError, ValueError, KeyError):
555 return u"Test Failed."
557 def _get_testbed(self, msg):
558 """Called when extraction of testbed IP is required.
559 The testbed is identified by TG node IP address.
561 :param msg: Message to process.
566 if msg.message.count(u"Setup of TG node") or \
567 msg.message.count(u"Setup of node TG host"):
568 reg_tg_ip = re.compile(
569 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
571 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
572 except (KeyError, ValueError, IndexError, AttributeError):
575 self._data[u"metadata"][u"testbed"] = self._testbed
576 self._msg_type = None
578 def _get_vpp_version(self, msg):
579 """Called when extraction of VPP version is required.
581 :param msg: Message to process.
586 if msg.message.count(u"return STDOUT Version:") or \
587 msg.message.count(u"VPP Version:") or \
588 msg.message.count(u"VPP version:"):
590 re.search(self.REGEX_VERSION_VPP, msg.message).group(2)
592 self._data[u"metadata"][u"version"] = self._version
593 self._msg_type = None
595 def _get_dpdk_version(self, msg):
596 """Called when extraction of DPDK version is required.
598 :param msg: Message to process.
603 if msg.message.count(u"DPDK Version:"):
605 self._version = str(re.search(
606 self.REGEX_VERSION_DPDK, msg.message).group(2))
607 self._data[u"metadata"][u"version"] = self._version
611 self._msg_type = None
613 def _get_timestamp(self, msg):
614 """Called when extraction of timestamp is required.
616 :param msg: Message to process.
621 self._timestamp = msg.timestamp[:14]
622 self._data[u"metadata"][u"generated"] = self._timestamp
623 self._msg_type = None
625 def _get_papi_history(self, msg):
626 """Called when extraction of PAPI command history is required.
628 :param msg: Message to process.
632 if msg.message.count(u"PAPI command history:"):
633 self._conf_history_lookup_nr += 1
634 if self._conf_history_lookup_nr == 1:
635 self._data[u"tests"][self._test_id][u"conf-history"] = str()
637 self._msg_type = None
639 r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} PAPI command history:",
643 ).replace(u'"', u"'")
644 self._data[u"tests"][self._test_id][u"conf-history"] += (
645 f"**DUT{str(self._conf_history_lookup_nr)}:** {text}"
648 def _get_show_run(self, msg):
649 """Called when extraction of VPP operational data (output of CLI command
650 Show Runtime) is required.
652 :param msg: Message to process.
657 if not msg.message.count(u"stats runtime"):
661 if self._sh_run_counter > 1:
664 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
665 self._data[u"tests"][self._test_id][u"show-run"] = dict()
667 groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
671 host = groups.group(1)
672 except (AttributeError, IndexError):
675 sock = groups.group(2)
676 except (AttributeError, IndexError):
679 dut = u"dut{nr}".format(
680 nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
682 self._data[u'tests'][self._test_id][u'show-run'][dut] = \
687 u"runtime": str(msg.message).replace(u' ', u'').
688 replace(u'\n', u'').replace(u"'", u'"').
689 replace(u'b"', u'"').replace(u'u"', u'"').
694 def _get_telemetry(self, msg):
695 """Called when extraction of VPP telemetry data is required.
697 :param msg: Message to process.
702 if self._telemetry_kw_counter > 1:
704 if not msg.message.count(u"vpp_runtime_calls"):
707 if u"telemetry-show-run" not in \
708 self._data[u"tests"][self._test_id].keys():
709 self._data[u"tests"][self._test_id][u"telemetry-show-run"] = dict()
711 self._telemetry_msg_counter += 1
712 dut = f"dut{self._telemetry_msg_counter}"
714 u"source_type": u"node",
716 u"msg_type": u"metric",
717 u"log_level": u"INFO",
718 u"timestamp": msg.timestamp,
719 u"msg": u"show_runtime",
720 u"host": dut, # No info, should be host IP
721 u"socket": u"", # No info
724 for line in msg.message.splitlines():
725 if not line.startswith(u"vpp_runtime_"):
728 params, value = line.rsplit(u" ", maxsplit=2)[:-1]
729 cut = params.index(u"{")
730 name = params[:cut].split(u"_", maxsplit=2)[-1]
732 u"dict" + params[cut:].replace('{', '(').replace('}', ')')
734 labels[u"graph_node"] = labels.pop(u"name")
735 runtime[u"data"].append(
742 except (TypeError, ValueError, IndexError):
745 self._data[u'tests'][self._test_id][u'telemetry-show-run'][dut] = \
754 def _get_ndrpdr_throughput(self, msg):
755 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
758 :param msg: The test message to be parsed.
760 :returns: Parsed data as a dict and the status (PASS/FAIL).
761 :rtype: tuple(dict, str)
765 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
766 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
769 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
771 if groups is not None:
773 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
774 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
775 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
776 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
778 except (IndexError, ValueError):
781 return throughput, status
783 def _get_ndrpdr_throughput_gbps(self, msg):
784 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
787 :param msg: The test message to be parsed.
789 :returns: Parsed data as a dict and the status (PASS/FAIL).
790 :rtype: tuple(dict, str)
794 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
795 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
798 groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
800 if groups is not None:
802 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
803 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
804 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
805 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
807 except (IndexError, ValueError):
812 def _get_plr_throughput(self, msg):
813 """Get PLRsearch lower bound and PLRsearch upper bound from the test
816 :param msg: The test message to be parsed.
818 :returns: Parsed data as a dict and the status (PASS/FAIL).
819 :rtype: tuple(dict, str)
827 groups = re.search(self.REGEX_PLR_RATE, msg)
829 if groups is not None:
831 throughput[u"LOWER"] = float(groups.group(1))
832 throughput[u"UPPER"] = float(groups.group(2))
834 except (IndexError, ValueError):
837 return throughput, status
839 def _get_ndrpdr_latency(self, msg):
840 """Get LATENCY from the test message.
842 :param msg: The test message to be parsed.
844 :returns: Parsed data as a dict and the status (PASS/FAIL).
845 :rtype: tuple(dict, str)
855 u"direction1": copy.copy(latency_default),
856 u"direction2": copy.copy(latency_default)
859 u"direction1": copy.copy(latency_default),
860 u"direction2": copy.copy(latency_default)
863 u"direction1": copy.copy(latency_default),
864 u"direction2": copy.copy(latency_default)
867 u"direction1": copy.copy(latency_default),
868 u"direction2": copy.copy(latency_default)
871 u"direction1": copy.copy(latency_default),
872 u"direction2": copy.copy(latency_default)
875 u"direction1": copy.copy(latency_default),
876 u"direction2": copy.copy(latency_default)
880 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
882 groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
884 return latency, u"FAIL"
886 def process_latency(in_str):
887 """Return object with parsed latency values.
889 TODO: Define class for the return type.
891 :param in_str: Input string, min/avg/max/hdrh format.
893 :returns: Dict with corresponding keys, except hdrh float values.
895 :throws IndexError: If in_str does not have enough substrings.
896 :throws ValueError: If a substring does not convert to float.
898 in_list = in_str.split('/', 3)
901 u"min": float(in_list[0]),
902 u"avg": float(in_list[1]),
903 u"max": float(in_list[2]),
907 if len(in_list) == 4:
908 rval[u"hdrh"] = str(in_list[3])
913 latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
914 latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
915 latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
916 latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
917 if groups.lastindex == 4:
918 return latency, u"PASS"
919 except (IndexError, ValueError):
923 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
924 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
925 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
926 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
927 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
928 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
929 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
930 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
931 if groups.lastindex == 12:
932 return latency, u"PASS"
933 except (IndexError, ValueError):
936 return latency, u"FAIL"
939 def _get_hoststack_data(msg, tags):
940 """Get data from the hoststack test message.
942 :param msg: The test message to be parsed.
943 :param tags: Test tags.
946 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
947 :rtype: tuple(dict, str)
952 msg = msg.replace(u"'", u'"').replace(u" ", u"")
953 if u"LDPRELOAD" in tags:
957 except JSONDecodeError:
959 elif u"VPPECHO" in tags:
961 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
963 client=loads(msg_lst[0]),
964 server=loads(msg_lst[1])
967 except (JSONDecodeError, IndexError):
970 return result, status
972 def _get_vsap_data(self, msg, tags):
973 """Get data from the vsap test message.
975 :param msg: The test message to be parsed.
976 :param tags: Test tags.
979 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
980 :rtype: tuple(dict, str)
985 groups = re.search(self.REGEX_VSAP_MSG_INFO, msg)
986 if groups is not None:
988 result[u"transfer-rate"] = float(groups.group(1)) * 1e3
989 result[u"latency"] = float(groups.group(2))
990 result[u"completed-requests"] = int(groups.group(3))
991 result[u"failed-requests"] = int(groups.group(4))
992 result[u"bytes-transferred"] = int(groups.group(5))
993 if u"TCP_CPS"in tags:
994 result[u"cps"] = float(groups.group(6))
995 elif u"TCP_RPS" in tags:
996 result[u"rps"] = float(groups.group(6))
998 return result, status
1000 except (IndexError, ValueError):
1003 return result, status
1005 def visit_suite(self, suite):
1006 """Implements traversing through the suite and its direct children.
1008 :param suite: Suite to process.
1012 if self.start_suite(suite) is not False:
1013 suite.suites.visit(self)
1014 suite.tests.visit(self)
1015 self.end_suite(suite)
1017 def start_suite(self, suite):
1018 """Called when suite starts.
1020 :param suite: Suite to process.
1026 parent_name = suite.parent.name
1027 except AttributeError:
1030 self._data[u"suites"][suite.longname.lower().
1031 replace(u'"', u"'").
1032 replace(u" ", u"_")] = {
1033 u"name": suite.name.lower(),
1035 u"parent": parent_name,
1036 u"level": len(suite.longname.split(u"."))
1039 suite.keywords.visit(self)
1041 def end_suite(self, suite):
1042 """Called when suite ends.
1044 :param suite: Suite to process.
1049 def visit_test(self, test):
1050 """Implements traversing through the test.
1052 :param test: Test to process.
1056 if self.start_test(test) is not False:
1057 test.keywords.visit(self)
1060 def start_test(self, test):
1061 """Called when test starts.
1063 :param test: Test to process.
1068 self._sh_run_counter = 0
1069 self._telemetry_kw_counter = 0
1070 self._telemetry_msg_counter = 0
1072 longname_orig = test.longname.lower()
1074 # Check the ignore list
1075 if longname_orig in self._ignore:
1078 tags = [str(tag) for tag in test.tags]
1079 test_result = dict()
1081 # Change the TC long name and name if defined in the mapping table
1082 longname = self._mapping.get(longname_orig, None)
1083 if longname is not None:
1084 name = longname.split(u'.')[-1]
1086 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1090 longname = longname_orig
1091 name = test.name.lower()
1093 # Remove TC number from the TC long name (backward compatibility):
1094 self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1095 # Remove TC number from the TC name (not needed):
1096 test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1098 test_result[u"parent"] = test.parent.name.lower()
1099 test_result[u"tags"] = tags
1100 test_result["doc"] = test.doc
1101 test_result[u"type"] = u""
1102 test_result[u"status"] = test.status
1103 test_result[u"starttime"] = test.starttime
1104 test_result[u"endtime"] = test.endtime
1106 if test.status == u"PASS":
1107 if u"NDRPDR" in tags:
1108 if u"TCP_PPS" in tags or u"UDP_PPS" in tags:
1109 test_result[u"msg"] = self._get_data_from_pps_test_msg(
1111 elif u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1112 test_result[u"msg"] = self._get_data_from_cps_test_msg(
1115 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1117 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1118 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1121 test_result[u"msg"] = test.message
1123 test_result[u"msg"] = test.message
1125 if u"PERFTEST" in tags:
1126 # Replace info about cores (e.g. -1c-) with the info about threads
1127 # and cores (e.g. -1t1c-) in the long test case names and in the
1128 # test case names if necessary.
1129 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1133 for tag in test_result[u"tags"]:
1134 groups = re.search(self.REGEX_TC_TAG, tag)
1140 self._test_id = re.sub(
1141 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1142 self._test_id, count=1
1144 test_result[u"name"] = re.sub(
1145 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1146 test_result["name"], count=1
1149 test_result[u"status"] = u"FAIL"
1150 self._data[u"tests"][self._test_id] = test_result
1152 f"The test {self._test_id} has no or more than one "
1153 f"multi-threading tags.\n"
1154 f"Tags: {test_result[u'tags']}"
1158 if u"DEVICETEST" in tags:
1159 test_result[u"type"] = u"DEVICETEST"
1160 elif u"NDRPDR" in tags:
1161 if u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1162 test_result[u"type"] = u"CPS"
1164 test_result[u"type"] = u"NDRPDR"
1165 if test.status == u"PASS":
1166 test_result[u"throughput"], test_result[u"status"] = \
1167 self._get_ndrpdr_throughput(test.message)
1168 test_result[u"gbps"], test_result[u"status"] = \
1169 self._get_ndrpdr_throughput_gbps(test.message)
1170 test_result[u"latency"], test_result[u"status"] = \
1171 self._get_ndrpdr_latency(test.message)
1172 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1174 test_result[u"type"] = u"MRR"
1176 test_result[u"type"] = u"BMRR"
1177 if test.status == u"PASS":
1178 test_result[u"result"] = dict()
1179 groups = re.search(self.REGEX_BMRR, test.message)
1180 if groups is not None:
1181 items_str = groups.group(1)
1183 float(item.strip().replace(u"'", u""))
1184 for item in items_str.split(",")
1186 # Use whole list in CSIT-1180.
1187 stats = jumpavg.AvgStdevStats.for_runs(items_float)
1188 test_result[u"result"][u"samples"] = items_float
1189 test_result[u"result"][u"receive-rate"] = stats.avg
1190 test_result[u"result"][u"receive-stdev"] = stats.stdev
1192 groups = re.search(self.REGEX_MRR, test.message)
1193 test_result[u"result"][u"receive-rate"] = \
1194 float(groups.group(3)) / float(groups.group(1))
1195 elif u"SOAK" in tags:
1196 test_result[u"type"] = u"SOAK"
1197 if test.status == u"PASS":
1198 test_result[u"throughput"], test_result[u"status"] = \
1199 self._get_plr_throughput(test.message)
1200 elif u"HOSTSTACK" in tags:
1201 test_result[u"type"] = u"HOSTSTACK"
1202 if test.status == u"PASS":
1203 test_result[u"result"], test_result[u"status"] = \
1204 self._get_hoststack_data(test.message, tags)
1205 elif u"LDP_NGINX" in tags:
1206 test_result[u"type"] = u"LDP_NGINX"
1207 test_result[u"result"], test_result[u"status"] = \
1208 self._get_vsap_data(test.message, tags)
1209 # elif u"TCP" in tags: # This might be not used
1210 # test_result[u"type"] = u"TCP"
1211 # if test.status == u"PASS":
1212 # groups = re.search(self.REGEX_TCP, test.message)
1213 # test_result[u"result"] = int(groups.group(2))
1214 elif u"RECONF" in tags:
1215 test_result[u"type"] = u"RECONF"
1216 if test.status == u"PASS":
1217 test_result[u"result"] = None
1219 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1220 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1221 test_result[u"result"] = {
1222 u"loss": int(grps_loss.group(1)),
1223 u"time": float(grps_time.group(1))
1225 except (AttributeError, IndexError, ValueError, TypeError):
1226 test_result[u"status"] = u"FAIL"
1228 test_result[u"status"] = u"FAIL"
1230 self._data[u"tests"][self._test_id] = test_result
1232 def end_test(self, test):
1233 """Called when test ends.
1235 :param test: Test to process.
1240 def visit_keyword(self, keyword):
1241 """Implements traversing through the keyword and its child keywords.
1243 :param keyword: Keyword to process.
1244 :type keyword: Keyword
1247 if self.start_keyword(keyword) is not False:
1248 self.end_keyword(keyword)
1250 def start_keyword(self, keyword):
1251 """Called when keyword starts. Default implementation does nothing.
1253 :param keyword: Keyword to process.
1254 :type keyword: Keyword
1258 if keyword.type == u"setup":
1259 self.visit_setup_kw(keyword)
1260 elif keyword.type == u"teardown":
1261 self.visit_teardown_kw(keyword)
1263 self.visit_test_kw(keyword)
1264 except AttributeError:
1267 def end_keyword(self, keyword):
1268 """Called when keyword ends. Default implementation does nothing.
1270 :param keyword: Keyword to process.
1271 :type keyword: Keyword
1275 def visit_test_kw(self, test_kw):
1276 """Implements traversing through the test keyword and its child
1279 :param test_kw: Keyword to process.
1280 :type test_kw: Keyword
1283 for keyword in test_kw.keywords:
1284 if self.start_test_kw(keyword) is not False:
1285 self.visit_test_kw(keyword)
1286 self.end_test_kw(keyword)
1288 def start_test_kw(self, test_kw):
1289 """Called when test keyword starts. Default implementation does
1292 :param test_kw: Keyword to process.
1293 :type test_kw: Keyword
1296 if self._for_output == u"trending":
1299 if test_kw.name.count(u"Run Telemetry On All Duts"):
1300 self._msg_type = u"test-telemetry"
1301 self._telemetry_kw_counter += 1
1302 elif test_kw.name.count(u"Show Runtime On All Duts"):
1303 self._msg_type = u"test-show-runtime"
1304 self._sh_run_counter += 1
1307 test_kw.messages.visit(self)
1309 def end_test_kw(self, test_kw):
1310 """Called when keyword ends. Default implementation does nothing.
1312 :param test_kw: Keyword to process.
1313 :type test_kw: Keyword
1317 def visit_setup_kw(self, setup_kw):
1318 """Implements traversing through the teardown keyword and its child
1321 :param setup_kw: Keyword to process.
1322 :type setup_kw: Keyword
1325 for keyword in setup_kw.keywords:
1326 if self.start_setup_kw(keyword) is not False:
1327 self.visit_setup_kw(keyword)
1328 self.end_setup_kw(keyword)
1330 def start_setup_kw(self, setup_kw):
1331 """Called when teardown keyword starts. Default implementation does
1334 :param setup_kw: Keyword to process.
1335 :type setup_kw: Keyword
1338 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1339 and not self._version:
1340 self._msg_type = u"vpp-version"
1341 elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1343 self._msg_type = u"dpdk-version"
1344 elif setup_kw.name.count(u"Set Global Variable") \
1345 and not self._timestamp:
1346 self._msg_type = u"timestamp"
1347 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1348 self._msg_type = u"testbed"
1351 setup_kw.messages.visit(self)
1353 def end_setup_kw(self, setup_kw):
1354 """Called when keyword ends. Default implementation does nothing.
1356 :param setup_kw: Keyword to process.
1357 :type setup_kw: Keyword
1361 def visit_teardown_kw(self, teardown_kw):
1362 """Implements traversing through the teardown keyword and its child
1365 :param teardown_kw: Keyword to process.
1366 :type teardown_kw: Keyword
1369 for keyword in teardown_kw.keywords:
1370 if self.start_teardown_kw(keyword) is not False:
1371 self.visit_teardown_kw(keyword)
1372 self.end_teardown_kw(keyword)
1374 def start_teardown_kw(self, teardown_kw):
1375 """Called when teardown keyword starts
1377 :param teardown_kw: Keyword to process.
1378 :type teardown_kw: Keyword
1381 if teardown_kw.name.count(u"Show Papi History On All Duts"):
1382 self._conf_history_lookup_nr = 0
1383 self._msg_type = u"teardown-papi-history"
1384 teardown_kw.messages.visit(self)
1386 def end_teardown_kw(self, teardown_kw):
1387 """Called when keyword ends. Default implementation does nothing.
1389 :param teardown_kw: Keyword to process.
1390 :type teardown_kw: Keyword
1394 def visit_message(self, msg):
1395 """Implements visiting the message.
1397 :param msg: Message to process.
1401 if self.start_message(msg) is not False:
1402 self.end_message(msg)
1404 def start_message(self, msg):
1405 """Called when message starts. Get required information from messages:
1408 :param msg: Message to process.
1413 self.parse_msg[self._msg_type](msg)
1415 def end_message(self, msg):
1416 """Called when message ends. Default implementation does nothing.
1418 :param msg: Message to process.
1427 The data is extracted from output.xml files generated by Jenkins jobs and
1428 stored in pandas' DataFrames.
1434 (as described in ExecutionChecker documentation)
1436 (as described in ExecutionChecker documentation)
1438 (as described in ExecutionChecker documentation)
1441 def __init__(self, spec, for_output):
1444 :param spec: Specification.
1445 :param for_output: Output to be generated from downloaded data.
1446 :type spec: Specification
1447 :type for_output: str
1453 self._for_output = for_output
1456 self._input_data = pd.Series()
1460 """Getter - Input data.
1462 :returns: Input data
1463 :rtype: pandas.Series
1465 return self._input_data
1467 def metadata(self, job, build):
1468 """Getter - metadata
1470 :param job: Job which metadata we want.
1471 :param build: Build which metadata we want.
1475 :rtype: pandas.Series
1477 return self.data[job][build][u"metadata"]
1479 def suites(self, job, build):
1482 :param job: Job which suites we want.
1483 :param build: Build which suites we want.
1487 :rtype: pandas.Series
1489 return self.data[job][str(build)][u"suites"]
1491 def tests(self, job, build):
1494 :param job: Job which tests we want.
1495 :param build: Build which tests we want.
1499 :rtype: pandas.Series
1501 return self.data[job][build][u"tests"]
1503 def _parse_tests(self, job, build):
1504 """Process data from robot output.xml file and return JSON structured
1507 :param job: The name of job which build output data will be processed.
1508 :param build: The build which output data will be processed.
1511 :returns: JSON data structure.
1520 with open(build[u"file-name"], u'r') as data_file:
1522 result = ExecutionResult(data_file)
1523 except errors.DataError as err:
1525 f"Error occurred while parsing output.xml: {repr(err)}"
1528 checker = ExecutionChecker(
1529 metadata, self._cfg.mapping, self._cfg.ignore, self._for_output
1531 result.visit(checker)
1535 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1536 """Download and parse the input data file.
1538 :param pid: PID of the process executing this method.
1539 :param job: Name of the Jenkins job which generated the processed input
1541 :param build: Information about the Jenkins build which generated the
1542 processed input file.
1543 :param repeat: Repeat the download specified number of times if not
1551 logging.info(f"Processing the job/build: {job}: {build[u'build']}")
1558 success = download_and_unzip_data_file(self._cfg, job, build, pid)
1564 f"It is not possible to download the input data file from the "
1565 f"job {job}, build {build[u'build']}, or it is damaged. "
1569 logging.info(f" Processing data from build {build[u'build']}")
1570 data = self._parse_tests(job, build)
1573 f"Input data file from the job {job}, build "
1574 f"{build[u'build']} is damaged. Skipped."
1577 state = u"processed"
1580 remove(build[u"file-name"])
1581 except OSError as err:
1583 f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1586 # If the time-period is defined in the specification file, remove all
1587 # files which are outside the time period.
1589 timeperiod = self._cfg.environment.get(u"time-period", None)
1590 if timeperiod and data:
1592 timeperiod = timedelta(int(timeperiod))
1593 metadata = data.get(u"metadata", None)
1595 generated = metadata.get(u"generated", None)
1597 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1598 if (now - generated) > timeperiod:
1599 # Remove the data and the file:
1604 f" The build {job}/{build[u'build']} is "
1605 f"outdated, will be removed."
1615 def download_and_parse_data(self, repeat=1):
1616 """Download the input data files, parse input data from input files and
1617 store in pandas' Series.
1619 :param repeat: Repeat the download specified number of times if not
1624 logging.info(u"Downloading and parsing input files ...")
1626 for job, builds in self._cfg.input.items():
1627 for build in builds:
1629 result = self._download_and_parse_build(job, build, repeat)
1632 build_nr = result[u"build"][u"build"]
1635 data = result[u"data"]
1636 build_data = pd.Series({
1637 u"metadata": pd.Series(
1638 list(data[u"metadata"].values()),
1639 index=list(data[u"metadata"].keys())
1641 u"suites": pd.Series(
1642 list(data[u"suites"].values()),
1643 index=list(data[u"suites"].keys())
1645 u"tests": pd.Series(
1646 list(data[u"tests"].values()),
1647 index=list(data[u"tests"].keys())
1651 if self._input_data.get(job, None) is None:
1652 self._input_data[job] = pd.Series()
1653 self._input_data[job][str(build_nr)] = build_data
1654 self._cfg.set_input_file_name(
1655 job, build_nr, result[u"build"][u"file-name"]
1657 self._cfg.set_input_state(job, build_nr, result[u"state"])
1660 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1661 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1663 logging.info(u"Done.")
1665 msg = f"Successful downloads from the sources:\n"
1666 for source in self._cfg.environment[u"data-sources"]:
1667 if source[u"successful-downloads"]:
1669 f"{source[u'url']}/{source[u'path']}/"
1670 f"{source[u'file-name']}: "
1671 f"{source[u'successful-downloads']}\n"
1675 def process_local_file(self, local_file, job=u"local", build_nr=1,
1677 """Process local XML file given as a command-line parameter.
1679 :param local_file: The file to process.
1680 :param job: Job name.
1681 :param build_nr: Build number.
1682 :param replace: If True, the information about jobs and builds is
1683 replaced by the new one, otherwise the new jobs and builds are
1685 :type local_file: str
1689 :raises: PresentationError if an error occurs.
1691 if not isfile(local_file):
1692 raise PresentationError(f"The file {local_file} does not exist.")
1695 build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1696 except (IndexError, ValueError):
1701 u"status": u"failed",
1702 u"file-name": local_file
1705 self._cfg.input = dict()
1706 self._cfg.add_build(job, build)
1708 logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1709 data = self._parse_tests(job, build)
1711 raise PresentationError(
1712 f"Error occurred while parsing the file {local_file}"
1715 build_data = pd.Series({
1716 u"metadata": pd.Series(
1717 list(data[u"metadata"].values()),
1718 index=list(data[u"metadata"].keys())
1720 u"suites": pd.Series(
1721 list(data[u"suites"].values()),
1722 index=list(data[u"suites"].keys())
1724 u"tests": pd.Series(
1725 list(data[u"tests"].values()),
1726 index=list(data[u"tests"].keys())
1730 if self._input_data.get(job, None) is None:
1731 self._input_data[job] = pd.Series()
1732 self._input_data[job][str(build_nr)] = build_data
1734 self._cfg.set_input_state(job, build_nr, u"processed")
1736 def process_local_directory(self, local_dir, replace=True):
1737 """Process local directory with XML file(s). The directory is processed
1738 as a 'job' and the XML files in it as builds.
1739 If the given directory contains only sub-directories, these
1740 sub-directories processed as jobs and corresponding XML files as builds
1743 :param local_dir: Local directory to process.
1744 :param replace: If True, the information about jobs and builds is
1745 replaced by the new one, otherwise the new jobs and builds are
1747 :type local_dir: str
1750 if not isdir(local_dir):
1751 raise PresentationError(
1752 f"The directory {local_dir} does not exist."
1755 # Check if the given directory includes only files, or only directories
1756 _, dirnames, filenames = next(walk(local_dir))
1758 if filenames and not dirnames:
1761 # key: dir (job) name, value: list of file names (builds)
1763 local_dir: [join(local_dir, name) for name in filenames]
1766 elif dirnames and not filenames:
1769 # key: dir (job) name, value: list of file names (builds)
1770 local_builds = dict()
1771 for dirname in dirnames:
1773 join(local_dir, dirname, name)
1774 for name in listdir(join(local_dir, dirname))
1775 if isfile(join(local_dir, dirname, name))
1778 local_builds[dirname] = sorted(builds)
1780 elif not filenames and not dirnames:
1781 raise PresentationError(f"The directory {local_dir} is empty.")
1783 raise PresentationError(
1784 f"The directory {local_dir} can include only files or only "
1785 f"directories, not both.\nThe directory {local_dir} includes "
1786 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1790 self._cfg.input = dict()
1792 for job, files in local_builds.items():
1793 for idx, local_file in enumerate(files):
1794 self.process_local_file(local_file, job, idx + 1, replace=False)
1797 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1798 """Return the index of character in the string which is the end of tag.
1800 :param tag_filter: The string where the end of tag is being searched.
1801 :param start: The index where the searching is stated.
1802 :param closer: The character which is the tag closer.
1803 :type tag_filter: str
1806 :returns: The index of the tag closer.
1810 idx_opener = tag_filter.index(closer, start)
1811 return tag_filter.index(closer, idx_opener + 1)
1816 def _condition(tag_filter):
1817 """Create a conditional statement from the given tag filter.
1819 :param tag_filter: Filter based on tags from the element specification.
1820 :type tag_filter: str
1821 :returns: Conditional statement which can be evaluated.
1826 index = InputData._end_of_tag(tag_filter, index)
1830 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1832 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1833 continue_on_error=False):
1834 """Filter required data from the given jobs and builds.
1836 The output data structure is:
1839 - test (or suite) 1 ID:
1845 - test (or suite) n ID:
1852 :param element: Element which will use the filtered data.
1853 :param params: Parameters which will be included in the output. If None,
1854 all parameters are included.
1855 :param data: If not None, this data is used instead of data specified
1857 :param data_set: The set of data to be filtered: tests, suites,
1859 :param continue_on_error: Continue if there is error while reading the
1860 data. The Item will be empty then
1861 :type element: pandas.Series
1865 :type continue_on_error: bool
1866 :returns: Filtered data.
1867 :rtype pandas.Series
1871 if data_set == "suites":
1873 elif element[u"filter"] in (u"all", u"template"):
1876 cond = InputData._condition(element[u"filter"])
1877 logging.debug(f" Filter: {cond}")
1879 logging.error(u" No filter defined.")
1883 params = element.get(u"parameters", None)
1885 params.extend((u"type", u"status"))
1887 data_to_filter = data if data else element[u"data"]
1890 for job, builds in data_to_filter.items():
1891 data[job] = pd.Series()
1892 for build in builds:
1893 data[job][str(build)] = pd.Series()
1896 self.data[job][str(build)][data_set].items())
1898 if continue_on_error:
1902 for test_id, test_data in data_dict.items():
1903 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1904 data[job][str(build)][test_id] = pd.Series()
1906 for param, val in test_data.items():
1907 data[job][str(build)][test_id][param] = val
1909 for param in params:
1911 data[job][str(build)][test_id][param] =\
1914 data[job][str(build)][test_id][param] =\
1918 except (KeyError, IndexError, ValueError) as err:
1920 f"Missing mandatory parameter in the element specification: "
1924 except AttributeError as err:
1925 logging.error(repr(err))
1927 except SyntaxError as err:
1929 f"The filter {cond} is not correct. Check if all tags are "
1930 f"enclosed by apostrophes.\n{repr(err)}"
1934 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1935 continue_on_error=False):
1936 """Filter required data from the given jobs and builds.
1938 The output data structure is:
1941 - test (or suite) 1 ID:
1947 - test (or suite) n ID:
1954 :param element: Element which will use the filtered data.
1955 :param params: Parameters which will be included in the output. If None,
1956 all parameters are included.
1957 :param data_set: The set of data to be filtered: tests, suites,
1959 :param continue_on_error: Continue if there is error while reading the
1960 data. The Item will be empty then
1961 :type element: pandas.Series
1964 :type continue_on_error: bool
1965 :returns: Filtered data.
1966 :rtype pandas.Series
1969 include = element.get(u"include", None)
1971 logging.warning(u"No tests to include, skipping the element.")
1975 params = element.get(u"parameters", None)
1976 if params and u"type" not in params:
1977 params.append(u"type")
1979 cores = element.get(u"core", None)
1983 for test in include:
1984 tests.append(test.format(core=core))
1990 for job, builds in element[u"data"].items():
1991 data[job] = pd.Series()
1992 for build in builds:
1993 data[job][str(build)] = pd.Series()
1996 reg_ex = re.compile(str(test).lower())
1997 for test_id in self.data[job][
1998 str(build)][data_set].keys():
1999 if re.match(reg_ex, str(test_id).lower()):
2000 test_data = self.data[job][
2001 str(build)][data_set][test_id]
2002 data[job][str(build)][test_id] = pd.Series()
2004 for param, val in test_data.items():
2005 data[job][str(build)][test_id]\
2008 for param in params:
2010 data[job][str(build)][
2014 data[job][str(build)][
2015 test_id][param] = u"No Data"
2016 except KeyError as err:
2017 if continue_on_error:
2018 logging.debug(repr(err))
2020 logging.error(repr(err))
2024 except (KeyError, IndexError, ValueError) as err:
2026 f"Missing mandatory parameter in the element "
2027 f"specification: {repr(err)}"
2030 except AttributeError as err:
2031 logging.error(repr(err))
2035 def merge_data(data):
2036 """Merge data from more jobs and builds to a simple data structure.
2038 The output data structure is:
2040 - test (suite) 1 ID:
2046 - test (suite) n ID:
2049 :param data: Data to merge.
2050 :type data: pandas.Series
2051 :returns: Merged data.
2052 :rtype: pandas.Series
2055 logging.info(u" Merging data ...")
2057 merged_data = pd.Series()
2058 for builds in data.values:
2059 for item in builds.values:
2060 for item_id, item_data in item.items():
2061 merged_data[item_id] = item_data
2064 def print_all_oper_data(self):
2065 """Print all operational data to console.
2068 for job in self._input_data.values:
2069 for build in job.values:
2070 for test_id, test_data in build[u"tests"].items():
2072 if test_data.get(u"show-run", None) is None:
2074 for dut_name, data in test_data[u"show-run"].items():
2075 if data.get(u"runtime", None) is None:
2077 runtime = loads(data[u"runtime"])
2079 threads_nr = len(runtime[0][u"clocks"])
2080 except (IndexError, KeyError):
2082 threads = OrderedDict(
2083 {idx: list() for idx in range(threads_nr)})
2084 for item in runtime:
2085 for idx in range(threads_nr):
2086 if item[u"vectors"][idx] > 0:
2087 clocks = item[u"clocks"][idx] / \
2088 item[u"vectors"][idx]
2089 elif item[u"calls"][idx] > 0:
2090 clocks = item[u"clocks"][idx] / \
2092 elif item[u"suspends"][idx] > 0:
2093 clocks = item[u"clocks"][idx] / \
2094 item[u"suspends"][idx]
2098 if item[u"calls"][idx] > 0:
2099 vectors_call = item[u"vectors"][idx] / \
2104 if int(item[u"calls"][idx]) + int(
2105 item[u"vectors"][idx]) + \
2106 int(item[u"suspends"][idx]):
2107 threads[idx].append([
2109 item[u"calls"][idx],
2110 item[u"vectors"][idx],
2111 item[u"suspends"][idx],
2116 print(f"Host IP: {data.get(u'host', '')}, "
2117 f"Socket: {data.get(u'socket', '')}")
2118 for thread_nr, thread in threads.items():
2119 txt_table = prettytable.PrettyTable(
2125 u"Cycles per Packet",
2126 u"Average Vector Size"
2131 txt_table.add_row(row)
2133 if len(thread) == 0:
2136 avg = f", Average Vector Size per Node: " \
2137 f"{(avg / len(thread)):.2f}"
2138 th_name = u"main" if thread_nr == 0 \
2139 else f"worker_{thread_nr}"
2140 print(f"{dut_name}, {th_name}{avg}")
2141 txt_table.float_format = u".2"
2142 txt_table.align = u"r"
2143 txt_table.align[u"Name"] = u"l"
2144 print(f"{txt_table.get_string()}\n")