1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
28 from os import remove, walk, listdir
29 from os.path import isfile, isdir, join
30 from datetime import datetime as dt
31 from datetime import timedelta
32 from json import loads
33 from json.decoder import JSONDecodeError
40 from robot.api import ExecutionResult, ResultVisitor
41 from robot import errors
43 from resources.libraries.python import jumpavg
44 from input_data_files import download_and_unzip_data_file
45 from pal_errors import PresentationError
48 # Separator used in file names
52 class ExecutionChecker(ResultVisitor):
53 """Class to traverse through the test suite structure.
55 The functionality implemented in this class generates a json structure:
61 "generated": "Timestamp",
62 "version": "SUT version",
63 "job": "Jenkins job name",
64 "build": "Information about the build"
67 "Suite long name 1": {
69 "doc": "Suite 1 documentation",
70 "parent": "Suite 1 parent",
71 "level": "Level of the suite in the suite hierarchy"
73 "Suite long name N": {
75 "doc": "Suite N documentation",
76 "parent": "Suite 2 parent",
77 "level": "Level of the suite in the suite hierarchy"
84 "parent": "Name of the parent of the test",
85 "doc": "Test documentation",
86 "msg": "Test message",
87 "conf-history": "DUT1 and DUT2 VAT History",
88 "show-run": "Show Run",
89 "tags": ["tag 1", "tag 2", "tag n"],
91 "status": "PASS" | "FAIL",
137 "parent": "Name of the parent of the test",
138 "doc": "Test documentation",
139 "msg": "Test message",
140 "tags": ["tag 1", "tag 2", "tag n"],
142 "status": "PASS" | "FAIL",
149 "parent": "Name of the parent of the test",
150 "doc": "Test documentation",
151 "msg": "Test message",
152 "tags": ["tag 1", "tag 2", "tag n"],
153 "type": "MRR" | "BMRR",
154 "status": "PASS" | "FAIL",
156 "receive-rate": float,
157 # Average of a list, computed using AvgStdevStats.
158 # In CSIT-1180, replace with List[float].
172 "metadata": { # Optional
173 "version": "VPP version",
174 "job": "Jenkins job name",
175 "build": "Information about the build"
179 "doc": "Suite 1 documentation",
180 "parent": "Suite 1 parent",
181 "level": "Level of the suite in the suite hierarchy"
184 "doc": "Suite N documentation",
185 "parent": "Suite 2 parent",
186 "level": "Level of the suite in the suite hierarchy"
192 "parent": "Name of the parent of the test",
193 "doc": "Test documentation"
194 "msg": "Test message"
195 "tags": ["tag 1", "tag 2", "tag n"],
196 "conf-history": "DUT1 and DUT2 VAT History"
197 "show-run": "Show Run"
198 "status": "PASS" | "FAIL"
206 .. note:: ID is the lowercase full path to the test.
209 REGEX_PLR_RATE = re.compile(
210 r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
211 r'PLRsearch upper bound::?\s(\d+.\d+)'
213 REGEX_NDRPDR_RATE = re.compile(
214 r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
215 r'NDR_UPPER:\s(\d+.\d+).*\n'
216 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
217 r'PDR_UPPER:\s(\d+.\d+)'
219 REGEX_NDRPDR_GBPS = re.compile(
220 r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
221 r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
222 r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
223 r'PDR_UPPER:.*,\s(\d+.\d+)'
225 REGEX_PERF_MSG_INFO = re.compile(
226 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
227 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
228 r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
229 r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
230 r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
232 REGEX_CPS_MSG_INFO = re.compile(
233 r'NDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*\n'
234 r'PDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*'
236 REGEX_PPS_MSG_INFO = re.compile(
237 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
238 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*'
240 REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
242 REGEX_VSAP_MSG_INFO = re.compile(
243 r'Transfer Rate: (\d*.\d*).*\n'
244 r'Latency: (\d*.\d*).*\n'
245 r'Completed requests: (\d*).*\n'
246 r'Failed requests: (\d*).*\n'
247 r'Total data transferred: (\d*).*\n'
248 r'Connection [cr]ps rate:\s*(\d*.\d*)'
251 # Needed for CPS and PPS tests
252 REGEX_NDRPDR_LAT_BASE = re.compile(
253 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
254 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
256 REGEX_NDRPDR_LAT = re.compile(
257 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
258 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
259 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
260 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
261 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
262 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
265 REGEX_VERSION_VPP = re.compile(
266 r"(return STDOUT Version:\s*|"
267 r"VPP Version:\s*|VPP version:\s*)(.*)"
269 REGEX_VERSION_DPDK = re.compile(
270 r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
272 REGEX_TCP = re.compile(
273 r'Total\s(rps|cps|throughput):\s(\d*).*$'
275 REGEX_MRR = re.compile(
276 r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
277 r'tx\s(\d*),\srx\s(\d*)'
279 REGEX_BMRR = re.compile(
280 r'.*trial results.*: \[(.*)\]'
282 REGEX_RECONF_LOSS = re.compile(
283 r'Packets lost due to reconfig: (\d*)'
285 REGEX_RECONF_TIME = re.compile(
286 r'Implied time lost: (\d*.[\de-]*)'
288 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
290 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
292 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
294 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
296 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
298 REGEX_SH_RUN_HOST = re.compile(
299 r'hostname=\"(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})\",hook=\"(.*)\"'
302 def __init__(self, metadata, mapping, ignore, for_output):
305 :param metadata: Key-value pairs to be included in "metadata" part of
307 :param mapping: Mapping of the old names of test cases to the new
309 :param ignore: List of TCs to be ignored.
310 :param for_output: Output to be generated from downloaded data.
314 :type for_output: str
317 # Type of message to parse out from the test messages
318 self._msg_type = None
324 self._timestamp = None
326 # Testbed. The testbed is identified by TG node IP address.
329 # Mapping of TCs long names
330 self._mapping = mapping
333 self._ignore = ignore
335 self._for_output = for_output
337 # Number of PAPI History messages found:
339 # 1 - PAPI History of DUT1
340 # 2 - PAPI History of DUT2
341 self._conf_history_lookup_nr = 0
343 self._sh_run_counter = 0
344 self._telemetry_kw_counter = 0
345 self._telemetry_msg_counter = 0
347 # Test ID of currently processed test- the lowercase full path to the
351 # The main data structure
353 u"metadata": OrderedDict(),
354 u"suites": OrderedDict(),
355 u"tests": OrderedDict()
358 # Save the provided metadata
359 for key, val in metadata.items():
360 self._data[u"metadata"][key] = val
362 # Dictionary defining the methods used to parse different types of
365 u"vpp-version": self._get_vpp_version,
366 u"dpdk-version": self._get_dpdk_version,
367 u"teardown-papi-history": self._get_papi_history,
368 u"test-show-runtime": self._get_show_run,
369 u"testbed": self._get_testbed,
370 u"test-telemetry": self._get_telemetry
375 """Getter - Data parsed from the XML file.
377 :returns: Data parsed from the XML file.
382 def _get_data_from_mrr_test_msg(self, msg):
383 """Get info from message of MRR performance tests.
385 :param msg: Message to be processed.
387 :returns: Processed message or original message if a problem occurs.
391 groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
392 if not groups or groups.lastindex != 1:
393 return u"Test Failed."
396 data = groups.group(1).split(u", ")
397 except (AttributeError, IndexError, ValueError, KeyError):
398 return u"Test Failed."
403 out_str += f"{(float(item) / 1e6):.2f}, "
404 return out_str[:-2] + u"]"
405 except (AttributeError, IndexError, ValueError, KeyError):
406 return u"Test Failed."
408 def _get_data_from_cps_test_msg(self, msg):
409 """Get info from message of NDRPDR CPS tests.
411 :param msg: Message to be processed.
413 :returns: Processed message or "Test Failed." if a problem occurs.
417 groups = re.search(self.REGEX_CPS_MSG_INFO, msg)
418 if not groups or groups.lastindex != 2:
419 return u"Test Failed."
423 f"1. {(float(groups.group(1)) / 1e6):5.2f}\n"
424 f"2. {(float(groups.group(2)) / 1e6):5.2f}"
426 except (AttributeError, IndexError, ValueError, KeyError):
427 return u"Test Failed."
429 def _get_data_from_pps_test_msg(self, msg):
430 """Get info from message of NDRPDR PPS tests.
432 :param msg: Message to be processed.
434 :returns: Processed message or "Test Failed." if a problem occurs.
438 groups = re.search(self.REGEX_PPS_MSG_INFO, msg)
439 if not groups or groups.lastindex != 4:
440 return u"Test Failed."
444 f"1. {(float(groups.group(1)) / 1e6):5.2f} "
445 f"{float(groups.group(2)):5.2f}\n"
446 f"2. {(float(groups.group(3)) / 1e6):5.2f} "
447 f"{float(groups.group(4)):5.2f}"
449 except (AttributeError, IndexError, ValueError, KeyError):
450 return u"Test Failed."
452 def _get_data_from_perf_test_msg(self, msg):
453 """Get info from message of NDRPDR performance tests.
455 :param msg: Message to be processed.
457 :returns: Processed message or "Test Failed." if a problem occurs.
461 groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
462 if not groups or groups.lastindex != 10:
463 return u"Test Failed."
467 u"ndr_low": float(groups.group(1)),
468 u"ndr_low_b": float(groups.group(2)),
469 u"pdr_low": float(groups.group(3)),
470 u"pdr_low_b": float(groups.group(4)),
471 u"pdr_lat_90_1": groups.group(5),
472 u"pdr_lat_90_2": groups.group(6),
473 u"pdr_lat_50_1": groups.group(7),
474 u"pdr_lat_50_2": groups.group(8),
475 u"pdr_lat_10_1": groups.group(9),
476 u"pdr_lat_10_2": groups.group(10),
478 except (AttributeError, IndexError, ValueError, KeyError):
479 return u"Test Failed."
481 def _process_lat(in_str_1, in_str_2):
482 """Extract min, avg, max values from latency string.
484 :param in_str_1: Latency string for one direction produced by robot
486 :param in_str_2: Latency string for second direction produced by
490 :returns: Processed latency string or None if a problem occurs.
493 in_list_1 = in_str_1.split('/', 3)
494 in_list_2 = in_str_2.split('/', 3)
496 if len(in_list_1) != 4 and len(in_list_2) != 4:
499 in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
501 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
502 except hdrh.codec.HdrLengthException:
505 in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
507 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
508 except hdrh.codec.HdrLengthException:
511 if hdr_lat_1 and hdr_lat_2:
513 hdr_lat_1.get_value_at_percentile(50.0),
514 hdr_lat_1.get_value_at_percentile(90.0),
515 hdr_lat_1.get_value_at_percentile(99.0),
516 hdr_lat_2.get_value_at_percentile(50.0),
517 hdr_lat_2.get_value_at_percentile(90.0),
518 hdr_lat_2.get_value_at_percentile(99.0)
528 f"1. {(data[u'ndr_low'] / 1e6):5.2f} "
529 f"{data[u'ndr_low_b']:5.2f}"
530 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f} "
531 f"{data[u'pdr_low_b']:5.2f}"
534 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
535 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
536 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
539 max_len = len(str(max((max(item) for item in latency))))
540 max_len = 4 if max_len < 4 else max_len
542 for idx, lat in enumerate(latency):
547 f"{lat[0]:{max_len}d} "
548 f"{lat[1]:{max_len}d} "
549 f"{lat[2]:{max_len}d} "
550 f"{lat[3]:{max_len}d} "
551 f"{lat[4]:{max_len}d} "
552 f"{lat[5]:{max_len}d} "
557 except (AttributeError, IndexError, ValueError, KeyError):
558 return u"Test Failed."
560 def _get_testbed(self, msg):
561 """Called when extraction of testbed IP is required.
562 The testbed is identified by TG node IP address.
564 :param msg: Message to process.
569 if msg.message.count(u"Setup of TG node") or \
570 msg.message.count(u"Setup of node TG host"):
571 reg_tg_ip = re.compile(
572 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
574 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
575 except (KeyError, ValueError, IndexError, AttributeError):
578 self._data[u"metadata"][u"testbed"] = self._testbed
579 self._msg_type = None
581 def _get_vpp_version(self, msg):
582 """Called when extraction of VPP version is required.
584 :param msg: Message to process.
589 if msg.message.count(u"return STDOUT Version:") or \
590 msg.message.count(u"VPP Version:") or \
591 msg.message.count(u"VPP version:"):
593 re.search(self.REGEX_VERSION_VPP, msg.message).group(2)
595 self._data[u"metadata"][u"version"] = self._version
596 self._msg_type = None
598 def _get_dpdk_version(self, msg):
599 """Called when extraction of DPDK version is required.
601 :param msg: Message to process.
606 if msg.message.count(u"DPDK Version:"):
608 self._version = str(re.search(
609 self.REGEX_VERSION_DPDK, msg.message).group(2))
610 self._data[u"metadata"][u"version"] = self._version
614 self._msg_type = None
616 def _get_papi_history(self, msg):
617 """Called when extraction of PAPI command history is required.
619 :param msg: Message to process.
623 if msg.message.count(u"PAPI command history:"):
624 self._conf_history_lookup_nr += 1
625 if self._conf_history_lookup_nr == 1:
626 self._data[u"tests"][self._test_id][u"conf-history"] = str()
628 self._msg_type = None
630 r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} PAPI command history:",
634 ).replace(u'"', u"'")
635 self._data[u"tests"][self._test_id][u"conf-history"] += (
636 f"**DUT{str(self._conf_history_lookup_nr)}:** {text}"
639 def _get_show_run(self, msg):
640 """Called when extraction of VPP operational data (output of CLI command
641 Show Runtime) is required.
643 :param msg: Message to process.
648 if not msg.message.count(u"stats runtime"):
652 if self._sh_run_counter > 1:
655 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
656 self._data[u"tests"][self._test_id][u"show-run"] = dict()
658 groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
662 host = groups.group(1)
663 except (AttributeError, IndexError):
666 sock = groups.group(2)
667 except (AttributeError, IndexError):
670 dut = u"dut{nr}".format(
671 nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
673 self._data[u'tests'][self._test_id][u'show-run'][dut] = \
678 u"runtime": str(msg.message).replace(u' ', u'').
679 replace(u'\n', u'').replace(u"'", u'"').
680 replace(u'b"', u'"').replace(u'u"', u'"').
685 def _get_telemetry(self, msg):
686 """Called when extraction of VPP telemetry data is required.
688 :param msg: Message to process.
693 if self._telemetry_kw_counter > 1:
695 if not msg.message.count(u"# TYPE vpp_runtime_calls"):
698 if u"telemetry-show-run" not in \
699 self._data[u"tests"][self._test_id].keys():
700 self._data[u"tests"][self._test_id][u"telemetry-show-run"] = dict()
702 self._telemetry_msg_counter += 1
703 groups = re.search(self.REGEX_SH_RUN_HOST, msg.message)
707 host = groups.group(1)
708 except (AttributeError, IndexError):
711 sock = groups.group(2)
712 except (AttributeError, IndexError):
715 u"source_type": u"node",
717 u"msg_type": u"metric",
718 u"log_level": u"INFO",
719 u"timestamp": msg.timestamp,
720 u"msg": u"show_runtime",
725 for line in msg.message.splitlines():
726 if not line.startswith(u"vpp_runtime_"):
729 params, value, timestamp = line.rsplit(u" ", maxsplit=2)
730 cut = params.index(u"{")
731 name = params[:cut].split(u"_", maxsplit=2)[-1]
733 u"dict" + params[cut:].replace('{', '(').replace('}', ')')
735 labels[u"graph_node"] = labels.pop(u"name")
736 runtime[u"data"].append(
740 u"timestamp": timestamp,
744 except (TypeError, ValueError, IndexError):
746 self._data[u'tests'][self._test_id][u'telemetry-show-run']\
747 [f"dut{self._telemetry_msg_counter}"] = copy.copy(
755 def _get_ndrpdr_throughput(self, msg):
756 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
759 :param msg: The test message to be parsed.
761 :returns: Parsed data as a dict and the status (PASS/FAIL).
762 :rtype: tuple(dict, str)
766 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
767 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
770 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
772 if groups is not None:
774 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
775 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
776 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
777 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
779 except (IndexError, ValueError):
782 return throughput, status
784 def _get_ndrpdr_throughput_gbps(self, msg):
785 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
788 :param msg: The test message to be parsed.
790 :returns: Parsed data as a dict and the status (PASS/FAIL).
791 :rtype: tuple(dict, str)
795 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
796 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
799 groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
801 if groups is not None:
803 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
804 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
805 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
806 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
808 except (IndexError, ValueError):
813 def _get_plr_throughput(self, msg):
814 """Get PLRsearch lower bound and PLRsearch upper bound from the test
817 :param msg: The test message to be parsed.
819 :returns: Parsed data as a dict and the status (PASS/FAIL).
820 :rtype: tuple(dict, str)
828 groups = re.search(self.REGEX_PLR_RATE, msg)
830 if groups is not None:
832 throughput[u"LOWER"] = float(groups.group(1))
833 throughput[u"UPPER"] = float(groups.group(2))
835 except (IndexError, ValueError):
838 return throughput, status
840 def _get_ndrpdr_latency(self, msg):
841 """Get LATENCY from the test message.
843 :param msg: The test message to be parsed.
845 :returns: Parsed data as a dict and the status (PASS/FAIL).
846 :rtype: tuple(dict, str)
856 u"direction1": copy.copy(latency_default),
857 u"direction2": copy.copy(latency_default)
860 u"direction1": copy.copy(latency_default),
861 u"direction2": copy.copy(latency_default)
864 u"direction1": copy.copy(latency_default),
865 u"direction2": copy.copy(latency_default)
868 u"direction1": copy.copy(latency_default),
869 u"direction2": copy.copy(latency_default)
872 u"direction1": copy.copy(latency_default),
873 u"direction2": copy.copy(latency_default)
876 u"direction1": copy.copy(latency_default),
877 u"direction2": copy.copy(latency_default)
881 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
883 groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
885 return latency, u"FAIL"
887 def process_latency(in_str):
888 """Return object with parsed latency values.
890 TODO: Define class for the return type.
892 :param in_str: Input string, min/avg/max/hdrh format.
894 :returns: Dict with corresponding keys, except hdrh float values.
896 :throws IndexError: If in_str does not have enough substrings.
897 :throws ValueError: If a substring does not convert to float.
899 in_list = in_str.split('/', 3)
902 u"min": float(in_list[0]),
903 u"avg": float(in_list[1]),
904 u"max": float(in_list[2]),
908 if len(in_list) == 4:
909 rval[u"hdrh"] = str(in_list[3])
914 latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
915 latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
916 latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
917 latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
918 if groups.lastindex == 4:
919 return latency, u"PASS"
920 except (IndexError, ValueError):
924 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
925 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
926 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
927 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
928 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
929 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
930 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
931 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
932 if groups.lastindex == 12:
933 return latency, u"PASS"
934 except (IndexError, ValueError):
937 return latency, u"FAIL"
940 def _get_hoststack_data(msg, tags):
941 """Get data from the hoststack test message.
943 :param msg: The test message to be parsed.
944 :param tags: Test tags.
947 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
948 :rtype: tuple(dict, str)
953 msg = msg.replace(u"'", u'"').replace(u" ", u"")
954 if u"LDPRELOAD" in tags:
958 except JSONDecodeError:
960 elif u"VPPECHO" in tags:
962 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
964 client=loads(msg_lst[0]),
965 server=loads(msg_lst[1])
968 except (JSONDecodeError, IndexError):
971 return result, status
973 def _get_vsap_data(self, msg, tags):
974 """Get data from the vsap test message.
976 :param msg: The test message to be parsed.
977 :param tags: Test tags.
980 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
981 :rtype: tuple(dict, str)
986 groups = re.search(self.REGEX_VSAP_MSG_INFO, msg)
987 if groups is not None:
989 result[u"transfer-rate"] = float(groups.group(1)) * 1e3
990 result[u"latency"] = float(groups.group(2))
991 result[u"completed-requests"] = int(groups.group(3))
992 result[u"failed-requests"] = int(groups.group(4))
993 result[u"bytes-transferred"] = int(groups.group(5))
994 if u"TCP_CPS"in tags:
995 result[u"cps"] = float(groups.group(6))
996 elif u"TCP_RPS" in tags:
997 result[u"rps"] = float(groups.group(6))
999 return result, status
1001 except (IndexError, ValueError):
1004 return result, status
1006 def visit_suite(self, suite):
1007 """Implements traversing through the suite and its direct children.
1009 :param suite: Suite to process.
1013 if self.start_suite(suite) is not False:
1014 suite.suites.visit(self)
1015 suite.tests.visit(self)
1016 self.end_suite(suite)
1018 def start_suite(self, suite):
1019 """Called when suite starts.
1021 :param suite: Suite to process.
1027 parent_name = suite.parent.name
1028 except AttributeError:
1031 self._data[u"suites"][suite.longname.lower().
1032 replace(u'"', u"'").
1033 replace(u" ", u"_")] = {
1034 u"name": suite.name.lower(),
1036 u"parent": parent_name,
1037 u"level": len(suite.longname.split(u"."))
1040 suite.keywords.visit(self)
1042 def end_suite(self, suite):
1043 """Called when suite ends.
1045 :param suite: Suite to process.
1050 def visit_test(self, test):
1051 """Implements traversing through the test.
1053 :param test: Test to process.
1057 if self.start_test(test) is not False:
1058 test.keywords.visit(self)
1061 def start_test(self, test):
1062 """Called when test starts.
1064 :param test: Test to process.
1069 self._sh_run_counter = 0
1070 self._telemetry_kw_counter = 0
1071 self._telemetry_msg_counter = 0
1073 longname_orig = test.longname.lower()
1075 # Check the ignore list
1076 if longname_orig in self._ignore:
1079 tags = [str(tag) for tag in test.tags]
1080 test_result = dict()
1082 # Change the TC long name and name if defined in the mapping table
1083 longname = self._mapping.get(longname_orig, None)
1084 if longname is not None:
1085 name = longname.split(u'.')[-1]
1087 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1091 longname = longname_orig
1092 name = test.name.lower()
1094 # Remove TC number from the TC long name (backward compatibility):
1095 self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1096 # Remove TC number from the TC name (not needed):
1097 test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1099 test_result[u"parent"] = test.parent.name.lower()
1100 test_result[u"tags"] = tags
1101 test_result["doc"] = test.doc
1102 test_result[u"type"] = u""
1103 test_result[u"status"] = test.status
1104 test_result[u"starttime"] = test.starttime
1105 test_result[u"endtime"] = test.endtime
1107 if test.status == u"PASS":
1108 if u"NDRPDR" in tags:
1109 if u"TCP_PPS" in tags or u"UDP_PPS" in tags:
1110 test_result[u"msg"] = self._get_data_from_pps_test_msg(
1112 elif u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1113 test_result[u"msg"] = self._get_data_from_cps_test_msg(
1116 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1118 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1119 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1122 test_result[u"msg"] = test.message
1124 test_result[u"msg"] = test.message
1126 if u"PERFTEST" in tags:
1127 # Replace info about cores (e.g. -1c-) with the info about threads
1128 # and cores (e.g. -1t1c-) in the long test case names and in the
1129 # test case names if necessary.
1130 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1134 for tag in test_result[u"tags"]:
1135 groups = re.search(self.REGEX_TC_TAG, tag)
1141 self._test_id = re.sub(
1142 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1143 self._test_id, count=1
1145 test_result[u"name"] = re.sub(
1146 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1147 test_result["name"], count=1
1150 test_result[u"status"] = u"FAIL"
1151 self._data[u"tests"][self._test_id] = test_result
1153 f"The test {self._test_id} has no or more than one "
1154 f"multi-threading tags.\n"
1155 f"Tags: {test_result[u'tags']}"
1159 if u"DEVICETEST" in tags:
1160 test_result[u"type"] = u"DEVICETEST"
1161 elif u"NDRPDR" in tags:
1162 if u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1163 test_result[u"type"] = u"CPS"
1165 test_result[u"type"] = u"NDRPDR"
1166 if test.status == u"PASS":
1167 test_result[u"throughput"], test_result[u"status"] = \
1168 self._get_ndrpdr_throughput(test.message)
1169 test_result[u"gbps"], test_result[u"status"] = \
1170 self._get_ndrpdr_throughput_gbps(test.message)
1171 test_result[u"latency"], test_result[u"status"] = \
1172 self._get_ndrpdr_latency(test.message)
1173 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1175 test_result[u"type"] = u"MRR"
1177 test_result[u"type"] = u"BMRR"
1178 if test.status == u"PASS":
1179 test_result[u"result"] = dict()
1180 groups = re.search(self.REGEX_BMRR, test.message)
1181 if groups is not None:
1182 items_str = groups.group(1)
1184 float(item.strip().replace(u"'", u""))
1185 for item in items_str.split(",")
1187 # Use whole list in CSIT-1180.
1188 stats = jumpavg.AvgStdevStats.for_runs(items_float)
1189 test_result[u"result"][u"samples"] = items_float
1190 test_result[u"result"][u"receive-rate"] = stats.avg
1191 test_result[u"result"][u"receive-stdev"] = stats.stdev
1193 groups = re.search(self.REGEX_MRR, test.message)
1194 test_result[u"result"][u"receive-rate"] = \
1195 float(groups.group(3)) / float(groups.group(1))
1196 elif u"SOAK" in tags:
1197 test_result[u"type"] = u"SOAK"
1198 if test.status == u"PASS":
1199 test_result[u"throughput"], test_result[u"status"] = \
1200 self._get_plr_throughput(test.message)
1201 elif u"HOSTSTACK" in tags:
1202 test_result[u"type"] = u"HOSTSTACK"
1203 if test.status == u"PASS":
1204 test_result[u"result"], test_result[u"status"] = \
1205 self._get_hoststack_data(test.message, tags)
1206 elif u"LDP_NGINX" in tags:
1207 test_result[u"type"] = u"LDP_NGINX"
1208 test_result[u"result"], test_result[u"status"] = \
1209 self._get_vsap_data(test.message, tags)
1210 # elif u"TCP" in tags: # This might be not used
1211 # test_result[u"type"] = u"TCP"
1212 # if test.status == u"PASS":
1213 # groups = re.search(self.REGEX_TCP, test.message)
1214 # test_result[u"result"] = int(groups.group(2))
1215 elif u"RECONF" in tags:
1216 test_result[u"type"] = u"RECONF"
1217 if test.status == u"PASS":
1218 test_result[u"result"] = None
1220 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1221 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1222 test_result[u"result"] = {
1223 u"loss": int(grps_loss.group(1)),
1224 u"time": float(grps_time.group(1))
1226 except (AttributeError, IndexError, ValueError, TypeError):
1227 test_result[u"status"] = u"FAIL"
1229 test_result[u"status"] = u"FAIL"
1231 self._data[u"tests"][self._test_id] = test_result
1233 def end_test(self, test):
1234 """Called when test ends.
1236 :param test: Test to process.
1241 def visit_keyword(self, keyword):
1242 """Implements traversing through the keyword and its child keywords.
1244 :param keyword: Keyword to process.
1245 :type keyword: Keyword
1248 if self.start_keyword(keyword) is not False:
1249 self.end_keyword(keyword)
1251 def start_keyword(self, keyword):
1252 """Called when keyword starts. Default implementation does nothing.
1254 :param keyword: Keyword to process.
1255 :type keyword: Keyword
1259 if keyword.type == u"setup":
1260 self.visit_setup_kw(keyword)
1261 elif keyword.type == u"teardown":
1262 self.visit_teardown_kw(keyword)
1264 self.visit_test_kw(keyword)
1265 except AttributeError:
1268 def end_keyword(self, keyword):
1269 """Called when keyword ends. Default implementation does nothing.
1271 :param keyword: Keyword to process.
1272 :type keyword: Keyword
1276 def visit_test_kw(self, test_kw):
1277 """Implements traversing through the test keyword and its child
1280 :param test_kw: Keyword to process.
1281 :type test_kw: Keyword
1284 for keyword in test_kw.keywords:
1285 if self.start_test_kw(keyword) is not False:
1286 self.visit_test_kw(keyword)
1287 self.end_test_kw(keyword)
1289 def start_test_kw(self, test_kw):
1290 """Called when test keyword starts. Default implementation does
1293 :param test_kw: Keyword to process.
1294 :type test_kw: Keyword
1297 if self._for_output == u"trending":
1300 if test_kw.name.count(u"Run Telemetry On All Duts"):
1301 self._msg_type = u"test-telemetry"
1302 self._telemetry_kw_counter += 1
1303 elif test_kw.name.count(u"Show Runtime On All Duts"):
1304 self._msg_type = u"test-show-runtime"
1305 self._sh_run_counter += 1
1308 test_kw.messages.visit(self)
1310 def end_test_kw(self, test_kw):
1311 """Called when keyword ends. Default implementation does nothing.
1313 :param test_kw: Keyword to process.
1314 :type test_kw: Keyword
1318 def visit_setup_kw(self, setup_kw):
1319 """Implements traversing through the teardown keyword and its child
1322 :param setup_kw: Keyword to process.
1323 :type setup_kw: Keyword
1326 for keyword in setup_kw.keywords:
1327 if self.start_setup_kw(keyword) is not False:
1328 self.visit_setup_kw(keyword)
1329 self.end_setup_kw(keyword)
1331 def start_setup_kw(self, setup_kw):
1332 """Called when teardown keyword starts. Default implementation does
1335 :param setup_kw: Keyword to process.
1336 :type setup_kw: Keyword
1339 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1340 and not self._version:
1341 self._msg_type = u"vpp-version"
1342 elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1344 self._msg_type = u"dpdk-version"
1345 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1346 self._msg_type = u"testbed"
1349 setup_kw.messages.visit(self)
1351 def end_setup_kw(self, setup_kw):
1352 """Called when keyword ends. Default implementation does nothing.
1354 :param setup_kw: Keyword to process.
1355 :type setup_kw: Keyword
1359 def visit_teardown_kw(self, teardown_kw):
1360 """Implements traversing through the teardown keyword and its child
1363 :param teardown_kw: Keyword to process.
1364 :type teardown_kw: Keyword
1367 for keyword in teardown_kw.keywords:
1368 if self.start_teardown_kw(keyword) is not False:
1369 self.visit_teardown_kw(keyword)
1370 self.end_teardown_kw(keyword)
1372 def start_teardown_kw(self, teardown_kw):
1373 """Called when teardown keyword starts
1375 :param teardown_kw: Keyword to process.
1376 :type teardown_kw: Keyword
1379 if teardown_kw.name.count(u"Show Papi History On All Duts"):
1380 self._conf_history_lookup_nr = 0
1381 self._msg_type = u"teardown-papi-history"
1382 teardown_kw.messages.visit(self)
1384 def end_teardown_kw(self, teardown_kw):
1385 """Called when keyword ends. Default implementation does nothing.
1387 :param teardown_kw: Keyword to process.
1388 :type teardown_kw: Keyword
1392 def visit_message(self, msg):
1393 """Implements visiting the message.
1395 :param msg: Message to process.
1399 if self.start_message(msg) is not False:
1400 self.end_message(msg)
1402 def start_message(self, msg):
1403 """Called when message starts. Get required information from messages:
1406 :param msg: Message to process.
1411 self.parse_msg[self._msg_type](msg)
1413 def end_message(self, msg):
1414 """Called when message ends. Default implementation does nothing.
1416 :param msg: Message to process.
1425 The data is extracted from output.xml files generated by Jenkins jobs and
1426 stored in pandas' DataFrames.
1432 (as described in ExecutionChecker documentation)
1434 (as described in ExecutionChecker documentation)
1436 (as described in ExecutionChecker documentation)
1439 def __init__(self, spec, for_output):
1442 :param spec: Specification.
1443 :param for_output: Output to be generated from downloaded data.
1444 :type spec: Specification
1445 :type for_output: str
1451 self._for_output = for_output
1454 self._input_data = pd.Series()
1458 """Getter - Input data.
1460 :returns: Input data
1461 :rtype: pandas.Series
1463 return self._input_data
1465 def metadata(self, job, build):
1466 """Getter - metadata
1468 :param job: Job which metadata we want.
1469 :param build: Build which metadata we want.
1473 :rtype: pandas.Series
1475 return self.data[job][build][u"metadata"]
1477 def suites(self, job, build):
1480 :param job: Job which suites we want.
1481 :param build: Build which suites we want.
1485 :rtype: pandas.Series
1487 return self.data[job][str(build)][u"suites"]
1489 def tests(self, job, build):
1492 :param job: Job which tests we want.
1493 :param build: Build which tests we want.
1497 :rtype: pandas.Series
1499 return self.data[job][build][u"tests"]
1501 def _parse_tests(self, job, build):
1502 """Process data from robot output.xml file and return JSON structured
1505 :param job: The name of job which build output data will be processed.
1506 :param build: The build which output data will be processed.
1509 :returns: JSON data structure.
1518 with open(build[u"file-name"], u'r') as data_file:
1520 result = ExecutionResult(data_file)
1521 except errors.DataError as err:
1523 f"Error occurred while parsing output.xml: {repr(err)}"
1526 checker = ExecutionChecker(
1527 metadata, self._cfg.mapping, self._cfg.ignore, self._for_output
1529 result.visit(checker)
1531 checker.data[u"metadata"][u"tests_total"] = \
1532 result.statistics.total.all.total
1533 checker.data[u"metadata"][u"tests_passed"] = \
1534 result.statistics.total.all.passed
1535 checker.data[u"metadata"][u"tests_failed"] = \
1536 result.statistics.total.all.failed
1537 checker.data[u"metadata"][u"elapsedtime"] = result.suite.elapsedtime
1538 checker.data[u"metadata"][u"generated"] = result.suite.endtime[:14]
1542 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1543 """Download and parse the input data file.
1545 :param pid: PID of the process executing this method.
1546 :param job: Name of the Jenkins job which generated the processed input
1548 :param build: Information about the Jenkins build which generated the
1549 processed input file.
1550 :param repeat: Repeat the download specified number of times if not
1558 logging.info(f"Processing the job/build: {job}: {build[u'build']}")
1565 success = download_and_unzip_data_file(self._cfg, job, build, pid)
1571 f"It is not possible to download the input data file from the "
1572 f"job {job}, build {build[u'build']}, or it is damaged. "
1576 logging.info(f" Processing data from build {build[u'build']}")
1577 data = self._parse_tests(job, build)
1580 f"Input data file from the job {job}, build "
1581 f"{build[u'build']} is damaged. Skipped."
1584 state = u"processed"
1587 remove(build[u"file-name"])
1588 except OSError as err:
1590 f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1593 # If the time-period is defined in the specification file, remove all
1594 # files which are outside the time period.
1596 timeperiod = self._cfg.environment.get(u"time-period", None)
1597 if timeperiod and data:
1599 timeperiod = timedelta(int(timeperiod))
1600 metadata = data.get(u"metadata", None)
1602 generated = metadata.get(u"generated", None)
1604 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1605 if (now - generated) > timeperiod:
1606 # Remove the data and the file:
1611 f" The build {job}/{build[u'build']} is "
1612 f"outdated, will be removed."
1622 def download_and_parse_data(self, repeat=1):
1623 """Download the input data files, parse input data from input files and
1624 store in pandas' Series.
1626 :param repeat: Repeat the download specified number of times if not
1631 logging.info(u"Downloading and parsing input files ...")
1633 for job, builds in self._cfg.input.items():
1634 for build in builds:
1636 result = self._download_and_parse_build(job, build, repeat)
1639 build_nr = result[u"build"][u"build"]
1642 data = result[u"data"]
1643 build_data = pd.Series({
1644 u"metadata": pd.Series(
1645 list(data[u"metadata"].values()),
1646 index=list(data[u"metadata"].keys())
1648 u"suites": pd.Series(
1649 list(data[u"suites"].values()),
1650 index=list(data[u"suites"].keys())
1652 u"tests": pd.Series(
1653 list(data[u"tests"].values()),
1654 index=list(data[u"tests"].keys())
1658 if self._input_data.get(job, None) is None:
1659 self._input_data[job] = pd.Series()
1660 self._input_data[job][str(build_nr)] = build_data
1661 self._cfg.set_input_file_name(
1662 job, build_nr, result[u"build"][u"file-name"]
1664 self._cfg.set_input_state(job, build_nr, result[u"state"])
1667 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1668 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1670 logging.info(u"Done.")
1672 msg = f"Successful downloads from the sources:\n"
1673 for source in self._cfg.environment[u"data-sources"]:
1674 if source[u"successful-downloads"]:
1676 f"{source[u'url']}/{source[u'path']}/"
1677 f"{source[u'file-name']}: "
1678 f"{source[u'successful-downloads']}\n"
1682 def process_local_file(self, local_file, job=u"local", build_nr=1,
1684 """Process local XML file given as a command-line parameter.
1686 :param local_file: The file to process.
1687 :param job: Job name.
1688 :param build_nr: Build number.
1689 :param replace: If True, the information about jobs and builds is
1690 replaced by the new one, otherwise the new jobs and builds are
1692 :type local_file: str
1696 :raises: PresentationError if an error occurs.
1698 if not isfile(local_file):
1699 raise PresentationError(f"The file {local_file} does not exist.")
1702 build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1703 except (IndexError, ValueError):
1708 u"status": u"failed",
1709 u"file-name": local_file
1712 self._cfg.input = dict()
1713 self._cfg.add_build(job, build)
1715 logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1716 data = self._parse_tests(job, build)
1718 raise PresentationError(
1719 f"Error occurred while parsing the file {local_file}"
1722 build_data = pd.Series({
1723 u"metadata": pd.Series(
1724 list(data[u"metadata"].values()),
1725 index=list(data[u"metadata"].keys())
1727 u"suites": pd.Series(
1728 list(data[u"suites"].values()),
1729 index=list(data[u"suites"].keys())
1731 u"tests": pd.Series(
1732 list(data[u"tests"].values()),
1733 index=list(data[u"tests"].keys())
1737 if self._input_data.get(job, None) is None:
1738 self._input_data[job] = pd.Series()
1739 self._input_data[job][str(build_nr)] = build_data
1741 self._cfg.set_input_state(job, build_nr, u"processed")
1743 def process_local_directory(self, local_dir, replace=True):
1744 """Process local directory with XML file(s). The directory is processed
1745 as a 'job' and the XML files in it as builds.
1746 If the given directory contains only sub-directories, these
1747 sub-directories processed as jobs and corresponding XML files as builds
1750 :param local_dir: Local directory to process.
1751 :param replace: If True, the information about jobs and builds is
1752 replaced by the new one, otherwise the new jobs and builds are
1754 :type local_dir: str
1757 if not isdir(local_dir):
1758 raise PresentationError(
1759 f"The directory {local_dir} does not exist."
1762 # Check if the given directory includes only files, or only directories
1763 _, dirnames, filenames = next(walk(local_dir))
1765 if filenames and not dirnames:
1768 # key: dir (job) name, value: list of file names (builds)
1770 local_dir: [join(local_dir, name) for name in filenames]
1773 elif dirnames and not filenames:
1776 # key: dir (job) name, value: list of file names (builds)
1777 local_builds = dict()
1778 for dirname in dirnames:
1780 join(local_dir, dirname, name)
1781 for name in listdir(join(local_dir, dirname))
1782 if isfile(join(local_dir, dirname, name))
1785 local_builds[dirname] = sorted(builds)
1787 elif not filenames and not dirnames:
1788 raise PresentationError(f"The directory {local_dir} is empty.")
1790 raise PresentationError(
1791 f"The directory {local_dir} can include only files or only "
1792 f"directories, not both.\nThe directory {local_dir} includes "
1793 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1797 self._cfg.input = dict()
1799 for job, files in local_builds.items():
1800 for idx, local_file in enumerate(files):
1801 self.process_local_file(local_file, job, idx + 1, replace=False)
1804 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1805 """Return the index of character in the string which is the end of tag.
1807 :param tag_filter: The string where the end of tag is being searched.
1808 :param start: The index where the searching is stated.
1809 :param closer: The character which is the tag closer.
1810 :type tag_filter: str
1813 :returns: The index of the tag closer.
1817 idx_opener = tag_filter.index(closer, start)
1818 return tag_filter.index(closer, idx_opener + 1)
1823 def _condition(tag_filter):
1824 """Create a conditional statement from the given tag filter.
1826 :param tag_filter: Filter based on tags from the element specification.
1827 :type tag_filter: str
1828 :returns: Conditional statement which can be evaluated.
1833 index = InputData._end_of_tag(tag_filter, index)
1837 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1839 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1840 continue_on_error=False):
1841 """Filter required data from the given jobs and builds.
1843 The output data structure is:
1846 - test (or suite) 1 ID:
1852 - test (or suite) n ID:
1859 :param element: Element which will use the filtered data.
1860 :param params: Parameters which will be included in the output. If None,
1861 all parameters are included.
1862 :param data: If not None, this data is used instead of data specified
1864 :param data_set: The set of data to be filtered: tests, suites,
1866 :param continue_on_error: Continue if there is error while reading the
1867 data. The Item will be empty then
1868 :type element: pandas.Series
1872 :type continue_on_error: bool
1873 :returns: Filtered data.
1874 :rtype pandas.Series
1878 if data_set == "suites":
1880 elif element[u"filter"] in (u"all", u"template"):
1883 cond = InputData._condition(element[u"filter"])
1884 logging.debug(f" Filter: {cond}")
1886 logging.error(u" No filter defined.")
1890 params = element.get(u"parameters", None)
1892 params.extend((u"type", u"status"))
1894 data_to_filter = data if data else element[u"data"]
1897 for job, builds in data_to_filter.items():
1898 data[job] = pd.Series()
1899 for build in builds:
1900 data[job][str(build)] = pd.Series()
1903 self.data[job][str(build)][data_set].items())
1905 if continue_on_error:
1909 for test_id, test_data in data_dict.items():
1910 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1911 data[job][str(build)][test_id] = pd.Series()
1913 for param, val in test_data.items():
1914 data[job][str(build)][test_id][param] = val
1916 for param in params:
1918 data[job][str(build)][test_id][param] =\
1921 data[job][str(build)][test_id][param] =\
1925 except (KeyError, IndexError, ValueError) as err:
1927 f"Missing mandatory parameter in the element specification: "
1931 except AttributeError as err:
1932 logging.error(repr(err))
1934 except SyntaxError as err:
1936 f"The filter {cond} is not correct. Check if all tags are "
1937 f"enclosed by apostrophes.\n{repr(err)}"
1941 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1942 continue_on_error=False):
1943 """Filter required data from the given jobs and builds.
1945 The output data structure is:
1948 - test (or suite) 1 ID:
1954 - test (or suite) n ID:
1961 :param element: Element which will use the filtered data.
1962 :param params: Parameters which will be included in the output. If None,
1963 all parameters are included.
1964 :param data_set: The set of data to be filtered: tests, suites,
1966 :param continue_on_error: Continue if there is error while reading the
1967 data. The Item will be empty then
1968 :type element: pandas.Series
1971 :type continue_on_error: bool
1972 :returns: Filtered data.
1973 :rtype pandas.Series
1976 include = element.get(u"include", None)
1978 logging.warning(u"No tests to include, skipping the element.")
1982 params = element.get(u"parameters", None)
1983 if params and u"type" not in params:
1984 params.append(u"type")
1986 cores = element.get(u"core", None)
1990 for test in include:
1991 tests.append(test.format(core=core))
1997 for job, builds in element[u"data"].items():
1998 data[job] = pd.Series()
1999 for build in builds:
2000 data[job][str(build)] = pd.Series()
2003 reg_ex = re.compile(str(test).lower())
2004 for test_id in self.data[job][
2005 str(build)][data_set].keys():
2006 if re.match(reg_ex, str(test_id).lower()):
2007 test_data = self.data[job][
2008 str(build)][data_set][test_id]
2009 data[job][str(build)][test_id] = pd.Series()
2011 for param, val in test_data.items():
2012 data[job][str(build)][test_id]\
2015 for param in params:
2017 data[job][str(build)][
2021 data[job][str(build)][
2022 test_id][param] = u"No Data"
2023 except KeyError as err:
2024 if continue_on_error:
2025 logging.debug(repr(err))
2027 logging.error(repr(err))
2031 except (KeyError, IndexError, ValueError) as err:
2033 f"Missing mandatory parameter in the element "
2034 f"specification: {repr(err)}"
2037 except AttributeError as err:
2038 logging.error(repr(err))
2042 def merge_data(data):
2043 """Merge data from more jobs and builds to a simple data structure.
2045 The output data structure is:
2047 - test (suite) 1 ID:
2053 - test (suite) n ID:
2056 :param data: Data to merge.
2057 :type data: pandas.Series
2058 :returns: Merged data.
2059 :rtype: pandas.Series
2062 logging.info(u" Merging data ...")
2064 merged_data = pd.Series()
2065 for builds in data.values:
2066 for item in builds.values:
2067 for item_id, item_data in item.items():
2068 merged_data[item_id] = item_data
2071 def print_all_oper_data(self):
2072 """Print all operational data to console.
2075 for job in self._input_data.values:
2076 for build in job.values:
2077 for test_id, test_data in build[u"tests"].items():
2079 if test_data.get(u"show-run", None) is None:
2081 for dut_name, data in test_data[u"show-run"].items():
2082 if data.get(u"runtime", None) is None:
2084 runtime = loads(data[u"runtime"])
2086 threads_nr = len(runtime[0][u"clocks"])
2087 except (IndexError, KeyError):
2089 threads = OrderedDict(
2090 {idx: list() for idx in range(threads_nr)})
2091 for item in runtime:
2092 for idx in range(threads_nr):
2093 if item[u"vectors"][idx] > 0:
2094 clocks = item[u"clocks"][idx] / \
2095 item[u"vectors"][idx]
2096 elif item[u"calls"][idx] > 0:
2097 clocks = item[u"clocks"][idx] / \
2099 elif item[u"suspends"][idx] > 0:
2100 clocks = item[u"clocks"][idx] / \
2101 item[u"suspends"][idx]
2105 if item[u"calls"][idx] > 0:
2106 vectors_call = item[u"vectors"][idx] / \
2111 if int(item[u"calls"][idx]) + int(
2112 item[u"vectors"][idx]) + \
2113 int(item[u"suspends"][idx]):
2114 threads[idx].append([
2116 item[u"calls"][idx],
2117 item[u"vectors"][idx],
2118 item[u"suspends"][idx],
2123 print(f"Host IP: {data.get(u'host', '')}, "
2124 f"Socket: {data.get(u'socket', '')}")
2125 for thread_nr, thread in threads.items():
2126 txt_table = prettytable.PrettyTable(
2132 u"Cycles per Packet",
2133 u"Average Vector Size"
2138 txt_table.add_row(row)
2140 if len(thread) == 0:
2143 avg = f", Average Vector Size per Node: " \
2144 f"{(avg / len(thread)):.2f}"
2145 th_name = u"main" if thread_nr == 0 \
2146 else f"worker_{thread_nr}"
2147 print(f"{dut_name}, {th_name}{avg}")
2148 txt_table.float_format = u".2"
2149 txt_table.align = u"r"
2150 txt_table.align[u"Name"] = u"l"
2151 print(f"{txt_table.get_string()}\n")