1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
28 from os import remove, walk, listdir
29 from os.path import isfile, isdir, join
30 from datetime import datetime as dt
31 from datetime import timedelta
32 from json import loads
33 from json.decoder import JSONDecodeError
40 from robot.api import ExecutionResult, ResultVisitor
41 from robot import errors
43 from resources.libraries.python import jumpavg
44 from input_data_files import download_and_unzip_data_file
45 from pal_errors import PresentationError
48 # Separator used in file names
52 class ExecutionChecker(ResultVisitor):
53 """Class to traverse through the test suite structure.
55 The functionality implemented in this class generates a json structure:
61 "generated": "Timestamp",
62 "version": "SUT version",
63 "job": "Jenkins job name",
64 "build": "Information about the build"
67 "Suite long name 1": {
69 "doc": "Suite 1 documentation",
70 "parent": "Suite 1 parent",
71 "level": "Level of the suite in the suite hierarchy"
73 "Suite long name N": {
75 "doc": "Suite N documentation",
76 "parent": "Suite 2 parent",
77 "level": "Level of the suite in the suite hierarchy"
84 "parent": "Name of the parent of the test",
85 "doc": "Test documentation",
86 "msg": "Test message",
87 "conf-history": "DUT1 and DUT2 VAT History",
88 "show-run": "Show Run",
89 "tags": ["tag 1", "tag 2", "tag n"],
91 "status": "PASS" | "FAIL",
137 "parent": "Name of the parent of the test",
138 "doc": "Test documentation",
139 "msg": "Test message",
140 "tags": ["tag 1", "tag 2", "tag n"],
142 "status": "PASS" | "FAIL",
149 "parent": "Name of the parent of the test",
150 "doc": "Test documentation",
151 "msg": "Test message",
152 "tags": ["tag 1", "tag 2", "tag n"],
153 "type": "MRR" | "BMRR",
154 "status": "PASS" | "FAIL",
156 "receive-rate": float,
157 # Average of a list, computed using AvgStdevStats.
158 # In CSIT-1180, replace with List[float].
172 "metadata": { # Optional
173 "version": "VPP version",
174 "job": "Jenkins job name",
175 "build": "Information about the build"
179 "doc": "Suite 1 documentation",
180 "parent": "Suite 1 parent",
181 "level": "Level of the suite in the suite hierarchy"
184 "doc": "Suite N documentation",
185 "parent": "Suite 2 parent",
186 "level": "Level of the suite in the suite hierarchy"
192 "parent": "Name of the parent of the test",
193 "doc": "Test documentation"
194 "msg": "Test message"
195 "tags": ["tag 1", "tag 2", "tag n"],
196 "conf-history": "DUT1 and DUT2 VAT History"
197 "show-run": "Show Run"
198 "status": "PASS" | "FAIL"
206 .. note:: ID is the lowercase full path to the test.
209 REGEX_PLR_RATE = re.compile(
210 r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
211 r'PLRsearch upper bound::?\s(\d+.\d+)'
213 REGEX_NDRPDR_RATE = re.compile(
214 r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
215 r'NDR_UPPER:\s(\d+.\d+).*\n'
216 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
217 r'PDR_UPPER:\s(\d+.\d+)'
219 REGEX_NDRPDR_GBPS = re.compile(
220 r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
221 r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
222 r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
223 r'PDR_UPPER:.*,\s(\d+.\d+)'
225 REGEX_PERF_MSG_INFO = re.compile(
226 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
227 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
228 r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
229 r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
230 r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
232 REGEX_CPS_MSG_INFO = re.compile(
233 r'NDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*\n'
234 r'PDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*'
236 REGEX_PPS_MSG_INFO = re.compile(
237 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
238 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*'
240 REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
242 REGEX_VSAP_MSG_INFO = re.compile(
243 r'Transfer Rate: (\d*.\d*).*\n'
244 r'Latency: (\d*.\d*).*\n'
245 r'Completed requests: (\d*).*\n'
246 r'Failed requests: (\d*).*\n'
247 r'Total data transferred: (\d*).*\n'
248 r'Connection [cr]ps rate:\s*(\d*.\d*)'
251 # Needed for CPS and PPS tests
252 REGEX_NDRPDR_LAT_BASE = re.compile(
253 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
254 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
256 REGEX_NDRPDR_LAT = re.compile(
257 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
258 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
259 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
260 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
261 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
262 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
265 REGEX_VERSION_VPP = re.compile(
266 r"(return STDOUT Version:\s*|"
267 r"VPP Version:\s*|VPP version:\s*)(.*)"
269 REGEX_VERSION_DPDK = re.compile(
270 r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
272 REGEX_TCP = re.compile(
273 r'Total\s(rps|cps|throughput):\s(\d*).*$'
275 REGEX_MRR = re.compile(
276 r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
277 r'tx\s(\d*),\srx\s(\d*)'
279 REGEX_BMRR = re.compile(
280 r'.*trial results.*: \[(.*)\]'
282 REGEX_RECONF_LOSS = re.compile(
283 r'Packets lost due to reconfig: (\d*)'
285 REGEX_RECONF_TIME = re.compile(
286 r'Implied time lost: (\d*.[\de-]*)'
288 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
290 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
292 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
294 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
296 REGEX_SH_RUN_HOST = re.compile(
297 r'hostname=\"(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})\",hook=\"(.*)\"'
300 def __init__(self, metadata, mapping, ignore, for_output):
303 :param metadata: Key-value pairs to be included in "metadata" part of
305 :param mapping: Mapping of the old names of test cases to the new
307 :param ignore: List of TCs to be ignored.
308 :param for_output: Output to be generated from downloaded data.
312 :type for_output: str
315 # Type of message to parse out from the test messages
316 self._msg_type = None
322 self._timestamp = None
324 # Testbed. The testbed is identified by TG node IP address.
327 # Mapping of TCs long names
328 self._mapping = mapping
331 self._ignore = ignore
333 self._for_output = for_output
335 # Number of PAPI History messages found:
337 # 1 - PAPI History of DUT1
338 # 2 - PAPI History of DUT2
339 self._conf_history_lookup_nr = 0
341 self._sh_run_counter = 0
342 self._telemetry_kw_counter = 0
343 self._telemetry_msg_counter = 0
345 # Test ID of currently processed test- the lowercase full path to the
349 # The main data structure
351 u"metadata": OrderedDict(),
352 u"suites": OrderedDict(),
353 u"tests": OrderedDict()
356 # Save the provided metadata
357 for key, val in metadata.items():
358 self._data[u"metadata"][key] = val
360 # Dictionary defining the methods used to parse different types of
363 u"vpp-version": self._get_vpp_version,
364 u"dpdk-version": self._get_dpdk_version,
365 u"teardown-papi-history": self._get_papi_history,
366 u"test-show-runtime": self._get_show_run,
367 u"testbed": self._get_testbed,
368 u"test-telemetry": self._get_telemetry
373 """Getter - Data parsed from the XML file.
375 :returns: Data parsed from the XML file.
380 def _get_data_from_mrr_test_msg(self, msg):
381 """Get info from message of MRR performance tests.
383 :param msg: Message to be processed.
385 :returns: Processed message or original message if a problem occurs.
389 groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
390 if not groups or groups.lastindex != 1:
391 return u"Test Failed."
394 data = groups.group(1).split(u", ")
395 except (AttributeError, IndexError, ValueError, KeyError):
396 return u"Test Failed."
401 out_str += f"{(float(item) / 1e6):.2f}, "
402 return out_str[:-2] + u"]"
403 except (AttributeError, IndexError, ValueError, KeyError):
404 return u"Test Failed."
406 def _get_data_from_cps_test_msg(self, msg):
407 """Get info from message of NDRPDR CPS tests.
409 :param msg: Message to be processed.
411 :returns: Processed message or "Test Failed." if a problem occurs.
415 groups = re.search(self.REGEX_CPS_MSG_INFO, msg)
416 if not groups or groups.lastindex != 2:
417 return u"Test Failed."
421 f"1. {(float(groups.group(1)) / 1e6):5.2f}\n"
422 f"2. {(float(groups.group(2)) / 1e6):5.2f}"
424 except (AttributeError, IndexError, ValueError, KeyError):
425 return u"Test Failed."
427 def _get_data_from_pps_test_msg(self, msg):
428 """Get info from message of NDRPDR PPS tests.
430 :param msg: Message to be processed.
432 :returns: Processed message or "Test Failed." if a problem occurs.
436 groups = re.search(self.REGEX_PPS_MSG_INFO, msg)
437 if not groups or groups.lastindex != 4:
438 return u"Test Failed."
442 f"1. {(float(groups.group(1)) / 1e6):5.2f} "
443 f"{float(groups.group(2)):5.2f}\n"
444 f"2. {(float(groups.group(3)) / 1e6):5.2f} "
445 f"{float(groups.group(4)):5.2f}"
447 except (AttributeError, IndexError, ValueError, KeyError):
448 return u"Test Failed."
450 def _get_data_from_perf_test_msg(self, msg):
451 """Get info from message of NDRPDR performance tests.
453 :param msg: Message to be processed.
455 :returns: Processed message or "Test Failed." if a problem occurs.
459 groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
460 if not groups or groups.lastindex != 10:
461 return u"Test Failed."
465 u"ndr_low": float(groups.group(1)),
466 u"ndr_low_b": float(groups.group(2)),
467 u"pdr_low": float(groups.group(3)),
468 u"pdr_low_b": float(groups.group(4)),
469 u"pdr_lat_90_1": groups.group(5),
470 u"pdr_lat_90_2": groups.group(6),
471 u"pdr_lat_50_1": groups.group(7),
472 u"pdr_lat_50_2": groups.group(8),
473 u"pdr_lat_10_1": groups.group(9),
474 u"pdr_lat_10_2": groups.group(10),
476 except (AttributeError, IndexError, ValueError, KeyError):
477 return u"Test Failed."
479 def _process_lat(in_str_1, in_str_2):
480 """Extract min, avg, max values from latency string.
482 :param in_str_1: Latency string for one direction produced by robot
484 :param in_str_2: Latency string for second direction produced by
488 :returns: Processed latency string or None if a problem occurs.
491 in_list_1 = in_str_1.split('/', 3)
492 in_list_2 = in_str_2.split('/', 3)
494 if len(in_list_1) != 4 and len(in_list_2) != 4:
497 in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
499 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
500 except hdrh.codec.HdrLengthException:
503 in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
505 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
506 except hdrh.codec.HdrLengthException:
509 if hdr_lat_1 and hdr_lat_2:
511 hdr_lat_1.get_value_at_percentile(50.0),
512 hdr_lat_1.get_value_at_percentile(90.0),
513 hdr_lat_1.get_value_at_percentile(99.0),
514 hdr_lat_2.get_value_at_percentile(50.0),
515 hdr_lat_2.get_value_at_percentile(90.0),
516 hdr_lat_2.get_value_at_percentile(99.0)
526 f"1. {(data[u'ndr_low'] / 1e6):5.2f} "
527 f"{data[u'ndr_low_b']:5.2f}"
528 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f} "
529 f"{data[u'pdr_low_b']:5.2f}"
532 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
533 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
534 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
537 max_len = len(str(max((max(item) for item in latency))))
538 max_len = 4 if max_len < 4 else max_len
540 for idx, lat in enumerate(latency):
545 f"{lat[0]:{max_len}d} "
546 f"{lat[1]:{max_len}d} "
547 f"{lat[2]:{max_len}d} "
548 f"{lat[3]:{max_len}d} "
549 f"{lat[4]:{max_len}d} "
550 f"{lat[5]:{max_len}d} "
555 except (AttributeError, IndexError, ValueError, KeyError):
556 return u"Test Failed."
558 def _get_testbed(self, msg):
559 """Called when extraction of testbed IP is required.
560 The testbed is identified by TG node IP address.
562 :param msg: Message to process.
567 if msg.message.count(u"Setup of TG node") or \
568 msg.message.count(u"Setup of node TG host"):
569 reg_tg_ip = re.compile(
570 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
572 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
573 except (KeyError, ValueError, IndexError, AttributeError):
576 self._data[u"metadata"][u"testbed"] = self._testbed
577 self._msg_type = None
579 def _get_vpp_version(self, msg):
580 """Called when extraction of VPP version is required.
582 :param msg: Message to process.
587 if msg.message.count(u"return STDOUT Version:") or \
588 msg.message.count(u"VPP Version:") or \
589 msg.message.count(u"VPP version:"):
591 re.search(self.REGEX_VERSION_VPP, msg.message).group(2)
593 self._data[u"metadata"][u"version"] = self._version
594 self._msg_type = None
596 def _get_dpdk_version(self, msg):
597 """Called when extraction of DPDK version is required.
599 :param msg: Message to process.
604 if msg.message.count(u"DPDK Version:"):
606 self._version = str(re.search(
607 self.REGEX_VERSION_DPDK, msg.message).group(2))
608 self._data[u"metadata"][u"version"] = self._version
612 self._msg_type = None
614 def _get_papi_history(self, msg):
615 """Called when extraction of PAPI command history is required.
617 :param msg: Message to process.
621 if msg.message.count(u"PAPI command history:"):
622 self._conf_history_lookup_nr += 1
623 if self._conf_history_lookup_nr == 1:
624 self._data[u"tests"][self._test_id][u"conf-history"] = str()
626 self._msg_type = None
628 r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} PAPI command history:",
632 ).replace(u'"', u"'")
633 self._data[u"tests"][self._test_id][u"conf-history"] += (
634 f"**DUT{str(self._conf_history_lookup_nr)}:** {text}"
637 def _get_show_run(self, msg):
638 """Called when extraction of VPP operational data (output of CLI command
639 Show Runtime) is required.
641 :param msg: Message to process.
646 if not msg.message.count(u"stats runtime"):
650 if self._sh_run_counter > 1:
653 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
654 self._data[u"tests"][self._test_id][u"show-run"] = dict()
656 groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
660 host = groups.group(1)
661 except (AttributeError, IndexError):
664 sock = groups.group(2)
665 except (AttributeError, IndexError):
668 dut = u"dut{nr}".format(
669 nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
671 self._data[u'tests'][self._test_id][u'show-run'][dut] = \
676 u"runtime": str(msg.message).replace(u' ', u'').
677 replace(u'\n', u'').replace(u"'", u'"').
678 replace(u'b"', u'"').replace(u'u"', u'"').
683 def _get_telemetry(self, msg):
684 """Called when extraction of VPP telemetry data is required.
686 :param msg: Message to process.
691 if self._telemetry_kw_counter > 1:
693 if not msg.message.count(u"# TYPE vpp_runtime_calls"):
696 if u"telemetry-show-run" not in \
697 self._data[u"tests"][self._test_id].keys():
698 self._data[u"tests"][self._test_id][u"telemetry-show-run"] = dict()
700 self._telemetry_msg_counter += 1
701 groups = re.search(self.REGEX_SH_RUN_HOST, msg.message)
705 host = groups.group(1)
706 except (AttributeError, IndexError):
709 sock = groups.group(2)
710 except (AttributeError, IndexError):
713 u"source_type": u"node",
715 u"msg_type": u"metric",
716 u"log_level": u"INFO",
717 u"timestamp": msg.timestamp,
718 u"msg": u"show_runtime",
723 for line in msg.message.splitlines():
724 if not line.startswith(u"vpp_runtime_"):
727 params, value, timestamp = line.rsplit(u" ", maxsplit=2)
728 cut = params.index(u"{")
729 name = params[:cut].split(u"_", maxsplit=2)[-1]
731 u"dict" + params[cut:].replace('{', '(').replace('}', ')')
733 labels[u"graph_node"] = labels.pop(u"name")
734 runtime[u"data"].append(
738 u"timestamp": timestamp,
742 except (TypeError, ValueError, IndexError):
744 self._data[u'tests'][self._test_id][u'telemetry-show-run']\
745 [f"dut{self._telemetry_msg_counter}"] = copy.copy(
753 def _get_ndrpdr_throughput(self, msg):
754 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
757 :param msg: The test message to be parsed.
759 :returns: Parsed data as a dict and the status (PASS/FAIL).
760 :rtype: tuple(dict, str)
764 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
765 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
768 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
770 if groups is not None:
772 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
773 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
774 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
775 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
777 except (IndexError, ValueError):
780 return throughput, status
782 def _get_ndrpdr_throughput_gbps(self, msg):
783 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
786 :param msg: The test message to be parsed.
788 :returns: Parsed data as a dict and the status (PASS/FAIL).
789 :rtype: tuple(dict, str)
793 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
794 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
797 groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
799 if groups is not None:
801 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
802 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
803 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
804 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
806 except (IndexError, ValueError):
811 def _get_plr_throughput(self, msg):
812 """Get PLRsearch lower bound and PLRsearch upper bound from the test
815 :param msg: The test message to be parsed.
817 :returns: Parsed data as a dict and the status (PASS/FAIL).
818 :rtype: tuple(dict, str)
826 groups = re.search(self.REGEX_PLR_RATE, msg)
828 if groups is not None:
830 throughput[u"LOWER"] = float(groups.group(1))
831 throughput[u"UPPER"] = float(groups.group(2))
833 except (IndexError, ValueError):
836 return throughput, status
838 def _get_ndrpdr_latency(self, msg):
839 """Get LATENCY from the test message.
841 :param msg: The test message to be parsed.
843 :returns: Parsed data as a dict and the status (PASS/FAIL).
844 :rtype: tuple(dict, str)
854 u"direction1": copy.copy(latency_default),
855 u"direction2": copy.copy(latency_default)
858 u"direction1": copy.copy(latency_default),
859 u"direction2": copy.copy(latency_default)
862 u"direction1": copy.copy(latency_default),
863 u"direction2": copy.copy(latency_default)
866 u"direction1": copy.copy(latency_default),
867 u"direction2": copy.copy(latency_default)
870 u"direction1": copy.copy(latency_default),
871 u"direction2": copy.copy(latency_default)
874 u"direction1": copy.copy(latency_default),
875 u"direction2": copy.copy(latency_default)
879 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
881 groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
883 return latency, u"FAIL"
885 def process_latency(in_str):
886 """Return object with parsed latency values.
888 TODO: Define class for the return type.
890 :param in_str: Input string, min/avg/max/hdrh format.
892 :returns: Dict with corresponding keys, except hdrh float values.
894 :throws IndexError: If in_str does not have enough substrings.
895 :throws ValueError: If a substring does not convert to float.
897 in_list = in_str.split('/', 3)
900 u"min": float(in_list[0]),
901 u"avg": float(in_list[1]),
902 u"max": float(in_list[2]),
906 if len(in_list) == 4:
907 rval[u"hdrh"] = str(in_list[3])
912 latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
913 latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
914 latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
915 latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
916 if groups.lastindex == 4:
917 return latency, u"PASS"
918 except (IndexError, ValueError):
922 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
923 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
924 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
925 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
926 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
927 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
928 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
929 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
930 if groups.lastindex == 12:
931 return latency, u"PASS"
932 except (IndexError, ValueError):
935 return latency, u"FAIL"
938 def _get_hoststack_data(msg, tags):
939 """Get data from the hoststack test message.
941 :param msg: The test message to be parsed.
942 :param tags: Test tags.
945 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
946 :rtype: tuple(dict, str)
951 msg = msg.replace(u"'", u'"').replace(u" ", u"")
952 if u"LDPRELOAD" in tags:
956 except JSONDecodeError:
958 elif u"VPPECHO" in tags:
960 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
962 client=loads(msg_lst[0]),
963 server=loads(msg_lst[1])
966 except (JSONDecodeError, IndexError):
969 return result, status
971 def _get_vsap_data(self, msg, tags):
972 """Get data from the vsap test message.
974 :param msg: The test message to be parsed.
975 :param tags: Test tags.
978 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
979 :rtype: tuple(dict, str)
984 groups = re.search(self.REGEX_VSAP_MSG_INFO, msg)
985 if groups is not None:
987 result[u"transfer-rate"] = float(groups.group(1)) * 1e3
988 result[u"latency"] = float(groups.group(2))
989 result[u"completed-requests"] = int(groups.group(3))
990 result[u"failed-requests"] = int(groups.group(4))
991 result[u"bytes-transferred"] = int(groups.group(5))
992 if u"TCP_CPS"in tags:
993 result[u"cps"] = float(groups.group(6))
994 elif u"TCP_RPS" in tags:
995 result[u"rps"] = float(groups.group(6))
997 return result, status
999 except (IndexError, ValueError):
1002 return result, status
1004 def visit_suite(self, suite):
1005 """Implements traversing through the suite and its direct children.
1007 :param suite: Suite to process.
1011 if self.start_suite(suite) is not False:
1012 suite.suites.visit(self)
1013 suite.tests.visit(self)
1014 self.end_suite(suite)
1016 def start_suite(self, suite):
1017 """Called when suite starts.
1019 :param suite: Suite to process.
1025 parent_name = suite.parent.name
1026 except AttributeError:
1029 self._data[u"suites"][suite.longname.lower().
1030 replace(u'"', u"'").
1031 replace(u" ", u"_")] = {
1032 u"name": suite.name.lower(),
1034 u"parent": parent_name,
1035 u"level": len(suite.longname.split(u"."))
1038 suite.keywords.visit(self)
1040 def end_suite(self, suite):
1041 """Called when suite ends.
1043 :param suite: Suite to process.
1048 def visit_test(self, test):
1049 """Implements traversing through the test.
1051 :param test: Test to process.
1055 if self.start_test(test) is not False:
1056 test.keywords.visit(self)
1059 def start_test(self, test):
1060 """Called when test starts.
1062 :param test: Test to process.
1067 self._sh_run_counter = 0
1068 self._telemetry_kw_counter = 0
1069 self._telemetry_msg_counter = 0
1071 longname_orig = test.longname.lower()
1073 # Check the ignore list
1074 if longname_orig in self._ignore:
1077 tags = [str(tag) for tag in test.tags]
1078 test_result = dict()
1080 # Change the TC long name and name if defined in the mapping table
1081 longname = self._mapping.get(longname_orig, None)
1082 if longname is not None:
1083 name = longname.split(u'.')[-1]
1085 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1089 longname = longname_orig
1090 name = test.name.lower()
1092 # Remove TC number from the TC long name (backward compatibility):
1093 self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1094 # Remove TC number from the TC name (not needed):
1095 test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1097 test_result[u"parent"] = test.parent.name.lower()
1098 test_result[u"tags"] = tags
1099 test_result["doc"] = test.doc
1100 test_result[u"type"] = u""
1101 test_result[u"status"] = test.status
1102 test_result[u"starttime"] = test.starttime
1103 test_result[u"endtime"] = test.endtime
1105 if test.status == u"PASS":
1106 if u"NDRPDR" in tags:
1107 if u"TCP_PPS" in tags or u"UDP_PPS" in tags:
1108 test_result[u"msg"] = self._get_data_from_pps_test_msg(
1110 elif u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1111 test_result[u"msg"] = self._get_data_from_cps_test_msg(
1114 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1116 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1117 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1120 test_result[u"msg"] = test.message
1122 test_result[u"msg"] = test.message
1124 if u"PERFTEST" in tags and u"TREX" not in tags:
1125 # Replace info about cores (e.g. -1c-) with the info about threads
1126 # and cores (e.g. -1t1c-) in the long test case names and in the
1127 # test case names if necessary.
1130 for tag in test_result[u"tags"]:
1131 groups = re.search(self.REGEX_TC_TAG, tag)
1137 self._test_id = re.sub(
1138 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1139 self._test_id, count=1
1141 test_result[u"name"] = re.sub(
1142 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1143 test_result["name"], count=1
1146 test_result[u"status"] = u"FAIL"
1147 self._data[u"tests"][self._test_id] = test_result
1149 f"The test {self._test_id} has no or more than one "
1150 f"multi-threading tags.\n"
1151 f"Tags: {test_result[u'tags']}"
1155 if u"DEVICETEST" in tags:
1156 test_result[u"type"] = u"DEVICETEST"
1157 elif u"NDRPDR" in tags:
1158 if u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1159 test_result[u"type"] = u"CPS"
1161 test_result[u"type"] = u"NDRPDR"
1162 if test.status == u"PASS":
1163 test_result[u"throughput"], test_result[u"status"] = \
1164 self._get_ndrpdr_throughput(test.message)
1165 test_result[u"gbps"], test_result[u"status"] = \
1166 self._get_ndrpdr_throughput_gbps(test.message)
1167 test_result[u"latency"], test_result[u"status"] = \
1168 self._get_ndrpdr_latency(test.message)
1169 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1171 test_result[u"type"] = u"MRR"
1173 test_result[u"type"] = u"BMRR"
1174 if test.status == u"PASS":
1175 test_result[u"result"] = dict()
1176 groups = re.search(self.REGEX_BMRR, test.message)
1177 if groups is not None:
1178 items_str = groups.group(1)
1180 float(item.strip().replace(u"'", u""))
1181 for item in items_str.split(",")
1183 # Use whole list in CSIT-1180.
1184 stats = jumpavg.AvgStdevStats.for_runs(items_float)
1185 test_result[u"result"][u"samples"] = items_float
1186 test_result[u"result"][u"receive-rate"] = stats.avg
1187 test_result[u"result"][u"receive-stdev"] = stats.stdev
1189 groups = re.search(self.REGEX_MRR, test.message)
1190 test_result[u"result"][u"receive-rate"] = \
1191 float(groups.group(3)) / float(groups.group(1))
1192 elif u"SOAK" in tags:
1193 test_result[u"type"] = u"SOAK"
1194 if test.status == u"PASS":
1195 test_result[u"throughput"], test_result[u"status"] = \
1196 self._get_plr_throughput(test.message)
1197 elif u"HOSTSTACK" in tags:
1198 test_result[u"type"] = u"HOSTSTACK"
1199 if test.status == u"PASS":
1200 test_result[u"result"], test_result[u"status"] = \
1201 self._get_hoststack_data(test.message, tags)
1202 elif u"LDP_NGINX" in tags:
1203 test_result[u"type"] = u"LDP_NGINX"
1204 test_result[u"result"], test_result[u"status"] = \
1205 self._get_vsap_data(test.message, tags)
1206 # elif u"TCP" in tags: # This might be not used
1207 # test_result[u"type"] = u"TCP"
1208 # if test.status == u"PASS":
1209 # groups = re.search(self.REGEX_TCP, test.message)
1210 # test_result[u"result"] = int(groups.group(2))
1211 elif u"RECONF" in tags:
1212 test_result[u"type"] = u"RECONF"
1213 if test.status == u"PASS":
1214 test_result[u"result"] = None
1216 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1217 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1218 test_result[u"result"] = {
1219 u"loss": int(grps_loss.group(1)),
1220 u"time": float(grps_time.group(1))
1222 except (AttributeError, IndexError, ValueError, TypeError):
1223 test_result[u"status"] = u"FAIL"
1225 test_result[u"status"] = u"FAIL"
1227 self._data[u"tests"][self._test_id] = test_result
1229 def end_test(self, test):
1230 """Called when test ends.
1232 :param test: Test to process.
1237 def visit_keyword(self, keyword):
1238 """Implements traversing through the keyword and its child keywords.
1240 :param keyword: Keyword to process.
1241 :type keyword: Keyword
1244 if self.start_keyword(keyword) is not False:
1245 self.end_keyword(keyword)
1247 def start_keyword(self, keyword):
1248 """Called when keyword starts. Default implementation does nothing.
1250 :param keyword: Keyword to process.
1251 :type keyword: Keyword
1255 if keyword.type == u"setup":
1256 self.visit_setup_kw(keyword)
1257 elif keyword.type == u"teardown":
1258 self.visit_teardown_kw(keyword)
1260 self.visit_test_kw(keyword)
1261 except AttributeError:
1264 def end_keyword(self, keyword):
1265 """Called when keyword ends. Default implementation does nothing.
1267 :param keyword: Keyword to process.
1268 :type keyword: Keyword
1272 def visit_test_kw(self, test_kw):
1273 """Implements traversing through the test keyword and its child
1276 :param test_kw: Keyword to process.
1277 :type test_kw: Keyword
1280 for keyword in test_kw.keywords:
1281 if self.start_test_kw(keyword) is not False:
1282 self.visit_test_kw(keyword)
1283 self.end_test_kw(keyword)
1285 def start_test_kw(self, test_kw):
1286 """Called when test keyword starts. Default implementation does
1289 :param test_kw: Keyword to process.
1290 :type test_kw: Keyword
1293 if self._for_output == u"trending":
1296 if test_kw.name.count(u"Run Telemetry On All Duts"):
1297 self._msg_type = u"test-telemetry"
1298 self._telemetry_kw_counter += 1
1299 elif test_kw.name.count(u"Show Runtime On All Duts"):
1300 self._msg_type = u"test-show-runtime"
1301 self._sh_run_counter += 1
1304 test_kw.messages.visit(self)
1306 def end_test_kw(self, test_kw):
1307 """Called when keyword ends. Default implementation does nothing.
1309 :param test_kw: Keyword to process.
1310 :type test_kw: Keyword
1314 def visit_setup_kw(self, setup_kw):
1315 """Implements traversing through the teardown keyword and its child
1318 :param setup_kw: Keyword to process.
1319 :type setup_kw: Keyword
1322 for keyword in setup_kw.keywords:
1323 if self.start_setup_kw(keyword) is not False:
1324 self.visit_setup_kw(keyword)
1325 self.end_setup_kw(keyword)
1327 def start_setup_kw(self, setup_kw):
1328 """Called when teardown keyword starts. Default implementation does
1331 :param setup_kw: Keyword to process.
1332 :type setup_kw: Keyword
1335 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1336 and not self._version:
1337 self._msg_type = u"vpp-version"
1338 elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1340 self._msg_type = u"dpdk-version"
1341 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1342 self._msg_type = u"testbed"
1345 setup_kw.messages.visit(self)
1347 def end_setup_kw(self, setup_kw):
1348 """Called when keyword ends. Default implementation does nothing.
1350 :param setup_kw: Keyword to process.
1351 :type setup_kw: Keyword
1355 def visit_teardown_kw(self, teardown_kw):
1356 """Implements traversing through the teardown keyword and its child
1359 :param teardown_kw: Keyword to process.
1360 :type teardown_kw: Keyword
1363 for keyword in teardown_kw.keywords:
1364 if self.start_teardown_kw(keyword) is not False:
1365 self.visit_teardown_kw(keyword)
1366 self.end_teardown_kw(keyword)
1368 def start_teardown_kw(self, teardown_kw):
1369 """Called when teardown keyword starts
1371 :param teardown_kw: Keyword to process.
1372 :type teardown_kw: Keyword
1375 if teardown_kw.name.count(u"Show Papi History On All Duts"):
1376 self._conf_history_lookup_nr = 0
1377 self._msg_type = u"teardown-papi-history"
1378 teardown_kw.messages.visit(self)
1380 def end_teardown_kw(self, teardown_kw):
1381 """Called when keyword ends. Default implementation does nothing.
1383 :param teardown_kw: Keyword to process.
1384 :type teardown_kw: Keyword
1388 def visit_message(self, msg):
1389 """Implements visiting the message.
1391 :param msg: Message to process.
1395 if self.start_message(msg) is not False:
1396 self.end_message(msg)
1398 def start_message(self, msg):
1399 """Called when message starts. Get required information from messages:
1402 :param msg: Message to process.
1407 self.parse_msg[self._msg_type](msg)
1409 def end_message(self, msg):
1410 """Called when message ends. Default implementation does nothing.
1412 :param msg: Message to process.
1421 The data is extracted from output.xml files generated by Jenkins jobs and
1422 stored in pandas' DataFrames.
1428 (as described in ExecutionChecker documentation)
1430 (as described in ExecutionChecker documentation)
1432 (as described in ExecutionChecker documentation)
1435 def __init__(self, spec, for_output):
1438 :param spec: Specification.
1439 :param for_output: Output to be generated from downloaded data.
1440 :type spec: Specification
1441 :type for_output: str
1447 self._for_output = for_output
1450 self._input_data = pd.Series()
1454 """Getter - Input data.
1456 :returns: Input data
1457 :rtype: pandas.Series
1459 return self._input_data
1461 def metadata(self, job, build):
1462 """Getter - metadata
1464 :param job: Job which metadata we want.
1465 :param build: Build which metadata we want.
1469 :rtype: pandas.Series
1471 return self.data[job][build][u"metadata"]
1473 def suites(self, job, build):
1476 :param job: Job which suites we want.
1477 :param build: Build which suites we want.
1481 :rtype: pandas.Series
1483 return self.data[job][str(build)][u"suites"]
1485 def tests(self, job, build):
1488 :param job: Job which tests we want.
1489 :param build: Build which tests we want.
1493 :rtype: pandas.Series
1495 return self.data[job][build][u"tests"]
1497 def _parse_tests(self, job, build):
1498 """Process data from robot output.xml file and return JSON structured
1501 :param job: The name of job which build output data will be processed.
1502 :param build: The build which output data will be processed.
1505 :returns: JSON data structure.
1514 with open(build[u"file-name"], u'r') as data_file:
1516 result = ExecutionResult(data_file)
1517 except errors.DataError as err:
1519 f"Error occurred while parsing output.xml: {repr(err)}"
1522 checker = ExecutionChecker(
1523 metadata, self._cfg.mapping, self._cfg.ignore, self._for_output
1525 result.visit(checker)
1527 checker.data[u"metadata"][u"tests_total"] = \
1528 result.statistics.total.all.total
1529 checker.data[u"metadata"][u"tests_passed"] = \
1530 result.statistics.total.all.passed
1531 checker.data[u"metadata"][u"tests_failed"] = \
1532 result.statistics.total.all.failed
1533 checker.data[u"metadata"][u"elapsedtime"] = result.suite.elapsedtime
1534 checker.data[u"metadata"][u"generated"] = result.suite.endtime[:14]
1538 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1539 """Download and parse the input data file.
1541 :param pid: PID of the process executing this method.
1542 :param job: Name of the Jenkins job which generated the processed input
1544 :param build: Information about the Jenkins build which generated the
1545 processed input file.
1546 :param repeat: Repeat the download specified number of times if not
1554 logging.info(f"Processing the job/build: {job}: {build[u'build']}")
1561 success = download_and_unzip_data_file(self._cfg, job, build, pid)
1567 f"It is not possible to download the input data file from the "
1568 f"job {job}, build {build[u'build']}, or it is damaged. "
1572 logging.info(f" Processing data from build {build[u'build']}")
1573 data = self._parse_tests(job, build)
1576 f"Input data file from the job {job}, build "
1577 f"{build[u'build']} is damaged. Skipped."
1580 state = u"processed"
1583 remove(build[u"file-name"])
1584 except OSError as err:
1586 f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1589 # If the time-period is defined in the specification file, remove all
1590 # files which are outside the time period.
1592 timeperiod = self._cfg.environment.get(u"time-period", None)
1593 if timeperiod and data:
1595 timeperiod = timedelta(int(timeperiod))
1596 metadata = data.get(u"metadata", None)
1598 generated = metadata.get(u"generated", None)
1600 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1601 if (now - generated) > timeperiod:
1602 # Remove the data and the file:
1607 f" The build {job}/{build[u'build']} is "
1608 f"outdated, will be removed."
1618 def download_and_parse_data(self, repeat=1):
1619 """Download the input data files, parse input data from input files and
1620 store in pandas' Series.
1622 :param repeat: Repeat the download specified number of times if not
1627 logging.info(u"Downloading and parsing input files ...")
1629 for job, builds in self._cfg.input.items():
1630 for build in builds:
1632 result = self._download_and_parse_build(job, build, repeat)
1635 build_nr = result[u"build"][u"build"]
1638 data = result[u"data"]
1639 build_data = pd.Series({
1640 u"metadata": pd.Series(
1641 list(data[u"metadata"].values()),
1642 index=list(data[u"metadata"].keys())
1644 u"suites": pd.Series(
1645 list(data[u"suites"].values()),
1646 index=list(data[u"suites"].keys())
1648 u"tests": pd.Series(
1649 list(data[u"tests"].values()),
1650 index=list(data[u"tests"].keys())
1654 if self._input_data.get(job, None) is None:
1655 self._input_data[job] = pd.Series()
1656 self._input_data[job][str(build_nr)] = build_data
1657 self._cfg.set_input_file_name(
1658 job, build_nr, result[u"build"][u"file-name"]
1660 self._cfg.set_input_state(job, build_nr, result[u"state"])
1663 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1664 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1666 logging.info(u"Done.")
1668 msg = f"Successful downloads from the sources:\n"
1669 for source in self._cfg.environment[u"data-sources"]:
1670 if source[u"successful-downloads"]:
1672 f"{source[u'url']}/{source[u'path']}/"
1673 f"{source[u'file-name']}: "
1674 f"{source[u'successful-downloads']}\n"
1678 def process_local_file(self, local_file, job=u"local", build_nr=1,
1680 """Process local XML file given as a command-line parameter.
1682 :param local_file: The file to process.
1683 :param job: Job name.
1684 :param build_nr: Build number.
1685 :param replace: If True, the information about jobs and builds is
1686 replaced by the new one, otherwise the new jobs and builds are
1688 :type local_file: str
1692 :raises: PresentationError if an error occurs.
1694 if not isfile(local_file):
1695 raise PresentationError(f"The file {local_file} does not exist.")
1698 build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1699 except (IndexError, ValueError):
1704 u"status": u"failed",
1705 u"file-name": local_file
1708 self._cfg.input = dict()
1709 self._cfg.add_build(job, build)
1711 logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1712 data = self._parse_tests(job, build)
1714 raise PresentationError(
1715 f"Error occurred while parsing the file {local_file}"
1718 build_data = pd.Series({
1719 u"metadata": pd.Series(
1720 list(data[u"metadata"].values()),
1721 index=list(data[u"metadata"].keys())
1723 u"suites": pd.Series(
1724 list(data[u"suites"].values()),
1725 index=list(data[u"suites"].keys())
1727 u"tests": pd.Series(
1728 list(data[u"tests"].values()),
1729 index=list(data[u"tests"].keys())
1733 if self._input_data.get(job, None) is None:
1734 self._input_data[job] = pd.Series()
1735 self._input_data[job][str(build_nr)] = build_data
1737 self._cfg.set_input_state(job, build_nr, u"processed")
1739 def process_local_directory(self, local_dir, replace=True):
1740 """Process local directory with XML file(s). The directory is processed
1741 as a 'job' and the XML files in it as builds.
1742 If the given directory contains only sub-directories, these
1743 sub-directories processed as jobs and corresponding XML files as builds
1746 :param local_dir: Local directory to process.
1747 :param replace: If True, the information about jobs and builds is
1748 replaced by the new one, otherwise the new jobs and builds are
1750 :type local_dir: str
1753 if not isdir(local_dir):
1754 raise PresentationError(
1755 f"The directory {local_dir} does not exist."
1758 # Check if the given directory includes only files, or only directories
1759 _, dirnames, filenames = next(walk(local_dir))
1761 if filenames and not dirnames:
1764 # key: dir (job) name, value: list of file names (builds)
1766 local_dir: [join(local_dir, name) for name in filenames]
1769 elif dirnames and not filenames:
1772 # key: dir (job) name, value: list of file names (builds)
1773 local_builds = dict()
1774 for dirname in dirnames:
1776 join(local_dir, dirname, name)
1777 for name in listdir(join(local_dir, dirname))
1778 if isfile(join(local_dir, dirname, name))
1781 local_builds[dirname] = sorted(builds)
1783 elif not filenames and not dirnames:
1784 raise PresentationError(f"The directory {local_dir} is empty.")
1786 raise PresentationError(
1787 f"The directory {local_dir} can include only files or only "
1788 f"directories, not both.\nThe directory {local_dir} includes "
1789 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1793 self._cfg.input = dict()
1795 for job, files in local_builds.items():
1796 for idx, local_file in enumerate(files):
1797 self.process_local_file(local_file, job, idx + 1, replace=False)
1800 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1801 """Return the index of character in the string which is the end of tag.
1803 :param tag_filter: The string where the end of tag is being searched.
1804 :param start: The index where the searching is stated.
1805 :param closer: The character which is the tag closer.
1806 :type tag_filter: str
1809 :returns: The index of the tag closer.
1813 idx_opener = tag_filter.index(closer, start)
1814 return tag_filter.index(closer, idx_opener + 1)
1819 def _condition(tag_filter):
1820 """Create a conditional statement from the given tag filter.
1822 :param tag_filter: Filter based on tags from the element specification.
1823 :type tag_filter: str
1824 :returns: Conditional statement which can be evaluated.
1829 index = InputData._end_of_tag(tag_filter, index)
1833 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1835 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1836 continue_on_error=False):
1837 """Filter required data from the given jobs and builds.
1839 The output data structure is:
1842 - test (or suite) 1 ID:
1848 - test (or suite) n ID:
1855 :param element: Element which will use the filtered data.
1856 :param params: Parameters which will be included in the output. If None,
1857 all parameters are included.
1858 :param data: If not None, this data is used instead of data specified
1860 :param data_set: The set of data to be filtered: tests, suites,
1862 :param continue_on_error: Continue if there is error while reading the
1863 data. The Item will be empty then
1864 :type element: pandas.Series
1868 :type continue_on_error: bool
1869 :returns: Filtered data.
1870 :rtype pandas.Series
1874 if data_set == "suites":
1876 elif element[u"filter"] in (u"all", u"template"):
1879 cond = InputData._condition(element[u"filter"])
1880 logging.debug(f" Filter: {cond}")
1882 logging.error(u" No filter defined.")
1886 params = element.get(u"parameters", None)
1888 params.extend((u"type", u"status"))
1890 data_to_filter = data if data else element[u"data"]
1893 for job, builds in data_to_filter.items():
1894 data[job] = pd.Series()
1895 for build in builds:
1896 data[job][str(build)] = pd.Series()
1899 self.data[job][str(build)][data_set].items())
1901 if continue_on_error:
1905 for test_id, test_data in data_dict.items():
1906 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1907 data[job][str(build)][test_id] = pd.Series()
1909 for param, val in test_data.items():
1910 data[job][str(build)][test_id][param] = val
1912 for param in params:
1914 data[job][str(build)][test_id][param] =\
1917 data[job][str(build)][test_id][param] =\
1921 except (KeyError, IndexError, ValueError) as err:
1923 f"Missing mandatory parameter in the element specification: "
1927 except AttributeError as err:
1928 logging.error(repr(err))
1930 except SyntaxError as err:
1932 f"The filter {cond} is not correct. Check if all tags are "
1933 f"enclosed by apostrophes.\n{repr(err)}"
1937 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1938 continue_on_error=False):
1939 """Filter required data from the given jobs and builds.
1941 The output data structure is:
1944 - test (or suite) 1 ID:
1950 - test (or suite) n ID:
1957 :param element: Element which will use the filtered data.
1958 :param params: Parameters which will be included in the output. If None,
1959 all parameters are included.
1960 :param data_set: The set of data to be filtered: tests, suites,
1962 :param continue_on_error: Continue if there is error while reading the
1963 data. The Item will be empty then
1964 :type element: pandas.Series
1967 :type continue_on_error: bool
1968 :returns: Filtered data.
1969 :rtype pandas.Series
1972 include = element.get(u"include", None)
1974 logging.warning(u"No tests to include, skipping the element.")
1978 params = element.get(u"parameters", None)
1979 if params and u"type" not in params:
1980 params.append(u"type")
1982 cores = element.get(u"core", None)
1986 for test in include:
1987 tests.append(test.format(core=core))
1993 for job, builds in element[u"data"].items():
1994 data[job] = pd.Series()
1995 for build in builds:
1996 data[job][str(build)] = pd.Series()
1999 reg_ex = re.compile(str(test).lower())
2000 for test_id in self.data[job][
2001 str(build)][data_set].keys():
2002 if re.match(reg_ex, str(test_id).lower()):
2003 test_data = self.data[job][
2004 str(build)][data_set][test_id]
2005 data[job][str(build)][test_id] = pd.Series()
2007 for param, val in test_data.items():
2008 data[job][str(build)][test_id]\
2011 for param in params:
2013 data[job][str(build)][
2017 data[job][str(build)][
2018 test_id][param] = u"No Data"
2019 except KeyError as err:
2020 if continue_on_error:
2021 logging.debug(repr(err))
2023 logging.error(repr(err))
2027 except (KeyError, IndexError, ValueError) as err:
2029 f"Missing mandatory parameter in the element "
2030 f"specification: {repr(err)}"
2033 except AttributeError as err:
2034 logging.error(repr(err))
2038 def merge_data(data):
2039 """Merge data from more jobs and builds to a simple data structure.
2041 The output data structure is:
2043 - test (suite) 1 ID:
2049 - test (suite) n ID:
2052 :param data: Data to merge.
2053 :type data: pandas.Series
2054 :returns: Merged data.
2055 :rtype: pandas.Series
2058 logging.info(u" Merging data ...")
2060 merged_data = pd.Series()
2061 for builds in data.values:
2062 for item in builds.values:
2063 for item_id, item_data in item.items():
2064 merged_data[item_id] = item_data
2067 def print_all_oper_data(self):
2068 """Print all operational data to console.
2071 for job in self._input_data.values:
2072 for build in job.values:
2073 for test_id, test_data in build[u"tests"].items():
2075 if test_data.get(u"show-run", None) is None:
2077 for dut_name, data in test_data[u"show-run"].items():
2078 if data.get(u"runtime", None) is None:
2080 runtime = loads(data[u"runtime"])
2082 threads_nr = len(runtime[0][u"clocks"])
2083 except (IndexError, KeyError):
2085 threads = OrderedDict(
2086 {idx: list() for idx in range(threads_nr)})
2087 for item in runtime:
2088 for idx in range(threads_nr):
2089 if item[u"vectors"][idx] > 0:
2090 clocks = item[u"clocks"][idx] / \
2091 item[u"vectors"][idx]
2092 elif item[u"calls"][idx] > 0:
2093 clocks = item[u"clocks"][idx] / \
2095 elif item[u"suspends"][idx] > 0:
2096 clocks = item[u"clocks"][idx] / \
2097 item[u"suspends"][idx]
2101 if item[u"calls"][idx] > 0:
2102 vectors_call = item[u"vectors"][idx] / \
2107 if int(item[u"calls"][idx]) + int(
2108 item[u"vectors"][idx]) + \
2109 int(item[u"suspends"][idx]):
2110 threads[idx].append([
2112 item[u"calls"][idx],
2113 item[u"vectors"][idx],
2114 item[u"suspends"][idx],
2119 print(f"Host IP: {data.get(u'host', '')}, "
2120 f"Socket: {data.get(u'socket', '')}")
2121 for thread_nr, thread in threads.items():
2122 txt_table = prettytable.PrettyTable(
2128 u"Cycles per Packet",
2129 u"Average Vector Size"
2134 txt_table.add_row(row)
2136 if len(thread) == 0:
2139 avg = f", Average Vector Size per Node: " \
2140 f"{(avg / len(thread)):.2f}"
2141 th_name = u"main" if thread_nr == 0 \
2142 else f"worker_{thread_nr}"
2143 print(f"{dut_name}, {th_name}{avg}")
2144 txt_table.float_format = u".2"
2145 txt_table.align = u"r"
2146 txt_table.align[u"Name"] = u"l"
2147 print(f"{txt_table.get_string()}\n")