1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
28 from os import remove, walk, listdir
29 from os.path import isfile, isdir, join
30 from datetime import datetime as dt
31 from datetime import timedelta
32 from json import loads
33 from json.decoder import JSONDecodeError
40 from robot.api import ExecutionResult, ResultVisitor
41 from robot import errors
43 from resources.libraries.python import jumpavg
44 from input_data_files import download_and_unzip_data_file
45 from pal_errors import PresentationError
48 # Separator used in file names
52 class ExecutionChecker(ResultVisitor):
53 """Class to traverse through the test suite structure.
55 The functionality implemented in this class generates a json structure:
61 "generated": "Timestamp",
62 "version": "SUT version",
63 "job": "Jenkins job name",
64 "build": "Information about the build"
67 "Suite long name 1": {
69 "doc": "Suite 1 documentation",
70 "parent": "Suite 1 parent",
71 "level": "Level of the suite in the suite hierarchy"
73 "Suite long name N": {
75 "doc": "Suite N documentation",
76 "parent": "Suite 2 parent",
77 "level": "Level of the suite in the suite hierarchy"
84 "parent": "Name of the parent of the test",
85 "doc": "Test documentation",
86 "msg": "Test message",
87 "conf-history": "DUT1 and DUT2 VAT History",
88 "show-run": "Show Run",
89 "tags": ["tag 1", "tag 2", "tag n"],
91 "status": "PASS" | "FAIL",
137 "parent": "Name of the parent of the test",
138 "doc": "Test documentation",
139 "msg": "Test message",
140 "tags": ["tag 1", "tag 2", "tag n"],
142 "status": "PASS" | "FAIL",
149 "parent": "Name of the parent of the test",
150 "doc": "Test documentation",
151 "msg": "Test message",
152 "tags": ["tag 1", "tag 2", "tag n"],
153 "type": "MRR" | "BMRR",
154 "status": "PASS" | "FAIL",
156 "receive-rate": float,
157 # Average of a list, computed using AvgStdevStats.
158 # In CSIT-1180, replace with List[float].
172 "metadata": { # Optional
173 "version": "VPP version",
174 "job": "Jenkins job name",
175 "build": "Information about the build"
179 "doc": "Suite 1 documentation",
180 "parent": "Suite 1 parent",
181 "level": "Level of the suite in the suite hierarchy"
184 "doc": "Suite N documentation",
185 "parent": "Suite 2 parent",
186 "level": "Level of the suite in the suite hierarchy"
192 "parent": "Name of the parent of the test",
193 "doc": "Test documentation"
194 "msg": "Test message"
195 "tags": ["tag 1", "tag 2", "tag n"],
196 "conf-history": "DUT1 and DUT2 VAT History"
197 "show-run": "Show Run"
198 "status": "PASS" | "FAIL"
206 .. note:: ID is the lowercase full path to the test.
209 REGEX_PLR_RATE = re.compile(
210 r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
211 r'PLRsearch upper bound::?\s(\d+.\d+)'
213 REGEX_NDRPDR_RATE = re.compile(
214 r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
215 r'NDR_UPPER:\s(\d+.\d+).*\n'
216 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
217 r'PDR_UPPER:\s(\d+.\d+)'
219 REGEX_NDRPDR_GBPS = re.compile(
220 r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
221 r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
222 r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
223 r'PDR_UPPER:.*,\s(\d+.\d+)'
225 REGEX_PERF_MSG_INFO = re.compile(
226 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
227 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
228 r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
229 r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
230 r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
232 REGEX_CPS_MSG_INFO = re.compile(
233 r'NDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*\n'
234 r'PDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*'
236 REGEX_PPS_MSG_INFO = re.compile(
237 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
238 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*'
240 REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
242 # Needed for CPS and PPS tests
243 REGEX_NDRPDR_LAT_BASE = re.compile(
244 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
245 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
247 REGEX_NDRPDR_LAT = re.compile(
248 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
249 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
250 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
251 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
252 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
253 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
256 REGEX_VERSION_VPP = re.compile(
257 r"(return STDOUT Version:\s*|"
258 r"VPP Version:\s*|VPP version:\s*)(.*)"
260 REGEX_VERSION_DPDK = re.compile(
261 r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
263 REGEX_TCP = re.compile(
264 r'Total\s(rps|cps|throughput):\s(\d*).*$'
266 REGEX_MRR = re.compile(
267 r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
268 r'tx\s(\d*),\srx\s(\d*)'
270 REGEX_BMRR = re.compile(
271 r'.*trial results.*: \[(.*)\]'
273 REGEX_RECONF_LOSS = re.compile(
274 r'Packets lost due to reconfig: (\d*)'
276 REGEX_RECONF_TIME = re.compile(
277 r'Implied time lost: (\d*.[\de-]*)'
279 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
281 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
283 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
285 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
287 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
289 def __init__(self, metadata, mapping, ignore):
292 :param metadata: Key-value pairs to be included in "metadata" part of
294 :param mapping: Mapping of the old names of test cases to the new
296 :param ignore: List of TCs to be ignored.
302 # Type of message to parse out from the test messages
303 self._msg_type = None
309 self._timestamp = None
311 # Testbed. The testbed is identified by TG node IP address.
314 # Mapping of TCs long names
315 self._mapping = mapping
318 self._ignore = ignore
320 # Number of PAPI History messages found:
322 # 1 - PAPI History of DUT1
323 # 2 - PAPI History of DUT2
324 self._conf_history_lookup_nr = 0
326 self._sh_run_counter = 0
328 # Test ID of currently processed test- the lowercase full path to the
332 # The main data structure
334 u"metadata": OrderedDict(),
335 u"suites": OrderedDict(),
336 u"tests": OrderedDict()
339 # Save the provided metadata
340 for key, val in metadata.items():
341 self._data[u"metadata"][key] = val
343 # Dictionary defining the methods used to parse different types of
346 u"timestamp": self._get_timestamp,
347 u"vpp-version": self._get_vpp_version,
348 u"dpdk-version": self._get_dpdk_version,
349 u"teardown-papi-history": self._get_papi_history,
350 u"test-show-runtime": self._get_show_run,
351 u"testbed": self._get_testbed
356 """Getter - Data parsed from the XML file.
358 :returns: Data parsed from the XML file.
363 def _get_data_from_mrr_test_msg(self, msg):
364 """Get info from message of MRR performance tests.
366 :param msg: Message to be processed.
368 :returns: Processed message or original message if a problem occurs.
372 groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
373 if not groups or groups.lastindex != 1:
374 return u"Test Failed."
377 data = groups.group(1).split(u", ")
378 except (AttributeError, IndexError, ValueError, KeyError):
379 return u"Test Failed."
384 out_str += f"{(float(item) / 1e6):.2f}, "
385 return out_str[:-2] + u"]"
386 except (AttributeError, IndexError, ValueError, KeyError):
387 return u"Test Failed."
389 def _get_data_from_cps_test_msg(self, msg):
390 """Get info from message of NDRPDR CPS tests.
392 :param msg: Message to be processed.
394 :returns: Processed message or "Test Failed." if a problem occurs.
398 groups = re.search(self.REGEX_CPS_MSG_INFO, msg)
399 if not groups or groups.lastindex != 2:
400 return u"Test Failed."
404 f"1. {(float(groups.group(1)) / 1e6):5.2f}\n"
405 f"2. {(float(groups.group(2)) / 1e6):5.2f}"
407 except (AttributeError, IndexError, ValueError, KeyError):
408 return u"Test Failed."
410 def _get_data_from_pps_test_msg(self, msg):
411 """Get info from message of NDRPDR PPS tests.
413 :param msg: Message to be processed.
415 :returns: Processed message or "Test Failed." if a problem occurs.
419 groups = re.search(self.REGEX_PPS_MSG_INFO, msg)
420 if not groups or groups.lastindex != 4:
421 return u"Test Failed."
425 f"1. {(float(groups.group(1)) / 1e6):5.2f} "
426 f"{float(groups.group(2)):5.2f}\n"
427 f"2. {(float(groups.group(3)) / 1e6):5.2f} "
428 f"{float(groups.group(4)):5.2f}"
430 except (AttributeError, IndexError, ValueError, KeyError):
431 return u"Test Failed."
433 def _get_data_from_perf_test_msg(self, msg):
434 """Get info from message of NDRPDR performance tests.
436 :param msg: Message to be processed.
438 :returns: Processed message or "Test Failed." if a problem occurs.
442 groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
443 if not groups or groups.lastindex != 10:
444 return u"Test Failed."
448 u"ndr_low": float(groups.group(1)),
449 u"ndr_low_b": float(groups.group(2)),
450 u"pdr_low": float(groups.group(3)),
451 u"pdr_low_b": float(groups.group(4)),
452 u"pdr_lat_90_1": groups.group(5),
453 u"pdr_lat_90_2": groups.group(6),
454 u"pdr_lat_50_1": groups.group(7),
455 u"pdr_lat_50_2": groups.group(8),
456 u"pdr_lat_10_1": groups.group(9),
457 u"pdr_lat_10_2": groups.group(10),
459 except (AttributeError, IndexError, ValueError, KeyError):
460 return u"Test Failed."
462 def _process_lat(in_str_1, in_str_2):
463 """Extract min, avg, max values from latency string.
465 :param in_str_1: Latency string for one direction produced by robot
467 :param in_str_2: Latency string for second direction produced by
471 :returns: Processed latency string or None if a problem occurs.
474 in_list_1 = in_str_1.split('/', 3)
475 in_list_2 = in_str_2.split('/', 3)
477 if len(in_list_1) != 4 and len(in_list_2) != 4:
480 in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
482 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
483 except hdrh.codec.HdrLengthException:
486 in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
488 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
489 except hdrh.codec.HdrLengthException:
492 if hdr_lat_1 and hdr_lat_2:
494 hdr_lat_1.get_value_at_percentile(50.0),
495 hdr_lat_1.get_value_at_percentile(90.0),
496 hdr_lat_1.get_value_at_percentile(99.0),
497 hdr_lat_2.get_value_at_percentile(50.0),
498 hdr_lat_2.get_value_at_percentile(90.0),
499 hdr_lat_2.get_value_at_percentile(99.0)
509 f"1. {(data[u'ndr_low'] / 1e6):5.2f} "
510 f"{data[u'ndr_low_b']:5.2f}"
511 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f} "
512 f"{data[u'pdr_low_b']:5.2f}"
515 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
516 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
517 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
520 max_len = len(str(max((max(item) for item in latency))))
521 max_len = 4 if max_len < 4 else max_len
523 for idx, lat in enumerate(latency):
528 f"{lat[0]:{max_len}d} "
529 f"{lat[1]:{max_len}d} "
530 f"{lat[2]:{max_len}d} "
531 f"{lat[3]:{max_len}d} "
532 f"{lat[4]:{max_len}d} "
533 f"{lat[5]:{max_len}d} "
538 except (AttributeError, IndexError, ValueError, KeyError):
539 return u"Test Failed."
541 def _get_testbed(self, msg):
542 """Called when extraction of testbed IP is required.
543 The testbed is identified by TG node IP address.
545 :param msg: Message to process.
550 if msg.message.count(u"Setup of TG node") or \
551 msg.message.count(u"Setup of node TG host"):
552 reg_tg_ip = re.compile(
553 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
555 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
556 except (KeyError, ValueError, IndexError, AttributeError):
559 self._data[u"metadata"][u"testbed"] = self._testbed
560 self._msg_type = None
562 def _get_vpp_version(self, msg):
563 """Called when extraction of VPP version is required.
565 :param msg: Message to process.
570 if msg.message.count(u"return STDOUT Version:") or \
571 msg.message.count(u"VPP Version:") or \
572 msg.message.count(u"VPP version:"):
574 re.search(self.REGEX_VERSION_VPP, msg.message).group(2)
576 self._data[u"metadata"][u"version"] = self._version
577 self._msg_type = None
579 def _get_dpdk_version(self, msg):
580 """Called when extraction of DPDK version is required.
582 :param msg: Message to process.
587 if msg.message.count(u"DPDK Version:"):
589 self._version = str(re.search(
590 self.REGEX_VERSION_DPDK, msg.message).group(2))
591 self._data[u"metadata"][u"version"] = self._version
595 self._msg_type = None
597 def _get_timestamp(self, msg):
598 """Called when extraction of timestamp is required.
600 :param msg: Message to process.
605 self._timestamp = msg.timestamp[:14]
606 self._data[u"metadata"][u"generated"] = self._timestamp
607 self._msg_type = None
609 def _get_papi_history(self, msg):
610 """Called when extraction of PAPI command history is required.
612 :param msg: Message to process.
616 if msg.message.count(u"PAPI command history:"):
617 self._conf_history_lookup_nr += 1
618 if self._conf_history_lookup_nr == 1:
619 self._data[u"tests"][self._test_id][u"conf-history"] = str()
621 self._msg_type = None
623 r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} PAPI command history:",
627 ).replace(u'"', u"'")
628 self._data[u"tests"][self._test_id][u"conf-history"] += (
629 f"**DUT{str(self._conf_history_lookup_nr)}:** {text}"
632 def _get_show_run(self, msg):
633 """Called when extraction of VPP operational data (output of CLI command
634 Show Runtime) is required.
636 :param msg: Message to process.
641 if not msg.message.count(u"stats runtime"):
645 if self._sh_run_counter > 1:
648 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
649 self._data[u"tests"][self._test_id][u"show-run"] = dict()
651 groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
655 host = groups.group(1)
656 except (AttributeError, IndexError):
659 sock = groups.group(2)
660 except (AttributeError, IndexError):
663 runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
664 replace(u"'", u'"').replace(u'b"', u'"').
665 replace(u'u"', u'"').split(u":", 1)[1])
668 threads_nr = len(runtime[0][u"clocks"])
669 except (IndexError, KeyError):
672 dut = u"dut{nr}".format(
673 nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
679 u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
683 for idx in range(threads_nr):
684 if item[u"vectors"][idx] > 0:
685 clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
686 elif item[u"calls"][idx] > 0:
687 clocks = item[u"clocks"][idx] / item[u"calls"][idx]
688 elif item[u"suspends"][idx] > 0:
689 clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
693 if item[u"calls"][idx] > 0:
694 vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
698 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
699 int(item[u"suspends"][idx]):
700 oper[u"threads"][idx].append([
703 item[u"vectors"][idx],
704 item[u"suspends"][idx],
709 self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
711 def _get_ndrpdr_throughput(self, msg):
712 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
715 :param msg: The test message to be parsed.
717 :returns: Parsed data as a dict and the status (PASS/FAIL).
718 :rtype: tuple(dict, str)
722 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
723 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
726 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
728 if groups is not None:
730 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
731 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
732 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
733 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
735 except (IndexError, ValueError):
738 return throughput, status
740 def _get_ndrpdr_throughput_gbps(self, msg):
741 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
744 :param msg: The test message to be parsed.
746 :returns: Parsed data as a dict and the status (PASS/FAIL).
747 :rtype: tuple(dict, str)
751 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
752 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
755 groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
757 if groups is not None:
759 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
760 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
761 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
762 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
764 except (IndexError, ValueError):
769 def _get_plr_throughput(self, msg):
770 """Get PLRsearch lower bound and PLRsearch upper bound from the test
773 :param msg: The test message to be parsed.
775 :returns: Parsed data as a dict and the status (PASS/FAIL).
776 :rtype: tuple(dict, str)
784 groups = re.search(self.REGEX_PLR_RATE, msg)
786 if groups is not None:
788 throughput[u"LOWER"] = float(groups.group(1))
789 throughput[u"UPPER"] = float(groups.group(2))
791 except (IndexError, ValueError):
794 return throughput, status
796 def _get_ndrpdr_latency(self, msg):
797 """Get LATENCY from the test message.
799 :param msg: The test message to be parsed.
801 :returns: Parsed data as a dict and the status (PASS/FAIL).
802 :rtype: tuple(dict, str)
812 u"direction1": copy.copy(latency_default),
813 u"direction2": copy.copy(latency_default)
816 u"direction1": copy.copy(latency_default),
817 u"direction2": copy.copy(latency_default)
820 u"direction1": copy.copy(latency_default),
821 u"direction2": copy.copy(latency_default)
824 u"direction1": copy.copy(latency_default),
825 u"direction2": copy.copy(latency_default)
828 u"direction1": copy.copy(latency_default),
829 u"direction2": copy.copy(latency_default)
832 u"direction1": copy.copy(latency_default),
833 u"direction2": copy.copy(latency_default)
837 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
839 groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
841 return latency, u"FAIL"
843 def process_latency(in_str):
844 """Return object with parsed latency values.
846 TODO: Define class for the return type.
848 :param in_str: Input string, min/avg/max/hdrh format.
850 :returns: Dict with corresponding keys, except hdrh float values.
852 :throws IndexError: If in_str does not have enough substrings.
853 :throws ValueError: If a substring does not convert to float.
855 in_list = in_str.split('/', 3)
858 u"min": float(in_list[0]),
859 u"avg": float(in_list[1]),
860 u"max": float(in_list[2]),
864 if len(in_list) == 4:
865 rval[u"hdrh"] = str(in_list[3])
870 latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
871 latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
872 latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
873 latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
874 if groups.lastindex == 4:
875 return latency, u"PASS"
876 except (IndexError, ValueError):
880 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
881 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
882 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
883 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
884 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
885 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
886 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
887 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
888 if groups.lastindex == 12:
889 return latency, u"PASS"
890 except (IndexError, ValueError):
893 return latency, u"FAIL"
896 def _get_hoststack_data(msg, tags):
897 """Get data from the hoststack test message.
899 :param msg: The test message to be parsed.
900 :param tags: Test tags.
903 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
904 :rtype: tuple(dict, str)
909 msg = msg.replace(u"'", u'"').replace(u" ", u"")
910 if u"LDPRELOAD" in tags:
914 except JSONDecodeError:
916 elif u"VPPECHO" in tags:
918 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
920 client=loads(msg_lst[0]),
921 server=loads(msg_lst[1])
924 except (JSONDecodeError, IndexError):
927 return result, status
929 def visit_suite(self, suite):
930 """Implements traversing through the suite and its direct children.
932 :param suite: Suite to process.
936 if self.start_suite(suite) is not False:
937 suite.suites.visit(self)
938 suite.tests.visit(self)
939 self.end_suite(suite)
941 def start_suite(self, suite):
942 """Called when suite starts.
944 :param suite: Suite to process.
950 parent_name = suite.parent.name
951 except AttributeError:
954 self._data[u"suites"][suite.longname.lower().
956 replace(u" ", u"_")] = {
957 u"name": suite.name.lower(),
959 u"parent": parent_name,
960 u"level": len(suite.longname.split(u"."))
963 suite.keywords.visit(self)
965 def end_suite(self, suite):
966 """Called when suite ends.
968 :param suite: Suite to process.
973 def visit_test(self, test):
974 """Implements traversing through the test.
976 :param test: Test to process.
980 if self.start_test(test) is not False:
981 test.keywords.visit(self)
984 def start_test(self, test):
985 """Called when test starts.
987 :param test: Test to process.
992 self._sh_run_counter = 0
994 longname_orig = test.longname.lower()
996 # Check the ignore list
997 if longname_orig in self._ignore:
1000 tags = [str(tag) for tag in test.tags]
1001 test_result = dict()
1003 # Change the TC long name and name if defined in the mapping table
1004 longname = self._mapping.get(longname_orig, None)
1005 if longname is not None:
1006 name = longname.split(u'.')[-1]
1008 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1012 longname = longname_orig
1013 name = test.name.lower()
1015 # Remove TC number from the TC long name (backward compatibility):
1016 self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1017 # Remove TC number from the TC name (not needed):
1018 test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1020 test_result[u"parent"] = test.parent.name.lower()
1021 test_result[u"tags"] = tags
1022 test_result["doc"] = test.doc
1023 test_result[u"type"] = u""
1024 test_result[u"status"] = test.status
1025 test_result[u"starttime"] = test.starttime
1026 test_result[u"endtime"] = test.endtime
1028 if test.status == u"PASS":
1029 if u"NDRPDR" in tags:
1030 if u"TCP_PPS" in tags or u"UDP_PPS" in tags:
1031 test_result[u"msg"] = self._get_data_from_pps_test_msg(
1033 elif u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1034 test_result[u"msg"] = self._get_data_from_cps_test_msg(
1037 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1039 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1040 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1043 test_result[u"msg"] = test.message
1045 test_result[u"msg"] = test.message
1047 if u"PERFTEST" in tags:
1048 # Replace info about cores (e.g. -1c-) with the info about threads
1049 # and cores (e.g. -1t1c-) in the long test case names and in the
1050 # test case names if necessary.
1051 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1055 for tag in test_result[u"tags"]:
1056 groups = re.search(self.REGEX_TC_TAG, tag)
1062 self._test_id = re.sub(
1063 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1064 self._test_id, count=1
1066 test_result[u"name"] = re.sub(
1067 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1068 test_result["name"], count=1
1071 test_result[u"status"] = u"FAIL"
1072 self._data[u"tests"][self._test_id] = test_result
1074 f"The test {self._test_id} has no or more than one "
1075 f"multi-threading tags.\n"
1076 f"Tags: {test_result[u'tags']}"
1080 if u"DEVICETEST" in tags:
1081 test_result[u"type"] = u"DEVICETEST"
1082 elif u"NDRPDR" in tags:
1083 if u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1084 test_result[u"type"] = u"CPS"
1086 test_result[u"type"] = u"NDRPDR"
1087 if test.status == u"PASS":
1088 test_result[u"throughput"], test_result[u"status"] = \
1089 self._get_ndrpdr_throughput(test.message)
1090 test_result[u"gbps"], test_result[u"status"] = \
1091 self._get_ndrpdr_throughput_gbps(test.message)
1092 test_result[u"latency"], test_result[u"status"] = \
1093 self._get_ndrpdr_latency(test.message)
1094 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1096 test_result[u"type"] = u"MRR"
1098 test_result[u"type"] = u"BMRR"
1099 if test.status == u"PASS":
1100 test_result[u"result"] = dict()
1101 groups = re.search(self.REGEX_BMRR, test.message)
1102 if groups is not None:
1103 items_str = groups.group(1)
1105 float(item.strip().replace(u"'", u""))
1106 for item in items_str.split(",")
1108 # Use whole list in CSIT-1180.
1109 stats = jumpavg.AvgStdevStats.for_runs(items_float)
1110 test_result[u"result"][u"samples"] = items_float
1111 test_result[u"result"][u"receive-rate"] = stats.avg
1112 test_result[u"result"][u"receive-stdev"] = stats.stdev
1114 groups = re.search(self.REGEX_MRR, test.message)
1115 test_result[u"result"][u"receive-rate"] = \
1116 float(groups.group(3)) / float(groups.group(1))
1117 elif u"SOAK" in tags:
1118 test_result[u"type"] = u"SOAK"
1119 if test.status == u"PASS":
1120 test_result[u"throughput"], test_result[u"status"] = \
1121 self._get_plr_throughput(test.message)
1122 elif u"HOSTSTACK" in tags:
1123 test_result[u"type"] = u"HOSTSTACK"
1124 if test.status == u"PASS":
1125 test_result[u"result"], test_result[u"status"] = \
1126 self._get_hoststack_data(test.message, tags)
1127 # elif u"TCP" in tags: # This might be not used
1128 # test_result[u"type"] = u"TCP"
1129 # if test.status == u"PASS":
1130 # groups = re.search(self.REGEX_TCP, test.message)
1131 # test_result[u"result"] = int(groups.group(2))
1132 elif u"RECONF" in tags:
1133 test_result[u"type"] = u"RECONF"
1134 if test.status == u"PASS":
1135 test_result[u"result"] = None
1137 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1138 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1139 test_result[u"result"] = {
1140 u"loss": int(grps_loss.group(1)),
1141 u"time": float(grps_time.group(1))
1143 except (AttributeError, IndexError, ValueError, TypeError):
1144 test_result[u"status"] = u"FAIL"
1146 test_result[u"status"] = u"FAIL"
1148 self._data[u"tests"][self._test_id] = test_result
1150 def end_test(self, test):
1151 """Called when test ends.
1153 :param test: Test to process.
1158 def visit_keyword(self, keyword):
1159 """Implements traversing through the keyword and its child keywords.
1161 :param keyword: Keyword to process.
1162 :type keyword: Keyword
1165 if self.start_keyword(keyword) is not False:
1166 self.end_keyword(keyword)
1168 def start_keyword(self, keyword):
1169 """Called when keyword starts. Default implementation does nothing.
1171 :param keyword: Keyword to process.
1172 :type keyword: Keyword
1176 if keyword.type == u"setup":
1177 self.visit_setup_kw(keyword)
1178 elif keyword.type == u"teardown":
1179 self.visit_teardown_kw(keyword)
1181 self.visit_test_kw(keyword)
1182 except AttributeError:
1185 def end_keyword(self, keyword):
1186 """Called when keyword ends. Default implementation does nothing.
1188 :param keyword: Keyword to process.
1189 :type keyword: Keyword
1193 def visit_test_kw(self, test_kw):
1194 """Implements traversing through the test keyword and its child
1197 :param test_kw: Keyword to process.
1198 :type test_kw: Keyword
1201 for keyword in test_kw.keywords:
1202 if self.start_test_kw(keyword) is not False:
1203 self.visit_test_kw(keyword)
1204 self.end_test_kw(keyword)
1206 def start_test_kw(self, test_kw):
1207 """Called when test keyword starts. Default implementation does
1210 :param test_kw: Keyword to process.
1211 :type test_kw: Keyword
1214 if test_kw.name.count(u"Show Runtime On All Duts") or \
1215 test_kw.name.count(u"Show Runtime Counters On All Duts") or \
1216 test_kw.name.count(u"Vpp Show Runtime On All Duts"):
1217 self._msg_type = u"test-show-runtime"
1218 self._sh_run_counter += 1
1221 test_kw.messages.visit(self)
1223 def end_test_kw(self, test_kw):
1224 """Called when keyword ends. Default implementation does nothing.
1226 :param test_kw: Keyword to process.
1227 :type test_kw: Keyword
1231 def visit_setup_kw(self, setup_kw):
1232 """Implements traversing through the teardown keyword and its child
1235 :param setup_kw: Keyword to process.
1236 :type setup_kw: Keyword
1239 for keyword in setup_kw.keywords:
1240 if self.start_setup_kw(keyword) is not False:
1241 self.visit_setup_kw(keyword)
1242 self.end_setup_kw(keyword)
1244 def start_setup_kw(self, setup_kw):
1245 """Called when teardown keyword starts. Default implementation does
1248 :param setup_kw: Keyword to process.
1249 :type setup_kw: Keyword
1252 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1253 and not self._version:
1254 self._msg_type = u"vpp-version"
1255 elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1257 self._msg_type = u"dpdk-version"
1258 elif setup_kw.name.count(u"Set Global Variable") \
1259 and not self._timestamp:
1260 self._msg_type = u"timestamp"
1261 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1262 self._msg_type = u"testbed"
1265 setup_kw.messages.visit(self)
1267 def end_setup_kw(self, setup_kw):
1268 """Called when keyword ends. Default implementation does nothing.
1270 :param setup_kw: Keyword to process.
1271 :type setup_kw: Keyword
1275 def visit_teardown_kw(self, teardown_kw):
1276 """Implements traversing through the teardown keyword and its child
1279 :param teardown_kw: Keyword to process.
1280 :type teardown_kw: Keyword
1283 for keyword in teardown_kw.keywords:
1284 if self.start_teardown_kw(keyword) is not False:
1285 self.visit_teardown_kw(keyword)
1286 self.end_teardown_kw(keyword)
1288 def start_teardown_kw(self, teardown_kw):
1289 """Called when teardown keyword starts
1291 :param teardown_kw: Keyword to process.
1292 :type teardown_kw: Keyword
1295 if teardown_kw.name.count(u"Show Papi History On All Duts"):
1296 self._conf_history_lookup_nr = 0
1297 self._msg_type = u"teardown-papi-history"
1298 teardown_kw.messages.visit(self)
1300 def end_teardown_kw(self, teardown_kw):
1301 """Called when keyword ends. Default implementation does nothing.
1303 :param teardown_kw: Keyword to process.
1304 :type teardown_kw: Keyword
1308 def visit_message(self, msg):
1309 """Implements visiting the message.
1311 :param msg: Message to process.
1315 if self.start_message(msg) is not False:
1316 self.end_message(msg)
1318 def start_message(self, msg):
1319 """Called when message starts. Get required information from messages:
1322 :param msg: Message to process.
1327 self.parse_msg[self._msg_type](msg)
1329 def end_message(self, msg):
1330 """Called when message ends. Default implementation does nothing.
1332 :param msg: Message to process.
1341 The data is extracted from output.xml files generated by Jenkins jobs and
1342 stored in pandas' DataFrames.
1348 (as described in ExecutionChecker documentation)
1350 (as described in ExecutionChecker documentation)
1352 (as described in ExecutionChecker documentation)
1355 def __init__(self, spec):
1358 :param spec: Specification.
1359 :type spec: Specification
1366 self._input_data = pd.Series()
1370 """Getter - Input data.
1372 :returns: Input data
1373 :rtype: pandas.Series
1375 return self._input_data
1377 def metadata(self, job, build):
1378 """Getter - metadata
1380 :param job: Job which metadata we want.
1381 :param build: Build which metadata we want.
1385 :rtype: pandas.Series
1387 return self.data[job][build][u"metadata"]
1389 def suites(self, job, build):
1392 :param job: Job which suites we want.
1393 :param build: Build which suites we want.
1397 :rtype: pandas.Series
1399 return self.data[job][str(build)][u"suites"]
1401 def tests(self, job, build):
1404 :param job: Job which tests we want.
1405 :param build: Build which tests we want.
1409 :rtype: pandas.Series
1411 return self.data[job][build][u"tests"]
1413 def _parse_tests(self, job, build):
1414 """Process data from robot output.xml file and return JSON structured
1417 :param job: The name of job which build output data will be processed.
1418 :param build: The build which output data will be processed.
1421 :returns: JSON data structure.
1430 with open(build[u"file-name"], u'r') as data_file:
1432 result = ExecutionResult(data_file)
1433 except errors.DataError as err:
1435 f"Error occurred while parsing output.xml: {repr(err)}"
1438 checker = ExecutionChecker(
1439 metadata, self._cfg.mapping, self._cfg.ignore
1441 result.visit(checker)
1445 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1446 """Download and parse the input data file.
1448 :param pid: PID of the process executing this method.
1449 :param job: Name of the Jenkins job which generated the processed input
1451 :param build: Information about the Jenkins build which generated the
1452 processed input file.
1453 :param repeat: Repeat the download specified number of times if not
1461 logging.info(f"Processing the job/build: {job}: {build[u'build']}")
1468 success = download_and_unzip_data_file(self._cfg, job, build, pid)
1474 f"It is not possible to download the input data file from the "
1475 f"job {job}, build {build[u'build']}, or it is damaged. "
1479 logging.info(f" Processing data from build {build[u'build']}")
1480 data = self._parse_tests(job, build)
1483 f"Input data file from the job {job}, build "
1484 f"{build[u'build']} is damaged. Skipped."
1487 state = u"processed"
1490 remove(build[u"file-name"])
1491 except OSError as err:
1493 f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1496 # If the time-period is defined in the specification file, remove all
1497 # files which are outside the time period.
1499 timeperiod = self._cfg.environment.get(u"time-period", None)
1500 if timeperiod and data:
1502 timeperiod = timedelta(int(timeperiod))
1503 metadata = data.get(u"metadata", None)
1505 generated = metadata.get(u"generated", None)
1507 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1508 if (now - generated) > timeperiod:
1509 # Remove the data and the file:
1514 f" The build {job}/{build[u'build']} is "
1515 f"outdated, will be removed."
1525 def download_and_parse_data(self, repeat=1):
1526 """Download the input data files, parse input data from input files and
1527 store in pandas' Series.
1529 :param repeat: Repeat the download specified number of times if not
1534 logging.info(u"Downloading and parsing input files ...")
1536 for job, builds in self._cfg.input.items():
1537 for build in builds:
1539 result = self._download_and_parse_build(job, build, repeat)
1542 build_nr = result[u"build"][u"build"]
1545 data = result[u"data"]
1546 build_data = pd.Series({
1547 u"metadata": pd.Series(
1548 list(data[u"metadata"].values()),
1549 index=list(data[u"metadata"].keys())
1551 u"suites": pd.Series(
1552 list(data[u"suites"].values()),
1553 index=list(data[u"suites"].keys())
1555 u"tests": pd.Series(
1556 list(data[u"tests"].values()),
1557 index=list(data[u"tests"].keys())
1561 if self._input_data.get(job, None) is None:
1562 self._input_data[job] = pd.Series()
1563 self._input_data[job][str(build_nr)] = build_data
1564 self._cfg.set_input_file_name(
1565 job, build_nr, result[u"build"][u"file-name"]
1567 self._cfg.set_input_state(job, build_nr, result[u"state"])
1570 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1571 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1573 logging.info(u"Done.")
1575 msg = f"Successful downloads from the sources:\n"
1576 for source in self._cfg.environment[u"data-sources"]:
1577 if source[u"successful-downloads"]:
1579 f"{source[u'url']}/{source[u'path']}/"
1580 f"{source[u'file-name']}: "
1581 f"{source[u'successful-downloads']}\n"
1585 def process_local_file(self, local_file, job=u"local", build_nr=1,
1587 """Process local XML file given as a command-line parameter.
1589 :param local_file: The file to process.
1590 :param job: Job name.
1591 :param build_nr: Build number.
1592 :param replace: If True, the information about jobs and builds is
1593 replaced by the new one, otherwise the new jobs and builds are
1595 :type local_file: str
1599 :raises: PresentationError if an error occurs.
1601 if not isfile(local_file):
1602 raise PresentationError(f"The file {local_file} does not exist.")
1605 build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1606 except (IndexError, ValueError):
1611 u"status": u"failed",
1612 u"file-name": local_file
1615 self._cfg.input = dict()
1616 self._cfg.add_build(job, build)
1618 logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1619 data = self._parse_tests(job, build)
1621 raise PresentationError(
1622 f"Error occurred while parsing the file {local_file}"
1625 build_data = pd.Series({
1626 u"metadata": pd.Series(
1627 list(data[u"metadata"].values()),
1628 index=list(data[u"metadata"].keys())
1630 u"suites": pd.Series(
1631 list(data[u"suites"].values()),
1632 index=list(data[u"suites"].keys())
1634 u"tests": pd.Series(
1635 list(data[u"tests"].values()),
1636 index=list(data[u"tests"].keys())
1640 if self._input_data.get(job, None) is None:
1641 self._input_data[job] = pd.Series()
1642 self._input_data[job][str(build_nr)] = build_data
1644 self._cfg.set_input_state(job, build_nr, u"processed")
1646 def process_local_directory(self, local_dir, replace=True):
1647 """Process local directory with XML file(s). The directory is processed
1648 as a 'job' and the XML files in it as builds.
1649 If the given directory contains only sub-directories, these
1650 sub-directories processed as jobs and corresponding XML files as builds
1653 :param local_dir: Local directory to process.
1654 :param replace: If True, the information about jobs and builds is
1655 replaced by the new one, otherwise the new jobs and builds are
1657 :type local_dir: str
1660 if not isdir(local_dir):
1661 raise PresentationError(
1662 f"The directory {local_dir} does not exist."
1665 # Check if the given directory includes only files, or only directories
1666 _, dirnames, filenames = next(walk(local_dir))
1668 if filenames and not dirnames:
1671 # key: dir (job) name, value: list of file names (builds)
1673 local_dir: [join(local_dir, name) for name in filenames]
1676 elif dirnames and not filenames:
1679 # key: dir (job) name, value: list of file names (builds)
1680 local_builds = dict()
1681 for dirname in dirnames:
1683 join(local_dir, dirname, name)
1684 for name in listdir(join(local_dir, dirname))
1685 if isfile(join(local_dir, dirname, name))
1688 local_builds[dirname] = sorted(builds)
1690 elif not filenames and not dirnames:
1691 raise PresentationError(f"The directory {local_dir} is empty.")
1693 raise PresentationError(
1694 f"The directory {local_dir} can include only files or only "
1695 f"directories, not both.\nThe directory {local_dir} includes "
1696 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1700 self._cfg.input = dict()
1702 for job, files in local_builds.items():
1703 for idx, local_file in enumerate(files):
1704 self.process_local_file(local_file, job, idx + 1, replace=False)
1707 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1708 """Return the index of character in the string which is the end of tag.
1710 :param tag_filter: The string where the end of tag is being searched.
1711 :param start: The index where the searching is stated.
1712 :param closer: The character which is the tag closer.
1713 :type tag_filter: str
1716 :returns: The index of the tag closer.
1720 idx_opener = tag_filter.index(closer, start)
1721 return tag_filter.index(closer, idx_opener + 1)
1726 def _condition(tag_filter):
1727 """Create a conditional statement from the given tag filter.
1729 :param tag_filter: Filter based on tags from the element specification.
1730 :type tag_filter: str
1731 :returns: Conditional statement which can be evaluated.
1736 index = InputData._end_of_tag(tag_filter, index)
1740 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1742 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1743 continue_on_error=False):
1744 """Filter required data from the given jobs and builds.
1746 The output data structure is:
1749 - test (or suite) 1 ID:
1755 - test (or suite) n ID:
1762 :param element: Element which will use the filtered data.
1763 :param params: Parameters which will be included in the output. If None,
1764 all parameters are included.
1765 :param data: If not None, this data is used instead of data specified
1767 :param data_set: The set of data to be filtered: tests, suites,
1769 :param continue_on_error: Continue if there is error while reading the
1770 data. The Item will be empty then
1771 :type element: pandas.Series
1775 :type continue_on_error: bool
1776 :returns: Filtered data.
1777 :rtype pandas.Series
1781 if data_set == "suites":
1783 elif element[u"filter"] in (u"all", u"template"):
1786 cond = InputData._condition(element[u"filter"])
1787 logging.debug(f" Filter: {cond}")
1789 logging.error(u" No filter defined.")
1793 params = element.get(u"parameters", None)
1795 params.extend((u"type", u"status"))
1797 data_to_filter = data if data else element[u"data"]
1800 for job, builds in data_to_filter.items():
1801 data[job] = pd.Series()
1802 for build in builds:
1803 data[job][str(build)] = pd.Series()
1806 self.data[job][str(build)][data_set].items())
1808 if continue_on_error:
1812 for test_id, test_data in data_dict.items():
1813 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1814 data[job][str(build)][test_id] = pd.Series()
1816 for param, val in test_data.items():
1817 data[job][str(build)][test_id][param] = val
1819 for param in params:
1821 data[job][str(build)][test_id][param] =\
1824 data[job][str(build)][test_id][param] =\
1828 except (KeyError, IndexError, ValueError) as err:
1830 f"Missing mandatory parameter in the element specification: "
1834 except AttributeError as err:
1835 logging.error(repr(err))
1837 except SyntaxError as err:
1839 f"The filter {cond} is not correct. Check if all tags are "
1840 f"enclosed by apostrophes.\n{repr(err)}"
1844 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1845 continue_on_error=False):
1846 """Filter required data from the given jobs and builds.
1848 The output data structure is:
1851 - test (or suite) 1 ID:
1857 - test (or suite) n ID:
1864 :param element: Element which will use the filtered data.
1865 :param params: Parameters which will be included in the output. If None,
1866 all parameters are included.
1867 :param data_set: The set of data to be filtered: tests, suites,
1869 :param continue_on_error: Continue if there is error while reading the
1870 data. The Item will be empty then
1871 :type element: pandas.Series
1874 :type continue_on_error: bool
1875 :returns: Filtered data.
1876 :rtype pandas.Series
1879 include = element.get(u"include", None)
1881 logging.warning(u"No tests to include, skipping the element.")
1885 params = element.get(u"parameters", None)
1886 if params and u"type" not in params:
1887 params.append(u"type")
1889 cores = element.get(u"core", None)
1893 for test in include:
1894 tests.append(test.format(core=core))
1900 for job, builds in element[u"data"].items():
1901 data[job] = pd.Series()
1902 for build in builds:
1903 data[job][str(build)] = pd.Series()
1906 reg_ex = re.compile(str(test).lower())
1907 for test_id in self.data[job][
1908 str(build)][data_set].keys():
1909 if re.match(reg_ex, str(test_id).lower()):
1910 test_data = self.data[job][
1911 str(build)][data_set][test_id]
1912 data[job][str(build)][test_id] = pd.Series()
1914 for param, val in test_data.items():
1915 data[job][str(build)][test_id]\
1918 for param in params:
1920 data[job][str(build)][
1924 data[job][str(build)][
1925 test_id][param] = u"No Data"
1926 except KeyError as err:
1927 if continue_on_error:
1928 logging.debug(repr(err))
1930 logging.error(repr(err))
1934 except (KeyError, IndexError, ValueError) as err:
1936 f"Missing mandatory parameter in the element "
1937 f"specification: {repr(err)}"
1940 except AttributeError as err:
1941 logging.error(repr(err))
1945 def merge_data(data):
1946 """Merge data from more jobs and builds to a simple data structure.
1948 The output data structure is:
1950 - test (suite) 1 ID:
1956 - test (suite) n ID:
1959 :param data: Data to merge.
1960 :type data: pandas.Series
1961 :returns: Merged data.
1962 :rtype: pandas.Series
1965 logging.info(u" Merging data ...")
1967 merged_data = pd.Series()
1968 for builds in data.values:
1969 for item in builds.values:
1970 for item_id, item_data in item.items():
1971 merged_data[item_id] = item_data
1974 def print_all_oper_data(self):
1975 """Print all operational data to console.
1983 u"Cycles per Packet",
1984 u"Average Vector Size"
1987 for job in self._input_data.values:
1988 for build in job.values:
1989 for test_id, test_data in build[u"tests"].items():
1991 if test_data.get(u"show-run", None) is None:
1993 for dut_name, data in test_data[u"show-run"].items():
1994 if data.get(u"threads", None) is None:
1996 print(f"Host IP: {data.get(u'host', '')}, "
1997 f"Socket: {data.get(u'socket', '')}")
1998 for thread_nr, thread in data[u"threads"].items():
1999 txt_table = prettytable.PrettyTable(tbl_hdr)
2002 txt_table.add_row(row)
2004 if len(thread) == 0:
2007 avg = f", Average Vector Size per Node: " \
2008 f"{(avg / len(thread)):.2f}"
2009 th_name = u"main" if thread_nr == 0 \
2010 else f"worker_{thread_nr}"
2011 print(f"{dut_name}, {th_name}{avg}")
2012 txt_table.float_format = u".2"
2013 txt_table.align = u"r"
2014 txt_table.align[u"Name"] = u"l"
2015 print(f"{txt_table.get_string()}\n")