1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
28 from os import remove, walk, listdir
29 from os.path import isfile, isdir, join
30 from datetime import datetime as dt
31 from datetime import timedelta
32 from json import loads
33 from json.decoder import JSONDecodeError
40 from robot.api import ExecutionResult, ResultVisitor
41 from robot import errors
43 from resources.libraries.python import jumpavg
44 from input_data_files import download_and_unzip_data_file
45 from pal_errors import PresentationError
48 # Separator used in file names
52 class ExecutionChecker(ResultVisitor):
53 """Class to traverse through the test suite structure.
55 The functionality implemented in this class generates a json structure:
61 "generated": "Timestamp",
62 "version": "SUT version",
63 "job": "Jenkins job name",
64 "build": "Information about the build"
67 "Suite long name 1": {
69 "doc": "Suite 1 documentation",
70 "parent": "Suite 1 parent",
71 "level": "Level of the suite in the suite hierarchy"
73 "Suite long name N": {
75 "doc": "Suite N documentation",
76 "parent": "Suite 2 parent",
77 "level": "Level of the suite in the suite hierarchy"
84 "parent": "Name of the parent of the test",
85 "doc": "Test documentation",
86 "msg": "Test message",
87 "conf-history": "DUT1 and DUT2 VAT History",
88 "show-run": "Show Run",
89 "tags": ["tag 1", "tag 2", "tag n"],
91 "status": "PASS" | "FAIL",
137 "parent": "Name of the parent of the test",
138 "doc": "Test documentation",
139 "msg": "Test message",
140 "tags": ["tag 1", "tag 2", "tag n"],
142 "status": "PASS" | "FAIL",
149 "parent": "Name of the parent of the test",
150 "doc": "Test documentation",
151 "msg": "Test message",
152 "tags": ["tag 1", "tag 2", "tag n"],
153 "type": "MRR" | "BMRR",
154 "status": "PASS" | "FAIL",
156 "receive-rate": float,
157 # Average of a list, computed using AvgStdevStats.
158 # In CSIT-1180, replace with List[float].
172 "metadata": { # Optional
173 "version": "VPP version",
174 "job": "Jenkins job name",
175 "build": "Information about the build"
179 "doc": "Suite 1 documentation",
180 "parent": "Suite 1 parent",
181 "level": "Level of the suite in the suite hierarchy"
184 "doc": "Suite N documentation",
185 "parent": "Suite 2 parent",
186 "level": "Level of the suite in the suite hierarchy"
192 "parent": "Name of the parent of the test",
193 "doc": "Test documentation"
194 "msg": "Test message"
195 "tags": ["tag 1", "tag 2", "tag n"],
196 "conf-history": "DUT1 and DUT2 VAT History"
197 "show-run": "Show Run"
198 "status": "PASS" | "FAIL"
206 .. note:: ID is the lowercase full path to the test.
209 REGEX_PLR_RATE = re.compile(
210 r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
211 r'PLRsearch upper bound::?\s(\d+.\d+)'
213 REGEX_NDRPDR_RATE = re.compile(
214 r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
215 r'NDR_UPPER:\s(\d+.\d+).*\n'
216 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
217 r'PDR_UPPER:\s(\d+.\d+)'
219 REGEX_NDRPDR_GBPS = re.compile(
220 r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
221 r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
222 r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
223 r'PDR_UPPER:.*,\s(\d+.\d+)'
225 REGEX_PERF_MSG_INFO = re.compile(
226 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
227 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
228 r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
229 r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
230 r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
232 REGEX_CPS_MSG_INFO = re.compile(
233 r'NDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*\n'
234 r'PDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*'
236 REGEX_PPS_MSG_INFO = re.compile(
237 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
238 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*'
240 REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
242 # Needed for CPS and PPS tests
243 REGEX_NDRPDR_LAT_BASE = re.compile(
244 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
245 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
247 REGEX_NDRPDR_LAT = re.compile(
248 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
249 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
250 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
251 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
252 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
253 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
256 REGEX_VERSION_VPP = re.compile(
257 r"(return STDOUT Version:\s*|"
258 r"VPP Version:\s*|VPP version:\s*)(.*)"
260 REGEX_VERSION_DPDK = re.compile(
261 r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
263 REGEX_TCP = re.compile(
264 r'Total\s(rps|cps|throughput):\s(\d*).*$'
266 REGEX_MRR = re.compile(
267 r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
268 r'tx\s(\d*),\srx\s(\d*)'
270 REGEX_BMRR = re.compile(
271 r'.*trial results.*: \[(.*)\]'
273 REGEX_RECONF_LOSS = re.compile(
274 r'Packets lost due to reconfig: (\d*)'
276 REGEX_RECONF_TIME = re.compile(
277 r'Implied time lost: (\d*.[\de-]*)'
279 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
281 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
283 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
285 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
287 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
289 def __init__(self, metadata, mapping, ignore):
292 :param metadata: Key-value pairs to be included in "metadata" part of
294 :param mapping: Mapping of the old names of test cases to the new
296 :param ignore: List of TCs to be ignored.
302 # Type of message to parse out from the test messages
303 self._msg_type = None
309 self._timestamp = None
311 # Testbed. The testbed is identified by TG node IP address.
314 # Mapping of TCs long names
315 self._mapping = mapping
318 self._ignore = ignore
320 # Number of PAPI History messages found:
322 # 1 - PAPI History of DUT1
323 # 2 - PAPI History of DUT2
324 self._conf_history_lookup_nr = 0
326 self._sh_run_counter = 0
328 # Test ID of currently processed test- the lowercase full path to the
332 # The main data structure
334 u"metadata": OrderedDict(),
335 u"suites": OrderedDict(),
336 u"tests": OrderedDict()
339 # Save the provided metadata
340 for key, val in metadata.items():
341 self._data[u"metadata"][key] = val
343 # Dictionary defining the methods used to parse different types of
346 u"timestamp": self._get_timestamp,
347 u"vpp-version": self._get_vpp_version,
348 u"dpdk-version": self._get_dpdk_version,
349 u"teardown-papi-history": self._get_papi_history,
350 u"test-show-runtime": self._get_show_run,
351 u"testbed": self._get_testbed
356 """Getter - Data parsed from the XML file.
358 :returns: Data parsed from the XML file.
363 def _get_data_from_mrr_test_msg(self, msg):
364 """Get info from message of MRR performance tests.
366 :param msg: Message to be processed.
368 :returns: Processed message or original message if a problem occurs.
372 groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
373 if not groups or groups.lastindex != 1:
374 return u"Test Failed."
377 data = groups.group(1).split(u", ")
378 except (AttributeError, IndexError, ValueError, KeyError):
379 return u"Test Failed."
384 out_str += f"{(float(item) / 1e6):.2f}, "
385 return out_str[:-2] + u"]"
386 except (AttributeError, IndexError, ValueError, KeyError):
387 return u"Test Failed."
389 def _get_data_from_cps_test_msg(self, msg):
390 """Get info from message of NDRPDR CPS tests.
392 :param msg: Message to be processed.
394 :returns: Processed message or "Test Failed." if a problem occurs.
398 groups = re.search(self.REGEX_CPS_MSG_INFO, msg)
399 if not groups or groups.lastindex != 2:
400 return u"Test Failed."
404 f"1. {(float(groups.group(1)) / 1e6):5.2f}\n"
405 f"2. {(float(groups.group(2)) / 1e6):5.2f}"
407 except (AttributeError, IndexError, ValueError, KeyError):
408 return u"Test Failed."
410 def _get_data_from_pps_test_msg(self, msg):
411 """Get info from message of NDRPDR PPS tests.
413 :param msg: Message to be processed.
415 :returns: Processed message or "Test Failed." if a problem occurs.
419 groups = re.search(self.REGEX_PPS_MSG_INFO, msg)
420 if not groups or groups.lastindex != 4:
421 return u"Test Failed."
425 f"1. {(float(groups.group(1)) / 1e6):5.2f} "
426 f"{float(groups.group(2)):5.2f}\n"
427 f"2. {(float(groups.group(3)) / 1e6):5.2f} "
428 f"{float(groups.group(4)):5.2f}"
430 except (AttributeError, IndexError, ValueError, KeyError):
431 return u"Test Failed."
433 def _get_data_from_perf_test_msg(self, msg):
434 """Get info from message of NDRPDR performance tests.
436 :param msg: Message to be processed.
438 :returns: Processed message or "Test Failed." if a problem occurs.
442 groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
443 if not groups or groups.lastindex != 10:
444 return u"Test Failed."
448 u"ndr_low": float(groups.group(1)),
449 u"ndr_low_b": float(groups.group(2)),
450 u"pdr_low": float(groups.group(3)),
451 u"pdr_low_b": float(groups.group(4)),
452 u"pdr_lat_90_1": groups.group(5),
453 u"pdr_lat_90_2": groups.group(6),
454 u"pdr_lat_50_1": groups.group(7),
455 u"pdr_lat_50_2": groups.group(8),
456 u"pdr_lat_10_1": groups.group(9),
457 u"pdr_lat_10_2": groups.group(10),
459 except (AttributeError, IndexError, ValueError, KeyError):
460 return u"Test Failed."
462 def _process_lat(in_str_1, in_str_2):
463 """Extract min, avg, max values from latency string.
465 :param in_str_1: Latency string for one direction produced by robot
467 :param in_str_2: Latency string for second direction produced by
471 :returns: Processed latency string or None if a problem occurs.
474 in_list_1 = in_str_1.split('/', 3)
475 in_list_2 = in_str_2.split('/', 3)
477 if len(in_list_1) != 4 and len(in_list_2) != 4:
480 in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
482 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
483 except hdrh.codec.HdrLengthException:
486 in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
488 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
489 except hdrh.codec.HdrLengthException:
492 if hdr_lat_1 and hdr_lat_2:
494 hdr_lat_1.get_value_at_percentile(50.0),
495 hdr_lat_1.get_value_at_percentile(90.0),
496 hdr_lat_1.get_value_at_percentile(99.0),
497 hdr_lat_2.get_value_at_percentile(50.0),
498 hdr_lat_2.get_value_at_percentile(90.0),
499 hdr_lat_2.get_value_at_percentile(99.0)
509 f"1. {(data[u'ndr_low'] / 1e6):5.2f} "
510 f"{data[u'ndr_low_b']:5.2f}"
511 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f} "
512 f"{data[u'pdr_low_b']:5.2f}"
515 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
516 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
517 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
520 max_len = len(str(max((max(item) for item in latency))))
521 max_len = 4 if max_len < 4 else max_len
523 for idx, lat in enumerate(latency):
528 f"{lat[0]:{max_len}d} "
529 f"{lat[1]:{max_len}d} "
530 f"{lat[2]:{max_len}d} "
531 f"{lat[3]:{max_len}d} "
532 f"{lat[4]:{max_len}d} "
533 f"{lat[5]:{max_len}d} "
538 except (AttributeError, IndexError, ValueError, KeyError):
539 return u"Test Failed."
541 def _get_testbed(self, msg):
542 """Called when extraction of testbed IP is required.
543 The testbed is identified by TG node IP address.
545 :param msg: Message to process.
550 if msg.message.count(u"Setup of TG node") or \
551 msg.message.count(u"Setup of node TG host"):
552 reg_tg_ip = re.compile(
553 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
555 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
556 except (KeyError, ValueError, IndexError, AttributeError):
559 self._data[u"metadata"][u"testbed"] = self._testbed
560 self._msg_type = None
562 def _get_vpp_version(self, msg):
563 """Called when extraction of VPP version is required.
565 :param msg: Message to process.
570 if msg.message.count(u"return STDOUT Version:") or \
571 msg.message.count(u"VPP Version:") or \
572 msg.message.count(u"VPP version:"):
574 re.search(self.REGEX_VERSION_VPP, msg.message).group(2)
576 self._data[u"metadata"][u"version"] = self._version
577 self._msg_type = None
579 def _get_dpdk_version(self, msg):
580 """Called when extraction of DPDK version is required.
582 :param msg: Message to process.
587 if msg.message.count(u"DPDK Version:"):
589 self._version = str(re.search(
590 self.REGEX_VERSION_DPDK, msg.message).group(2))
591 self._data[u"metadata"][u"version"] = self._version
595 self._msg_type = None
597 def _get_timestamp(self, msg):
598 """Called when extraction of timestamp is required.
600 :param msg: Message to process.
605 self._timestamp = msg.timestamp[:14]
606 self._data[u"metadata"][u"generated"] = self._timestamp
607 self._msg_type = None
609 def _get_papi_history(self, msg):
610 """Called when extraction of PAPI command history is required.
612 :param msg: Message to process.
616 if msg.message.count(u"PAPI command history:"):
617 self._conf_history_lookup_nr += 1
618 if self._conf_history_lookup_nr == 1:
619 self._data[u"tests"][self._test_id][u"conf-history"] = str()
621 self._msg_type = None
623 r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} PAPI command history:",
627 ).replace(u'"', u"'")
628 self._data[u"tests"][self._test_id][u"conf-history"] += (
629 f"**DUT{str(self._conf_history_lookup_nr)}:** {text}"
632 def _get_show_run(self, msg):
633 """Called when extraction of VPP operational data (output of CLI command
634 Show Runtime) is required.
636 :param msg: Message to process.
641 if not msg.message.count(u"stats runtime"):
645 if self._sh_run_counter > 1:
648 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
649 self._data[u"tests"][self._test_id][u"show-run"] = dict()
651 groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
655 host = groups.group(1)
656 except (AttributeError, IndexError):
659 sock = groups.group(2)
660 except (AttributeError, IndexError):
663 runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
664 replace(u"'", u'"').replace(u'b"', u'"').
665 replace(u'u"', u'"').split(u":", 1)[1])
668 threads_nr = len(runtime[0][u"clocks"])
669 except (IndexError, KeyError):
672 dut = u"dut{nr}".format(
673 nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
678 # Needed for json converter, enable when 'threads' is gone.
679 # u"runtime": runtime,
680 u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
684 for idx in range(threads_nr):
685 if item[u"vectors"][idx] > 0:
686 clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
687 elif item[u"calls"][idx] > 0:
688 clocks = item[u"clocks"][idx] / item[u"calls"][idx]
689 elif item[u"suspends"][idx] > 0:
690 clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
694 if item[u"calls"][idx] > 0:
695 vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
699 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
700 int(item[u"suspends"][idx]):
701 oper[u"threads"][idx].append([
704 item[u"vectors"][idx],
705 item[u"suspends"][idx],
710 self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
712 def _get_ndrpdr_throughput(self, msg):
713 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
716 :param msg: The test message to be parsed.
718 :returns: Parsed data as a dict and the status (PASS/FAIL).
719 :rtype: tuple(dict, str)
723 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
724 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
727 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
729 if groups is not None:
731 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
732 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
733 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
734 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
736 except (IndexError, ValueError):
739 return throughput, status
741 def _get_ndrpdr_throughput_gbps(self, msg):
742 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
745 :param msg: The test message to be parsed.
747 :returns: Parsed data as a dict and the status (PASS/FAIL).
748 :rtype: tuple(dict, str)
752 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
753 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
756 groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
758 if groups is not None:
760 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
761 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
762 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
763 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
765 except (IndexError, ValueError):
770 def _get_plr_throughput(self, msg):
771 """Get PLRsearch lower bound and PLRsearch upper bound from the test
774 :param msg: The test message to be parsed.
776 :returns: Parsed data as a dict and the status (PASS/FAIL).
777 :rtype: tuple(dict, str)
785 groups = re.search(self.REGEX_PLR_RATE, msg)
787 if groups is not None:
789 throughput[u"LOWER"] = float(groups.group(1))
790 throughput[u"UPPER"] = float(groups.group(2))
792 except (IndexError, ValueError):
795 return throughput, status
797 def _get_ndrpdr_latency(self, msg):
798 """Get LATENCY from the test message.
800 :param msg: The test message to be parsed.
802 :returns: Parsed data as a dict and the status (PASS/FAIL).
803 :rtype: tuple(dict, str)
813 u"direction1": copy.copy(latency_default),
814 u"direction2": copy.copy(latency_default)
817 u"direction1": copy.copy(latency_default),
818 u"direction2": copy.copy(latency_default)
821 u"direction1": copy.copy(latency_default),
822 u"direction2": copy.copy(latency_default)
825 u"direction1": copy.copy(latency_default),
826 u"direction2": copy.copy(latency_default)
829 u"direction1": copy.copy(latency_default),
830 u"direction2": copy.copy(latency_default)
833 u"direction1": copy.copy(latency_default),
834 u"direction2": copy.copy(latency_default)
838 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
840 groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
842 return latency, u"FAIL"
844 def process_latency(in_str):
845 """Return object with parsed latency values.
847 TODO: Define class for the return type.
849 :param in_str: Input string, min/avg/max/hdrh format.
851 :returns: Dict with corresponding keys, except hdrh float values.
853 :throws IndexError: If in_str does not have enough substrings.
854 :throws ValueError: If a substring does not convert to float.
856 in_list = in_str.split('/', 3)
859 u"min": float(in_list[0]),
860 u"avg": float(in_list[1]),
861 u"max": float(in_list[2]),
865 if len(in_list) == 4:
866 rval[u"hdrh"] = str(in_list[3])
871 latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
872 latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
873 latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
874 latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
875 if groups.lastindex == 4:
876 return latency, u"PASS"
877 except (IndexError, ValueError):
881 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
882 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
883 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
884 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
885 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
886 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
887 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
888 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
889 if groups.lastindex == 12:
890 return latency, u"PASS"
891 except (IndexError, ValueError):
894 return latency, u"FAIL"
897 def _get_hoststack_data(msg, tags):
898 """Get data from the hoststack test message.
900 :param msg: The test message to be parsed.
901 :param tags: Test tags.
904 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
905 :rtype: tuple(dict, str)
910 msg = msg.replace(u"'", u'"').replace(u" ", u"")
911 if u"LDPRELOAD" in tags:
915 except JSONDecodeError:
917 elif u"VPPECHO" in tags:
919 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
921 client=loads(msg_lst[0]),
922 server=loads(msg_lst[1])
925 except (JSONDecodeError, IndexError):
928 return result, status
930 def visit_suite(self, suite):
931 """Implements traversing through the suite and its direct children.
933 :param suite: Suite to process.
937 if self.start_suite(suite) is not False:
938 suite.suites.visit(self)
939 suite.tests.visit(self)
940 self.end_suite(suite)
942 def start_suite(self, suite):
943 """Called when suite starts.
945 :param suite: Suite to process.
951 parent_name = suite.parent.name
952 except AttributeError:
955 self._data[u"suites"][suite.longname.lower().
957 replace(u" ", u"_")] = {
958 u"name": suite.name.lower(),
960 u"parent": parent_name,
961 u"level": len(suite.longname.split(u"."))
964 suite.keywords.visit(self)
966 def end_suite(self, suite):
967 """Called when suite ends.
969 :param suite: Suite to process.
974 def visit_test(self, test):
975 """Implements traversing through the test.
977 :param test: Test to process.
981 if self.start_test(test) is not False:
982 test.keywords.visit(self)
985 def start_test(self, test):
986 """Called when test starts.
988 :param test: Test to process.
993 self._sh_run_counter = 0
995 longname_orig = test.longname.lower()
997 # Check the ignore list
998 if longname_orig in self._ignore:
1001 tags = [str(tag) for tag in test.tags]
1002 test_result = dict()
1004 # Change the TC long name and name if defined in the mapping table
1005 longname = self._mapping.get(longname_orig, None)
1006 if longname is not None:
1007 name = longname.split(u'.')[-1]
1009 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1013 longname = longname_orig
1014 name = test.name.lower()
1016 # Remove TC number from the TC long name (backward compatibility):
1017 self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1018 # Remove TC number from the TC name (not needed):
1019 test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1021 test_result[u"parent"] = test.parent.name.lower()
1022 test_result[u"tags"] = tags
1023 test_result["doc"] = test.doc
1024 test_result[u"type"] = u""
1025 test_result[u"status"] = test.status
1026 test_result[u"starttime"] = test.starttime
1027 test_result[u"endtime"] = test.endtime
1029 if test.status == u"PASS":
1030 if u"NDRPDR" in tags:
1031 if u"TCP_PPS" in tags or u"UDP_PPS" in tags:
1032 test_result[u"msg"] = self._get_data_from_pps_test_msg(
1034 elif u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1035 test_result[u"msg"] = self._get_data_from_cps_test_msg(
1038 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1040 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1041 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1044 test_result[u"msg"] = test.message
1046 test_result[u"msg"] = test.message
1048 if u"PERFTEST" in tags:
1049 # Replace info about cores (e.g. -1c-) with the info about threads
1050 # and cores (e.g. -1t1c-) in the long test case names and in the
1051 # test case names if necessary.
1052 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1056 for tag in test_result[u"tags"]:
1057 groups = re.search(self.REGEX_TC_TAG, tag)
1063 self._test_id = re.sub(
1064 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1065 self._test_id, count=1
1067 test_result[u"name"] = re.sub(
1068 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1069 test_result["name"], count=1
1072 test_result[u"status"] = u"FAIL"
1073 self._data[u"tests"][self._test_id] = test_result
1075 f"The test {self._test_id} has no or more than one "
1076 f"multi-threading tags.\n"
1077 f"Tags: {test_result[u'tags']}"
1081 if u"DEVICETEST" in tags:
1082 test_result[u"type"] = u"DEVICETEST"
1083 elif u"NDRPDR" in tags:
1084 if u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1085 test_result[u"type"] = u"CPS"
1087 test_result[u"type"] = u"NDRPDR"
1088 if test.status == u"PASS":
1089 test_result[u"throughput"], test_result[u"status"] = \
1090 self._get_ndrpdr_throughput(test.message)
1091 test_result[u"gbps"], test_result[u"status"] = \
1092 self._get_ndrpdr_throughput_gbps(test.message)
1093 test_result[u"latency"], test_result[u"status"] = \
1094 self._get_ndrpdr_latency(test.message)
1095 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1097 test_result[u"type"] = u"MRR"
1099 test_result[u"type"] = u"BMRR"
1100 if test.status == u"PASS":
1101 test_result[u"result"] = dict()
1102 groups = re.search(self.REGEX_BMRR, test.message)
1103 if groups is not None:
1104 items_str = groups.group(1)
1106 float(item.strip().replace(u"'", u""))
1107 for item in items_str.split(",")
1109 # Use whole list in CSIT-1180.
1110 stats = jumpavg.AvgStdevStats.for_runs(items_float)
1111 test_result[u"result"][u"samples"] = items_float
1112 test_result[u"result"][u"receive-rate"] = stats.avg
1113 test_result[u"result"][u"receive-stdev"] = stats.stdev
1115 groups = re.search(self.REGEX_MRR, test.message)
1116 test_result[u"result"][u"receive-rate"] = \
1117 float(groups.group(3)) / float(groups.group(1))
1118 elif u"SOAK" in tags:
1119 test_result[u"type"] = u"SOAK"
1120 if test.status == u"PASS":
1121 test_result[u"throughput"], test_result[u"status"] = \
1122 self._get_plr_throughput(test.message)
1123 elif u"HOSTSTACK" in tags:
1124 test_result[u"type"] = u"HOSTSTACK"
1125 if test.status == u"PASS":
1126 test_result[u"result"], test_result[u"status"] = \
1127 self._get_hoststack_data(test.message, tags)
1128 # elif u"TCP" in tags: # This might be not used
1129 # test_result[u"type"] = u"TCP"
1130 # if test.status == u"PASS":
1131 # groups = re.search(self.REGEX_TCP, test.message)
1132 # test_result[u"result"] = int(groups.group(2))
1133 elif u"RECONF" in tags:
1134 test_result[u"type"] = u"RECONF"
1135 if test.status == u"PASS":
1136 test_result[u"result"] = None
1138 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1139 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1140 test_result[u"result"] = {
1141 u"loss": int(grps_loss.group(1)),
1142 u"time": float(grps_time.group(1))
1144 except (AttributeError, IndexError, ValueError, TypeError):
1145 test_result[u"status"] = u"FAIL"
1147 test_result[u"status"] = u"FAIL"
1149 self._data[u"tests"][self._test_id] = test_result
1151 def end_test(self, test):
1152 """Called when test ends.
1154 :param test: Test to process.
1159 def visit_keyword(self, keyword):
1160 """Implements traversing through the keyword and its child keywords.
1162 :param keyword: Keyword to process.
1163 :type keyword: Keyword
1166 if self.start_keyword(keyword) is not False:
1167 self.end_keyword(keyword)
1169 def start_keyword(self, keyword):
1170 """Called when keyword starts. Default implementation does nothing.
1172 :param keyword: Keyword to process.
1173 :type keyword: Keyword
1177 if keyword.type == u"setup":
1178 self.visit_setup_kw(keyword)
1179 elif keyword.type == u"teardown":
1180 self.visit_teardown_kw(keyword)
1182 self.visit_test_kw(keyword)
1183 except AttributeError:
1186 def end_keyword(self, keyword):
1187 """Called when keyword ends. Default implementation does nothing.
1189 :param keyword: Keyword to process.
1190 :type keyword: Keyword
1194 def visit_test_kw(self, test_kw):
1195 """Implements traversing through the test keyword and its child
1198 :param test_kw: Keyword to process.
1199 :type test_kw: Keyword
1202 for keyword in test_kw.keywords:
1203 if self.start_test_kw(keyword) is not False:
1204 self.visit_test_kw(keyword)
1205 self.end_test_kw(keyword)
1207 def start_test_kw(self, test_kw):
1208 """Called when test keyword starts. Default implementation does
1211 :param test_kw: Keyword to process.
1212 :type test_kw: Keyword
1215 if test_kw.name.count(u"Show Runtime On All Duts") or \
1216 test_kw.name.count(u"Show Runtime Counters On All Duts") or \
1217 test_kw.name.count(u"Vpp Show Runtime On All Duts"):
1218 self._msg_type = u"test-show-runtime"
1219 self._sh_run_counter += 1
1222 test_kw.messages.visit(self)
1224 def end_test_kw(self, test_kw):
1225 """Called when keyword ends. Default implementation does nothing.
1227 :param test_kw: Keyword to process.
1228 :type test_kw: Keyword
1232 def visit_setup_kw(self, setup_kw):
1233 """Implements traversing through the teardown keyword and its child
1236 :param setup_kw: Keyword to process.
1237 :type setup_kw: Keyword
1240 for keyword in setup_kw.keywords:
1241 if self.start_setup_kw(keyword) is not False:
1242 self.visit_setup_kw(keyword)
1243 self.end_setup_kw(keyword)
1245 def start_setup_kw(self, setup_kw):
1246 """Called when teardown keyword starts. Default implementation does
1249 :param setup_kw: Keyword to process.
1250 :type setup_kw: Keyword
1253 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1254 and not self._version:
1255 self._msg_type = u"vpp-version"
1256 elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1258 self._msg_type = u"dpdk-version"
1259 elif setup_kw.name.count(u"Set Global Variable") \
1260 and not self._timestamp:
1261 self._msg_type = u"timestamp"
1262 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1263 self._msg_type = u"testbed"
1266 setup_kw.messages.visit(self)
1268 def end_setup_kw(self, setup_kw):
1269 """Called when keyword ends. Default implementation does nothing.
1271 :param setup_kw: Keyword to process.
1272 :type setup_kw: Keyword
1276 def visit_teardown_kw(self, teardown_kw):
1277 """Implements traversing through the teardown keyword and its child
1280 :param teardown_kw: Keyword to process.
1281 :type teardown_kw: Keyword
1284 for keyword in teardown_kw.keywords:
1285 if self.start_teardown_kw(keyword) is not False:
1286 self.visit_teardown_kw(keyword)
1287 self.end_teardown_kw(keyword)
1289 def start_teardown_kw(self, teardown_kw):
1290 """Called when teardown keyword starts
1292 :param teardown_kw: Keyword to process.
1293 :type teardown_kw: Keyword
1296 if teardown_kw.name.count(u"Show Papi History On All Duts"):
1297 self._conf_history_lookup_nr = 0
1298 self._msg_type = u"teardown-papi-history"
1299 teardown_kw.messages.visit(self)
1301 def end_teardown_kw(self, teardown_kw):
1302 """Called when keyword ends. Default implementation does nothing.
1304 :param teardown_kw: Keyword to process.
1305 :type teardown_kw: Keyword
1309 def visit_message(self, msg):
1310 """Implements visiting the message.
1312 :param msg: Message to process.
1316 if self.start_message(msg) is not False:
1317 self.end_message(msg)
1319 def start_message(self, msg):
1320 """Called when message starts. Get required information from messages:
1323 :param msg: Message to process.
1328 self.parse_msg[self._msg_type](msg)
1330 def end_message(self, msg):
1331 """Called when message ends. Default implementation does nothing.
1333 :param msg: Message to process.
1342 The data is extracted from output.xml files generated by Jenkins jobs and
1343 stored in pandas' DataFrames.
1349 (as described in ExecutionChecker documentation)
1351 (as described in ExecutionChecker documentation)
1353 (as described in ExecutionChecker documentation)
1356 def __init__(self, spec):
1359 :param spec: Specification.
1360 :type spec: Specification
1367 self._input_data = pd.Series()
1371 """Getter - Input data.
1373 :returns: Input data
1374 :rtype: pandas.Series
1376 return self._input_data
1378 def metadata(self, job, build):
1379 """Getter - metadata
1381 :param job: Job which metadata we want.
1382 :param build: Build which metadata we want.
1386 :rtype: pandas.Series
1388 return self.data[job][build][u"metadata"]
1390 def suites(self, job, build):
1393 :param job: Job which suites we want.
1394 :param build: Build which suites we want.
1398 :rtype: pandas.Series
1400 return self.data[job][str(build)][u"suites"]
1402 def tests(self, job, build):
1405 :param job: Job which tests we want.
1406 :param build: Build which tests we want.
1410 :rtype: pandas.Series
1412 return self.data[job][build][u"tests"]
1414 def _parse_tests(self, job, build):
1415 """Process data from robot output.xml file and return JSON structured
1418 :param job: The name of job which build output data will be processed.
1419 :param build: The build which output data will be processed.
1422 :returns: JSON data structure.
1431 with open(build[u"file-name"], u'r') as data_file:
1433 result = ExecutionResult(data_file)
1434 except errors.DataError as err:
1436 f"Error occurred while parsing output.xml: {repr(err)}"
1439 checker = ExecutionChecker(
1440 metadata, self._cfg.mapping, self._cfg.ignore
1442 result.visit(checker)
1446 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1447 """Download and parse the input data file.
1449 :param pid: PID of the process executing this method.
1450 :param job: Name of the Jenkins job which generated the processed input
1452 :param build: Information about the Jenkins build which generated the
1453 processed input file.
1454 :param repeat: Repeat the download specified number of times if not
1462 logging.info(f"Processing the job/build: {job}: {build[u'build']}")
1469 success = download_and_unzip_data_file(self._cfg, job, build, pid)
1475 f"It is not possible to download the input data file from the "
1476 f"job {job}, build {build[u'build']}, or it is damaged. "
1480 logging.info(f" Processing data from build {build[u'build']}")
1481 data = self._parse_tests(job, build)
1484 f"Input data file from the job {job}, build "
1485 f"{build[u'build']} is damaged. Skipped."
1488 state = u"processed"
1491 remove(build[u"file-name"])
1492 except OSError as err:
1494 f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1497 # If the time-period is defined in the specification file, remove all
1498 # files which are outside the time period.
1500 timeperiod = self._cfg.environment.get(u"time-period", None)
1501 if timeperiod and data:
1503 timeperiod = timedelta(int(timeperiod))
1504 metadata = data.get(u"metadata", None)
1506 generated = metadata.get(u"generated", None)
1508 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1509 if (now - generated) > timeperiod:
1510 # Remove the data and the file:
1515 f" The build {job}/{build[u'build']} is "
1516 f"outdated, will be removed."
1526 def download_and_parse_data(self, repeat=1):
1527 """Download the input data files, parse input data from input files and
1528 store in pandas' Series.
1530 :param repeat: Repeat the download specified number of times if not
1535 logging.info(u"Downloading and parsing input files ...")
1537 for job, builds in self._cfg.input.items():
1538 for build in builds:
1540 result = self._download_and_parse_build(job, build, repeat)
1543 build_nr = result[u"build"][u"build"]
1546 data = result[u"data"]
1547 build_data = pd.Series({
1548 u"metadata": pd.Series(
1549 list(data[u"metadata"].values()),
1550 index=list(data[u"metadata"].keys())
1552 u"suites": pd.Series(
1553 list(data[u"suites"].values()),
1554 index=list(data[u"suites"].keys())
1556 u"tests": pd.Series(
1557 list(data[u"tests"].values()),
1558 index=list(data[u"tests"].keys())
1562 if self._input_data.get(job, None) is None:
1563 self._input_data[job] = pd.Series()
1564 self._input_data[job][str(build_nr)] = build_data
1565 self._cfg.set_input_file_name(
1566 job, build_nr, result[u"build"][u"file-name"]
1568 self._cfg.set_input_state(job, build_nr, result[u"state"])
1571 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1572 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1574 logging.info(u"Done.")
1576 msg = f"Successful downloads from the sources:\n"
1577 for source in self._cfg.environment[u"data-sources"]:
1578 if source[u"successful-downloads"]:
1580 f"{source[u'url']}/{source[u'path']}/"
1581 f"{source[u'file-name']}: "
1582 f"{source[u'successful-downloads']}\n"
1586 def process_local_file(self, local_file, job=u"local", build_nr=1,
1588 """Process local XML file given as a command-line parameter.
1590 :param local_file: The file to process.
1591 :param job: Job name.
1592 :param build_nr: Build number.
1593 :param replace: If True, the information about jobs and builds is
1594 replaced by the new one, otherwise the new jobs and builds are
1596 :type local_file: str
1600 :raises: PresentationError if an error occurs.
1602 if not isfile(local_file):
1603 raise PresentationError(f"The file {local_file} does not exist.")
1606 build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1607 except (IndexError, ValueError):
1612 u"status": u"failed",
1613 u"file-name": local_file
1616 self._cfg.input = dict()
1617 self._cfg.add_build(job, build)
1619 logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1620 data = self._parse_tests(job, build)
1622 raise PresentationError(
1623 f"Error occurred while parsing the file {local_file}"
1626 build_data = pd.Series({
1627 u"metadata": pd.Series(
1628 list(data[u"metadata"].values()),
1629 index=list(data[u"metadata"].keys())
1631 u"suites": pd.Series(
1632 list(data[u"suites"].values()),
1633 index=list(data[u"suites"].keys())
1635 u"tests": pd.Series(
1636 list(data[u"tests"].values()),
1637 index=list(data[u"tests"].keys())
1641 if self._input_data.get(job, None) is None:
1642 self._input_data[job] = pd.Series()
1643 self._input_data[job][str(build_nr)] = build_data
1645 self._cfg.set_input_state(job, build_nr, u"processed")
1647 def process_local_directory(self, local_dir, replace=True):
1648 """Process local directory with XML file(s). The directory is processed
1649 as a 'job' and the XML files in it as builds.
1650 If the given directory contains only sub-directories, these
1651 sub-directories processed as jobs and corresponding XML files as builds
1654 :param local_dir: Local directory to process.
1655 :param replace: If True, the information about jobs and builds is
1656 replaced by the new one, otherwise the new jobs and builds are
1658 :type local_dir: str
1661 if not isdir(local_dir):
1662 raise PresentationError(
1663 f"The directory {local_dir} does not exist."
1666 # Check if the given directory includes only files, or only directories
1667 _, dirnames, filenames = next(walk(local_dir))
1669 if filenames and not dirnames:
1672 # key: dir (job) name, value: list of file names (builds)
1674 local_dir: [join(local_dir, name) for name in filenames]
1677 elif dirnames and not filenames:
1680 # key: dir (job) name, value: list of file names (builds)
1681 local_builds = dict()
1682 for dirname in dirnames:
1684 join(local_dir, dirname, name)
1685 for name in listdir(join(local_dir, dirname))
1686 if isfile(join(local_dir, dirname, name))
1689 local_builds[dirname] = sorted(builds)
1691 elif not filenames and not dirnames:
1692 raise PresentationError(f"The directory {local_dir} is empty.")
1694 raise PresentationError(
1695 f"The directory {local_dir} can include only files or only "
1696 f"directories, not both.\nThe directory {local_dir} includes "
1697 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1701 self._cfg.input = dict()
1703 for job, files in local_builds.items():
1704 for idx, local_file in enumerate(files):
1705 self.process_local_file(local_file, job, idx + 1, replace=False)
1708 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1709 """Return the index of character in the string which is the end of tag.
1711 :param tag_filter: The string where the end of tag is being searched.
1712 :param start: The index where the searching is stated.
1713 :param closer: The character which is the tag closer.
1714 :type tag_filter: str
1717 :returns: The index of the tag closer.
1721 idx_opener = tag_filter.index(closer, start)
1722 return tag_filter.index(closer, idx_opener + 1)
1727 def _condition(tag_filter):
1728 """Create a conditional statement from the given tag filter.
1730 :param tag_filter: Filter based on tags from the element specification.
1731 :type tag_filter: str
1732 :returns: Conditional statement which can be evaluated.
1737 index = InputData._end_of_tag(tag_filter, index)
1741 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1743 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1744 continue_on_error=False):
1745 """Filter required data from the given jobs and builds.
1747 The output data structure is:
1750 - test (or suite) 1 ID:
1756 - test (or suite) n ID:
1763 :param element: Element which will use the filtered data.
1764 :param params: Parameters which will be included in the output. If None,
1765 all parameters are included.
1766 :param data: If not None, this data is used instead of data specified
1768 :param data_set: The set of data to be filtered: tests, suites,
1770 :param continue_on_error: Continue if there is error while reading the
1771 data. The Item will be empty then
1772 :type element: pandas.Series
1776 :type continue_on_error: bool
1777 :returns: Filtered data.
1778 :rtype pandas.Series
1782 if data_set == "suites":
1784 elif element[u"filter"] in (u"all", u"template"):
1787 cond = InputData._condition(element[u"filter"])
1788 logging.debug(f" Filter: {cond}")
1790 logging.error(u" No filter defined.")
1794 params = element.get(u"parameters", None)
1796 params.extend((u"type", u"status"))
1798 data_to_filter = data if data else element[u"data"]
1801 for job, builds in data_to_filter.items():
1802 data[job] = pd.Series()
1803 for build in builds:
1804 data[job][str(build)] = pd.Series()
1807 self.data[job][str(build)][data_set].items())
1809 if continue_on_error:
1813 for test_id, test_data in data_dict.items():
1814 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1815 data[job][str(build)][test_id] = pd.Series()
1817 for param, val in test_data.items():
1818 data[job][str(build)][test_id][param] = val
1820 for param in params:
1822 data[job][str(build)][test_id][param] =\
1825 data[job][str(build)][test_id][param] =\
1829 except (KeyError, IndexError, ValueError) as err:
1831 f"Missing mandatory parameter in the element specification: "
1835 except AttributeError as err:
1836 logging.error(repr(err))
1838 except SyntaxError as err:
1840 f"The filter {cond} is not correct. Check if all tags are "
1841 f"enclosed by apostrophes.\n{repr(err)}"
1845 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1846 continue_on_error=False):
1847 """Filter required data from the given jobs and builds.
1849 The output data structure is:
1852 - test (or suite) 1 ID:
1858 - test (or suite) n ID:
1865 :param element: Element which will use the filtered data.
1866 :param params: Parameters which will be included in the output. If None,
1867 all parameters are included.
1868 :param data_set: The set of data to be filtered: tests, suites,
1870 :param continue_on_error: Continue if there is error while reading the
1871 data. The Item will be empty then
1872 :type element: pandas.Series
1875 :type continue_on_error: bool
1876 :returns: Filtered data.
1877 :rtype pandas.Series
1880 include = element.get(u"include", None)
1882 logging.warning(u"No tests to include, skipping the element.")
1886 params = element.get(u"parameters", None)
1887 if params and u"type" not in params:
1888 params.append(u"type")
1890 cores = element.get(u"core", None)
1894 for test in include:
1895 tests.append(test.format(core=core))
1901 for job, builds in element[u"data"].items():
1902 data[job] = pd.Series()
1903 for build in builds:
1904 data[job][str(build)] = pd.Series()
1907 reg_ex = re.compile(str(test).lower())
1908 for test_id in self.data[job][
1909 str(build)][data_set].keys():
1910 if re.match(reg_ex, str(test_id).lower()):
1911 test_data = self.data[job][
1912 str(build)][data_set][test_id]
1913 data[job][str(build)][test_id] = pd.Series()
1915 for param, val in test_data.items():
1916 data[job][str(build)][test_id]\
1919 for param in params:
1921 data[job][str(build)][
1925 data[job][str(build)][
1926 test_id][param] = u"No Data"
1927 except KeyError as err:
1928 if continue_on_error:
1929 logging.debug(repr(err))
1931 logging.error(repr(err))
1935 except (KeyError, IndexError, ValueError) as err:
1937 f"Missing mandatory parameter in the element "
1938 f"specification: {repr(err)}"
1941 except AttributeError as err:
1942 logging.error(repr(err))
1946 def merge_data(data):
1947 """Merge data from more jobs and builds to a simple data structure.
1949 The output data structure is:
1951 - test (suite) 1 ID:
1957 - test (suite) n ID:
1960 :param data: Data to merge.
1961 :type data: pandas.Series
1962 :returns: Merged data.
1963 :rtype: pandas.Series
1966 logging.info(u" Merging data ...")
1968 merged_data = pd.Series()
1969 for builds in data.values:
1970 for item in builds.values:
1971 for item_id, item_data in item.items():
1972 merged_data[item_id] = item_data
1975 def print_all_oper_data(self):
1976 """Print all operational data to console.
1984 u"Cycles per Packet",
1985 u"Average Vector Size"
1988 for job in self._input_data.values:
1989 for build in job.values:
1990 for test_id, test_data in build[u"tests"].items():
1992 if test_data.get(u"show-run", None) is None:
1994 for dut_name, data in test_data[u"show-run"].items():
1995 if data.get(u"threads", None) is None:
1997 print(f"Host IP: {data.get(u'host', '')}, "
1998 f"Socket: {data.get(u'socket', '')}")
1999 for thread_nr, thread in data[u"threads"].items():
2000 txt_table = prettytable.PrettyTable(tbl_hdr)
2003 txt_table.add_row(row)
2005 if len(thread) == 0:
2008 avg = f", Average Vector Size per Node: " \
2009 f"{(avg / len(thread)):.2f}"
2010 th_name = u"main" if thread_nr == 0 \
2011 else f"worker_{thread_nr}"
2012 print(f"{dut_name}, {th_name}{avg}")
2013 txt_table.float_format = u".2"
2014 txt_table.align = u"r"
2015 txt_table.align[u"Name"] = u"l"
2016 print(f"{txt_table.get_string()}\n")