1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
28 from os import remove, walk, listdir
29 from os.path import isfile, isdir, join
30 from datetime import datetime as dt
31 from datetime import timedelta
32 from json import loads
33 from json.decoder import JSONDecodeError
40 from robot.api import ExecutionResult, ResultVisitor
41 from robot import errors
43 from resources.libraries.python import jumpavg
44 from input_data_files import download_and_unzip_data_file
45 from pal_errors import PresentationError
48 # Separator used in file names
52 class ExecutionChecker(ResultVisitor):
53 """Class to traverse through the test suite structure.
55 The functionality implemented in this class generates a json structure:
61 "generated": "Timestamp",
62 "version": "SUT version",
63 "job": "Jenkins job name",
64 "build": "Information about the build"
67 "Suite long name 1": {
69 "doc": "Suite 1 documentation",
70 "parent": "Suite 1 parent",
71 "level": "Level of the suite in the suite hierarchy"
73 "Suite long name N": {
75 "doc": "Suite N documentation",
76 "parent": "Suite 2 parent",
77 "level": "Level of the suite in the suite hierarchy"
84 "parent": "Name of the parent of the test",
85 "doc": "Test documentation",
86 "msg": "Test message",
87 "conf-history": "DUT1 and DUT2 VAT History",
88 "show-run": "Show Run",
89 "tags": ["tag 1", "tag 2", "tag n"],
91 "status": "PASS" | "FAIL",
137 "parent": "Name of the parent of the test",
138 "doc": "Test documentation",
139 "msg": "Test message",
140 "tags": ["tag 1", "tag 2", "tag n"],
142 "status": "PASS" | "FAIL",
149 "parent": "Name of the parent of the test",
150 "doc": "Test documentation",
151 "msg": "Test message",
152 "tags": ["tag 1", "tag 2", "tag n"],
153 "type": "MRR" | "BMRR",
154 "status": "PASS" | "FAIL",
156 "receive-rate": float,
157 # Average of a list, computed using AvgStdevStats.
158 # In CSIT-1180, replace with List[float].
172 "metadata": { # Optional
173 "version": "VPP version",
174 "job": "Jenkins job name",
175 "build": "Information about the build"
179 "doc": "Suite 1 documentation",
180 "parent": "Suite 1 parent",
181 "level": "Level of the suite in the suite hierarchy"
184 "doc": "Suite N documentation",
185 "parent": "Suite 2 parent",
186 "level": "Level of the suite in the suite hierarchy"
192 "parent": "Name of the parent of the test",
193 "doc": "Test documentation"
194 "msg": "Test message"
195 "tags": ["tag 1", "tag 2", "tag n"],
196 "conf-history": "DUT1 and DUT2 VAT History"
197 "show-run": "Show Run"
198 "status": "PASS" | "FAIL"
206 .. note:: ID is the lowercase full path to the test.
209 REGEX_PLR_RATE = re.compile(
210 r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
211 r'PLRsearch upper bound::?\s(\d+.\d+)'
213 REGEX_NDRPDR_RATE = re.compile(
214 r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
215 r'NDR_UPPER:\s(\d+.\d+).*\n'
216 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
217 r'PDR_UPPER:\s(\d+.\d+)'
219 REGEX_NDRPDR_GBPS = re.compile(
220 r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
221 r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
222 r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
223 r'PDR_UPPER:.*,\s(\d+.\d+)'
225 REGEX_PERF_MSG_INFO = re.compile(
226 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
227 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
228 r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
229 r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
230 r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
232 REGEX_CPS_MSG_INFO = re.compile(
233 r'NDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*\n'
234 r'PDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*'
236 REGEX_PPS_MSG_INFO = re.compile(
237 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
238 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*'
240 REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
242 REGEX_VSAP_MSG_INFO = re.compile(
243 r'Transfer Rate: (\d*.\d*).*\n'
244 r'Latency: (\d*.\d*).*\n'
245 r'Completed requests: (\d*).*\n'
246 r'Failed requests: (\d*).*\n'
247 r'Total data transferred: (\d*).*\n'
248 r'Connection [cr]ps rate:\s*(\d*.\d*)'
251 # Needed for CPS and PPS tests
252 REGEX_NDRPDR_LAT_BASE = re.compile(
253 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
254 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
256 REGEX_NDRPDR_LAT = re.compile(
257 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
258 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
259 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
260 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
261 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
262 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
265 REGEX_VERSION_VPP = re.compile(
266 r"(return STDOUT Version:\s*|"
267 r"VPP Version:\s*|VPP version:\s*)(.*)"
269 REGEX_VERSION_DPDK = re.compile(
270 r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
272 REGEX_TCP = re.compile(
273 r'Total\s(rps|cps|throughput):\s(\d*).*$'
275 REGEX_MRR = re.compile(
276 r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
277 r'tx\s(\d*),\srx\s(\d*)'
279 REGEX_BMRR = re.compile(
280 r'.*trial results.*: \[(.*)\]'
282 REGEX_RECONF_LOSS = re.compile(
283 r'Packets lost due to reconfig: (\d*)'
285 REGEX_RECONF_TIME = re.compile(
286 r'Implied time lost: (\d*.[\de-]*)'
288 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
290 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
292 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
294 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
296 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
298 def __init__(self, metadata, mapping, ignore, for_output):
301 :param metadata: Key-value pairs to be included in "metadata" part of
303 :param mapping: Mapping of the old names of test cases to the new
305 :param ignore: List of TCs to be ignored.
306 :param for_output: Output to be generated from downloaded data.
310 :type for_output: str
313 # Type of message to parse out from the test messages
314 self._msg_type = None
320 self._timestamp = None
322 # Testbed. The testbed is identified by TG node IP address.
325 # Mapping of TCs long names
326 self._mapping = mapping
329 self._ignore = ignore
331 self._for_output = for_output
333 # Number of PAPI History messages found:
335 # 1 - PAPI History of DUT1
336 # 2 - PAPI History of DUT2
337 self._conf_history_lookup_nr = 0
339 self._sh_run_counter = 0
341 # Test ID of currently processed test- the lowercase full path to the
345 # The main data structure
347 u"metadata": OrderedDict(),
348 u"suites": OrderedDict(),
349 u"tests": OrderedDict()
352 # Save the provided metadata
353 for key, val in metadata.items():
354 self._data[u"metadata"][key] = val
356 # Dictionary defining the methods used to parse different types of
359 u"timestamp": self._get_timestamp,
360 u"vpp-version": self._get_vpp_version,
361 u"dpdk-version": self._get_dpdk_version,
362 u"teardown-papi-history": self._get_papi_history,
363 u"test-show-runtime": self._get_show_run,
364 u"testbed": self._get_testbed
369 """Getter - Data parsed from the XML file.
371 :returns: Data parsed from the XML file.
376 def _get_data_from_mrr_test_msg(self, msg):
377 """Get info from message of MRR performance tests.
379 :param msg: Message to be processed.
381 :returns: Processed message or original message if a problem occurs.
385 groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
386 if not groups or groups.lastindex != 1:
387 return u"Test Failed."
390 data = groups.group(1).split(u", ")
391 except (AttributeError, IndexError, ValueError, KeyError):
392 return u"Test Failed."
397 out_str += f"{(float(item) / 1e6):.2f}, "
398 return out_str[:-2] + u"]"
399 except (AttributeError, IndexError, ValueError, KeyError):
400 return u"Test Failed."
402 def _get_data_from_cps_test_msg(self, msg):
403 """Get info from message of NDRPDR CPS tests.
405 :param msg: Message to be processed.
407 :returns: Processed message or "Test Failed." if a problem occurs.
411 groups = re.search(self.REGEX_CPS_MSG_INFO, msg)
412 if not groups or groups.lastindex != 2:
413 return u"Test Failed."
417 f"1. {(float(groups.group(1)) / 1e6):5.2f}\n"
418 f"2. {(float(groups.group(2)) / 1e6):5.2f}"
420 except (AttributeError, IndexError, ValueError, KeyError):
421 return u"Test Failed."
423 def _get_data_from_pps_test_msg(self, msg):
424 """Get info from message of NDRPDR PPS tests.
426 :param msg: Message to be processed.
428 :returns: Processed message or "Test Failed." if a problem occurs.
432 groups = re.search(self.REGEX_PPS_MSG_INFO, msg)
433 if not groups or groups.lastindex != 4:
434 return u"Test Failed."
438 f"1. {(float(groups.group(1)) / 1e6):5.2f} "
439 f"{float(groups.group(2)):5.2f}\n"
440 f"2. {(float(groups.group(3)) / 1e6):5.2f} "
441 f"{float(groups.group(4)):5.2f}"
443 except (AttributeError, IndexError, ValueError, KeyError):
444 return u"Test Failed."
446 def _get_data_from_perf_test_msg(self, msg):
447 """Get info from message of NDRPDR performance tests.
449 :param msg: Message to be processed.
451 :returns: Processed message or "Test Failed." if a problem occurs.
455 groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
456 if not groups or groups.lastindex != 10:
457 return u"Test Failed."
461 u"ndr_low": float(groups.group(1)),
462 u"ndr_low_b": float(groups.group(2)),
463 u"pdr_low": float(groups.group(3)),
464 u"pdr_low_b": float(groups.group(4)),
465 u"pdr_lat_90_1": groups.group(5),
466 u"pdr_lat_90_2": groups.group(6),
467 u"pdr_lat_50_1": groups.group(7),
468 u"pdr_lat_50_2": groups.group(8),
469 u"pdr_lat_10_1": groups.group(9),
470 u"pdr_lat_10_2": groups.group(10),
472 except (AttributeError, IndexError, ValueError, KeyError):
473 return u"Test Failed."
475 def _process_lat(in_str_1, in_str_2):
476 """Extract min, avg, max values from latency string.
478 :param in_str_1: Latency string for one direction produced by robot
480 :param in_str_2: Latency string for second direction produced by
484 :returns: Processed latency string or None if a problem occurs.
487 in_list_1 = in_str_1.split('/', 3)
488 in_list_2 = in_str_2.split('/', 3)
490 if len(in_list_1) != 4 and len(in_list_2) != 4:
493 in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
495 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
496 except hdrh.codec.HdrLengthException:
499 in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
501 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
502 except hdrh.codec.HdrLengthException:
505 if hdr_lat_1 and hdr_lat_2:
507 hdr_lat_1.get_value_at_percentile(50.0),
508 hdr_lat_1.get_value_at_percentile(90.0),
509 hdr_lat_1.get_value_at_percentile(99.0),
510 hdr_lat_2.get_value_at_percentile(50.0),
511 hdr_lat_2.get_value_at_percentile(90.0),
512 hdr_lat_2.get_value_at_percentile(99.0)
522 f"1. {(data[u'ndr_low'] / 1e6):5.2f} "
523 f"{data[u'ndr_low_b']:5.2f}"
524 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f} "
525 f"{data[u'pdr_low_b']:5.2f}"
528 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
529 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
530 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
533 max_len = len(str(max((max(item) for item in latency))))
534 max_len = 4 if max_len < 4 else max_len
536 for idx, lat in enumerate(latency):
541 f"{lat[0]:{max_len}d} "
542 f"{lat[1]:{max_len}d} "
543 f"{lat[2]:{max_len}d} "
544 f"{lat[3]:{max_len}d} "
545 f"{lat[4]:{max_len}d} "
546 f"{lat[5]:{max_len}d} "
551 except (AttributeError, IndexError, ValueError, KeyError):
552 return u"Test Failed."
554 def _get_testbed(self, msg):
555 """Called when extraction of testbed IP is required.
556 The testbed is identified by TG node IP address.
558 :param msg: Message to process.
563 if msg.message.count(u"Setup of TG node") or \
564 msg.message.count(u"Setup of node TG host"):
565 reg_tg_ip = re.compile(
566 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
568 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
569 except (KeyError, ValueError, IndexError, AttributeError):
572 self._data[u"metadata"][u"testbed"] = self._testbed
573 self._msg_type = None
575 def _get_vpp_version(self, msg):
576 """Called when extraction of VPP version is required.
578 :param msg: Message to process.
583 if msg.message.count(u"return STDOUT Version:") or \
584 msg.message.count(u"VPP Version:") or \
585 msg.message.count(u"VPP version:"):
587 re.search(self.REGEX_VERSION_VPP, msg.message).group(2)
589 self._data[u"metadata"][u"version"] = self._version
590 self._msg_type = None
592 def _get_dpdk_version(self, msg):
593 """Called when extraction of DPDK version is required.
595 :param msg: Message to process.
600 if msg.message.count(u"DPDK Version:"):
602 self._version = str(re.search(
603 self.REGEX_VERSION_DPDK, msg.message).group(2))
604 self._data[u"metadata"][u"version"] = self._version
608 self._msg_type = None
610 def _get_timestamp(self, msg):
611 """Called when extraction of timestamp is required.
613 :param msg: Message to process.
618 self._timestamp = msg.timestamp[:14]
619 self._data[u"metadata"][u"generated"] = self._timestamp
620 self._msg_type = None
622 def _get_papi_history(self, msg):
623 """Called when extraction of PAPI command history is required.
625 :param msg: Message to process.
629 if msg.message.count(u"PAPI command history:"):
630 self._conf_history_lookup_nr += 1
631 if self._conf_history_lookup_nr == 1:
632 self._data[u"tests"][self._test_id][u"conf-history"] = str()
634 self._msg_type = None
636 r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} PAPI command history:",
640 ).replace(u'"', u"'")
641 self._data[u"tests"][self._test_id][u"conf-history"] += (
642 f"**DUT{str(self._conf_history_lookup_nr)}:** {text}"
645 def _get_show_run(self, msg):
646 """Called when extraction of VPP operational data (output of CLI command
647 Show Runtime) is required.
649 :param msg: Message to process.
654 if not msg.message.count(u"stats runtime"):
658 if self._sh_run_counter > 1:
661 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
662 self._data[u"tests"][self._test_id][u"show-run"] = dict()
664 groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
668 host = groups.group(1)
669 except (AttributeError, IndexError):
672 sock = groups.group(2)
673 except (AttributeError, IndexError):
676 dut = u"dut{nr}".format(
677 nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
679 self._data[u'tests'][self._test_id][u'show-run'][dut] = \
684 u"runtime": str(msg.message).replace(u' ', u'').
685 replace(u'\n', u'').replace(u"'", u'"').
686 replace(u'b"', u'"').replace(u'u"', u'"').
691 def _get_ndrpdr_throughput(self, msg):
692 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
695 :param msg: The test message to be parsed.
697 :returns: Parsed data as a dict and the status (PASS/FAIL).
698 :rtype: tuple(dict, str)
702 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
703 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
706 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
708 if groups is not None:
710 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
711 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
712 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
713 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
715 except (IndexError, ValueError):
718 return throughput, status
720 def _get_ndrpdr_throughput_gbps(self, msg):
721 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
724 :param msg: The test message to be parsed.
726 :returns: Parsed data as a dict and the status (PASS/FAIL).
727 :rtype: tuple(dict, str)
731 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
732 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
735 groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
737 if groups is not None:
739 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
740 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
741 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
742 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
744 except (IndexError, ValueError):
749 def _get_plr_throughput(self, msg):
750 """Get PLRsearch lower bound and PLRsearch upper bound from the test
753 :param msg: The test message to be parsed.
755 :returns: Parsed data as a dict and the status (PASS/FAIL).
756 :rtype: tuple(dict, str)
764 groups = re.search(self.REGEX_PLR_RATE, msg)
766 if groups is not None:
768 throughput[u"LOWER"] = float(groups.group(1))
769 throughput[u"UPPER"] = float(groups.group(2))
771 except (IndexError, ValueError):
774 return throughput, status
776 def _get_ndrpdr_latency(self, msg):
777 """Get LATENCY from the test message.
779 :param msg: The test message to be parsed.
781 :returns: Parsed data as a dict and the status (PASS/FAIL).
782 :rtype: tuple(dict, str)
792 u"direction1": copy.copy(latency_default),
793 u"direction2": copy.copy(latency_default)
796 u"direction1": copy.copy(latency_default),
797 u"direction2": copy.copy(latency_default)
800 u"direction1": copy.copy(latency_default),
801 u"direction2": copy.copy(latency_default)
804 u"direction1": copy.copy(latency_default),
805 u"direction2": copy.copy(latency_default)
808 u"direction1": copy.copy(latency_default),
809 u"direction2": copy.copy(latency_default)
812 u"direction1": copy.copy(latency_default),
813 u"direction2": copy.copy(latency_default)
817 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
819 groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
821 return latency, u"FAIL"
823 def process_latency(in_str):
824 """Return object with parsed latency values.
826 TODO: Define class for the return type.
828 :param in_str: Input string, min/avg/max/hdrh format.
830 :returns: Dict with corresponding keys, except hdrh float values.
832 :throws IndexError: If in_str does not have enough substrings.
833 :throws ValueError: If a substring does not convert to float.
835 in_list = in_str.split('/', 3)
838 u"min": float(in_list[0]),
839 u"avg": float(in_list[1]),
840 u"max": float(in_list[2]),
844 if len(in_list) == 4:
845 rval[u"hdrh"] = str(in_list[3])
850 latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
851 latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
852 latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
853 latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
854 if groups.lastindex == 4:
855 return latency, u"PASS"
856 except (IndexError, ValueError):
860 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
861 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
862 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
863 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
864 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
865 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
866 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
867 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
868 if groups.lastindex == 12:
869 return latency, u"PASS"
870 except (IndexError, ValueError):
873 return latency, u"FAIL"
876 def _get_hoststack_data(msg, tags):
877 """Get data from the hoststack test message.
879 :param msg: The test message to be parsed.
880 :param tags: Test tags.
883 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
884 :rtype: tuple(dict, str)
889 msg = msg.replace(u"'", u'"').replace(u" ", u"")
890 if u"LDPRELOAD" in tags:
894 except JSONDecodeError:
896 elif u"VPPECHO" in tags:
898 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
900 client=loads(msg_lst[0]),
901 server=loads(msg_lst[1])
904 except (JSONDecodeError, IndexError):
907 return result, status
909 def _get_vsap_data(self, msg, tags):
910 """Get data from the vsap test message.
912 :param msg: The test message to be parsed.
913 :param tags: Test tags.
916 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
917 :rtype: tuple(dict, str)
922 groups = re.search(self.REGEX_VSAP_MSG_INFO, msg)
923 if groups is not None:
925 result[u"transfer-rate"] = float(groups.group(1)) * 1e3
926 result[u"latency"] = float(groups.group(2))
927 result[u"completed-requests"] = int(groups.group(3))
928 result[u"failed-requests"] = int(groups.group(4))
929 result[u"bytes-transferred"] = int(groups.group(5))
930 if u"TCP_CPS"in tags:
931 result[u"cps"] = float(groups.group(6))
932 elif u"TCP_RPS" in tags:
933 result[u"rps"] = float(groups.group(6))
935 return result, status
937 except (IndexError, ValueError):
940 return result, status
942 def visit_suite(self, suite):
943 """Implements traversing through the suite and its direct children.
945 :param suite: Suite to process.
949 if self.start_suite(suite) is not False:
950 suite.suites.visit(self)
951 suite.tests.visit(self)
952 self.end_suite(suite)
954 def start_suite(self, suite):
955 """Called when suite starts.
957 :param suite: Suite to process.
963 parent_name = suite.parent.name
964 except AttributeError:
967 self._data[u"suites"][suite.longname.lower().
969 replace(u" ", u"_")] = {
970 u"name": suite.name.lower(),
972 u"parent": parent_name,
973 u"level": len(suite.longname.split(u"."))
976 suite.keywords.visit(self)
978 def end_suite(self, suite):
979 """Called when suite ends.
981 :param suite: Suite to process.
986 def visit_test(self, test):
987 """Implements traversing through the test.
989 :param test: Test to process.
993 if self.start_test(test) is not False:
994 test.keywords.visit(self)
997 def start_test(self, test):
998 """Called when test starts.
1000 :param test: Test to process.
1005 self._sh_run_counter = 0
1007 longname_orig = test.longname.lower()
1009 # Check the ignore list
1010 if longname_orig in self._ignore:
1013 tags = [str(tag) for tag in test.tags]
1014 test_result = dict()
1016 # Change the TC long name and name if defined in the mapping table
1017 longname = self._mapping.get(longname_orig, None)
1018 if longname is not None:
1019 name = longname.split(u'.')[-1]
1021 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1025 longname = longname_orig
1026 name = test.name.lower()
1028 # Remove TC number from the TC long name (backward compatibility):
1029 self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1030 # Remove TC number from the TC name (not needed):
1031 test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1033 test_result[u"parent"] = test.parent.name.lower()
1034 test_result[u"tags"] = tags
1035 test_result["doc"] = test.doc
1036 test_result[u"type"] = u""
1037 test_result[u"status"] = test.status
1038 test_result[u"starttime"] = test.starttime
1039 test_result[u"endtime"] = test.endtime
1041 if test.status == u"PASS":
1042 if u"NDRPDR" in tags:
1043 if u"TCP_PPS" in tags or u"UDP_PPS" in tags:
1044 test_result[u"msg"] = self._get_data_from_pps_test_msg(
1046 elif u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1047 test_result[u"msg"] = self._get_data_from_cps_test_msg(
1050 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1052 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1053 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1056 test_result[u"msg"] = test.message
1058 test_result[u"msg"] = test.message
1060 if u"PERFTEST" in tags:
1061 # Replace info about cores (e.g. -1c-) with the info about threads
1062 # and cores (e.g. -1t1c-) in the long test case names and in the
1063 # test case names if necessary.
1064 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1068 for tag in test_result[u"tags"]:
1069 groups = re.search(self.REGEX_TC_TAG, tag)
1075 self._test_id = re.sub(
1076 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1077 self._test_id, count=1
1079 test_result[u"name"] = re.sub(
1080 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1081 test_result["name"], count=1
1084 test_result[u"status"] = u"FAIL"
1085 self._data[u"tests"][self._test_id] = test_result
1087 f"The test {self._test_id} has no or more than one "
1088 f"multi-threading tags.\n"
1089 f"Tags: {test_result[u'tags']}"
1093 if u"DEVICETEST" in tags:
1094 test_result[u"type"] = u"DEVICETEST"
1095 elif u"NDRPDR" in tags:
1096 if u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1097 test_result[u"type"] = u"CPS"
1099 test_result[u"type"] = u"NDRPDR"
1100 if test.status == u"PASS":
1101 test_result[u"throughput"], test_result[u"status"] = \
1102 self._get_ndrpdr_throughput(test.message)
1103 test_result[u"gbps"], test_result[u"status"] = \
1104 self._get_ndrpdr_throughput_gbps(test.message)
1105 test_result[u"latency"], test_result[u"status"] = \
1106 self._get_ndrpdr_latency(test.message)
1107 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1109 test_result[u"type"] = u"MRR"
1111 test_result[u"type"] = u"BMRR"
1112 if test.status == u"PASS":
1113 test_result[u"result"] = dict()
1114 groups = re.search(self.REGEX_BMRR, test.message)
1115 if groups is not None:
1116 items_str = groups.group(1)
1118 float(item.strip().replace(u"'", u""))
1119 for item in items_str.split(",")
1121 # Use whole list in CSIT-1180.
1122 stats = jumpavg.AvgStdevStats.for_runs(items_float)
1123 test_result[u"result"][u"samples"] = items_float
1124 test_result[u"result"][u"receive-rate"] = stats.avg
1125 test_result[u"result"][u"receive-stdev"] = stats.stdev
1127 groups = re.search(self.REGEX_MRR, test.message)
1128 test_result[u"result"][u"receive-rate"] = \
1129 float(groups.group(3)) / float(groups.group(1))
1130 elif u"SOAK" in tags:
1131 test_result[u"type"] = u"SOAK"
1132 if test.status == u"PASS":
1133 test_result[u"throughput"], test_result[u"status"] = \
1134 self._get_plr_throughput(test.message)
1135 elif u"HOSTSTACK" in tags:
1136 test_result[u"type"] = u"HOSTSTACK"
1137 if test.status == u"PASS":
1138 test_result[u"result"], test_result[u"status"] = \
1139 self._get_hoststack_data(test.message, tags)
1140 elif u"LDP_NGINX" in tags:
1141 test_result[u"type"] = u"LDP_NGINX"
1142 test_result[u"result"], test_result[u"status"] = \
1143 self._get_vsap_data(test.message, tags)
1144 # elif u"TCP" in tags: # This might be not used
1145 # test_result[u"type"] = u"TCP"
1146 # if test.status == u"PASS":
1147 # groups = re.search(self.REGEX_TCP, test.message)
1148 # test_result[u"result"] = int(groups.group(2))
1149 elif u"RECONF" in tags:
1150 test_result[u"type"] = u"RECONF"
1151 if test.status == u"PASS":
1152 test_result[u"result"] = None
1154 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1155 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1156 test_result[u"result"] = {
1157 u"loss": int(grps_loss.group(1)),
1158 u"time": float(grps_time.group(1))
1160 except (AttributeError, IndexError, ValueError, TypeError):
1161 test_result[u"status"] = u"FAIL"
1163 test_result[u"status"] = u"FAIL"
1165 self._data[u"tests"][self._test_id] = test_result
1167 def end_test(self, test):
1168 """Called when test ends.
1170 :param test: Test to process.
1175 def visit_keyword(self, keyword):
1176 """Implements traversing through the keyword and its child keywords.
1178 :param keyword: Keyword to process.
1179 :type keyword: Keyword
1182 if self.start_keyword(keyword) is not False:
1183 self.end_keyword(keyword)
1185 def start_keyword(self, keyword):
1186 """Called when keyword starts. Default implementation does nothing.
1188 :param keyword: Keyword to process.
1189 :type keyword: Keyword
1193 if keyword.type == u"setup":
1194 self.visit_setup_kw(keyword)
1195 elif keyword.type == u"teardown":
1196 self.visit_teardown_kw(keyword)
1198 self.visit_test_kw(keyword)
1199 except AttributeError:
1202 def end_keyword(self, keyword):
1203 """Called when keyword ends. Default implementation does nothing.
1205 :param keyword: Keyword to process.
1206 :type keyword: Keyword
1210 def visit_test_kw(self, test_kw):
1211 """Implements traversing through the test keyword and its child
1214 :param test_kw: Keyword to process.
1215 :type test_kw: Keyword
1218 for keyword in test_kw.keywords:
1219 if self.start_test_kw(keyword) is not False:
1220 self.visit_test_kw(keyword)
1221 self.end_test_kw(keyword)
1223 def start_test_kw(self, test_kw):
1224 """Called when test keyword starts. Default implementation does
1227 :param test_kw: Keyword to process.
1228 :type test_kw: Keyword
1231 if ((self._for_output != u"trending") and
1232 (test_kw.name.count(u"Show Runtime On All Duts") or
1233 test_kw.name.count(u"Show Runtime Counters On All Duts") or
1234 test_kw.name.count(u"Vpp Show Runtime On All Duts"))):
1235 self._msg_type = u"test-show-runtime"
1236 self._sh_run_counter += 1
1239 test_kw.messages.visit(self)
1241 def end_test_kw(self, test_kw):
1242 """Called when keyword ends. Default implementation does nothing.
1244 :param test_kw: Keyword to process.
1245 :type test_kw: Keyword
1249 def visit_setup_kw(self, setup_kw):
1250 """Implements traversing through the teardown keyword and its child
1253 :param setup_kw: Keyword to process.
1254 :type setup_kw: Keyword
1257 for keyword in setup_kw.keywords:
1258 if self.start_setup_kw(keyword) is not False:
1259 self.visit_setup_kw(keyword)
1260 self.end_setup_kw(keyword)
1262 def start_setup_kw(self, setup_kw):
1263 """Called when teardown keyword starts. Default implementation does
1266 :param setup_kw: Keyword to process.
1267 :type setup_kw: Keyword
1270 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1271 and not self._version:
1272 self._msg_type = u"vpp-version"
1273 elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1275 self._msg_type = u"dpdk-version"
1276 elif setup_kw.name.count(u"Set Global Variable") \
1277 and not self._timestamp:
1278 self._msg_type = u"timestamp"
1279 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1280 self._msg_type = u"testbed"
1283 setup_kw.messages.visit(self)
1285 def end_setup_kw(self, setup_kw):
1286 """Called when keyword ends. Default implementation does nothing.
1288 :param setup_kw: Keyword to process.
1289 :type setup_kw: Keyword
1293 def visit_teardown_kw(self, teardown_kw):
1294 """Implements traversing through the teardown keyword and its child
1297 :param teardown_kw: Keyword to process.
1298 :type teardown_kw: Keyword
1301 for keyword in teardown_kw.keywords:
1302 if self.start_teardown_kw(keyword) is not False:
1303 self.visit_teardown_kw(keyword)
1304 self.end_teardown_kw(keyword)
1306 def start_teardown_kw(self, teardown_kw):
1307 """Called when teardown keyword starts
1309 :param teardown_kw: Keyword to process.
1310 :type teardown_kw: Keyword
1313 if teardown_kw.name.count(u"Show Papi History On All Duts"):
1314 self._conf_history_lookup_nr = 0
1315 self._msg_type = u"teardown-papi-history"
1316 teardown_kw.messages.visit(self)
1318 def end_teardown_kw(self, teardown_kw):
1319 """Called when keyword ends. Default implementation does nothing.
1321 :param teardown_kw: Keyword to process.
1322 :type teardown_kw: Keyword
1326 def visit_message(self, msg):
1327 """Implements visiting the message.
1329 :param msg: Message to process.
1333 if self.start_message(msg) is not False:
1334 self.end_message(msg)
1336 def start_message(self, msg):
1337 """Called when message starts. Get required information from messages:
1340 :param msg: Message to process.
1345 self.parse_msg[self._msg_type](msg)
1347 def end_message(self, msg):
1348 """Called when message ends. Default implementation does nothing.
1350 :param msg: Message to process.
1359 The data is extracted from output.xml files generated by Jenkins jobs and
1360 stored in pandas' DataFrames.
1366 (as described in ExecutionChecker documentation)
1368 (as described in ExecutionChecker documentation)
1370 (as described in ExecutionChecker documentation)
1373 def __init__(self, spec, for_output):
1376 :param spec: Specification.
1377 :param for_output: Output to be generated from downloaded data.
1378 :type spec: Specification
1379 :type for_output: str
1385 self._for_output = for_output
1388 self._input_data = pd.Series()
1392 """Getter - Input data.
1394 :returns: Input data
1395 :rtype: pandas.Series
1397 return self._input_data
1399 def metadata(self, job, build):
1400 """Getter - metadata
1402 :param job: Job which metadata we want.
1403 :param build: Build which metadata we want.
1407 :rtype: pandas.Series
1409 return self.data[job][build][u"metadata"]
1411 def suites(self, job, build):
1414 :param job: Job which suites we want.
1415 :param build: Build which suites we want.
1419 :rtype: pandas.Series
1421 return self.data[job][str(build)][u"suites"]
1423 def tests(self, job, build):
1426 :param job: Job which tests we want.
1427 :param build: Build which tests we want.
1431 :rtype: pandas.Series
1433 return self.data[job][build][u"tests"]
1435 def _parse_tests(self, job, build):
1436 """Process data from robot output.xml file and return JSON structured
1439 :param job: The name of job which build output data will be processed.
1440 :param build: The build which output data will be processed.
1443 :returns: JSON data structure.
1452 with open(build[u"file-name"], u'r') as data_file:
1454 result = ExecutionResult(data_file)
1455 except errors.DataError as err:
1457 f"Error occurred while parsing output.xml: {repr(err)}"
1460 checker = ExecutionChecker(
1461 metadata, self._cfg.mapping, self._cfg.ignore, self._for_output
1463 result.visit(checker)
1467 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1468 """Download and parse the input data file.
1470 :param pid: PID of the process executing this method.
1471 :param job: Name of the Jenkins job which generated the processed input
1473 :param build: Information about the Jenkins build which generated the
1474 processed input file.
1475 :param repeat: Repeat the download specified number of times if not
1483 logging.info(f"Processing the job/build: {job}: {build[u'build']}")
1490 success = download_and_unzip_data_file(self._cfg, job, build, pid)
1496 f"It is not possible to download the input data file from the "
1497 f"job {job}, build {build[u'build']}, or it is damaged. "
1501 logging.info(f" Processing data from build {build[u'build']}")
1502 data = self._parse_tests(job, build)
1505 f"Input data file from the job {job}, build "
1506 f"{build[u'build']} is damaged. Skipped."
1509 state = u"processed"
1512 remove(build[u"file-name"])
1513 except OSError as err:
1515 f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1518 # If the time-period is defined in the specification file, remove all
1519 # files which are outside the time period.
1521 timeperiod = self._cfg.environment.get(u"time-period", None)
1522 if timeperiod and data:
1524 timeperiod = timedelta(int(timeperiod))
1525 metadata = data.get(u"metadata", None)
1527 generated = metadata.get(u"generated", None)
1529 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1530 if (now - generated) > timeperiod:
1531 # Remove the data and the file:
1536 f" The build {job}/{build[u'build']} is "
1537 f"outdated, will be removed."
1547 def download_and_parse_data(self, repeat=1):
1548 """Download the input data files, parse input data from input files and
1549 store in pandas' Series.
1551 :param repeat: Repeat the download specified number of times if not
1556 logging.info(u"Downloading and parsing input files ...")
1558 for job, builds in self._cfg.input.items():
1559 for build in builds:
1561 result = self._download_and_parse_build(job, build, repeat)
1564 build_nr = result[u"build"][u"build"]
1567 data = result[u"data"]
1568 build_data = pd.Series({
1569 u"metadata": pd.Series(
1570 list(data[u"metadata"].values()),
1571 index=list(data[u"metadata"].keys())
1573 u"suites": pd.Series(
1574 list(data[u"suites"].values()),
1575 index=list(data[u"suites"].keys())
1577 u"tests": pd.Series(
1578 list(data[u"tests"].values()),
1579 index=list(data[u"tests"].keys())
1583 if self._input_data.get(job, None) is None:
1584 self._input_data[job] = pd.Series()
1585 self._input_data[job][str(build_nr)] = build_data
1586 self._cfg.set_input_file_name(
1587 job, build_nr, result[u"build"][u"file-name"]
1589 self._cfg.set_input_state(job, build_nr, result[u"state"])
1592 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1593 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1595 logging.info(u"Done.")
1597 msg = f"Successful downloads from the sources:\n"
1598 for source in self._cfg.environment[u"data-sources"]:
1599 if source[u"successful-downloads"]:
1601 f"{source[u'url']}/{source[u'path']}/"
1602 f"{source[u'file-name']}: "
1603 f"{source[u'successful-downloads']}\n"
1607 def process_local_file(self, local_file, job=u"local", build_nr=1,
1609 """Process local XML file given as a command-line parameter.
1611 :param local_file: The file to process.
1612 :param job: Job name.
1613 :param build_nr: Build number.
1614 :param replace: If True, the information about jobs and builds is
1615 replaced by the new one, otherwise the new jobs and builds are
1617 :type local_file: str
1621 :raises: PresentationError if an error occurs.
1623 if not isfile(local_file):
1624 raise PresentationError(f"The file {local_file} does not exist.")
1627 build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1628 except (IndexError, ValueError):
1633 u"status": u"failed",
1634 u"file-name": local_file
1637 self._cfg.input = dict()
1638 self._cfg.add_build(job, build)
1640 logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1641 data = self._parse_tests(job, build)
1643 raise PresentationError(
1644 f"Error occurred while parsing the file {local_file}"
1647 build_data = pd.Series({
1648 u"metadata": pd.Series(
1649 list(data[u"metadata"].values()),
1650 index=list(data[u"metadata"].keys())
1652 u"suites": pd.Series(
1653 list(data[u"suites"].values()),
1654 index=list(data[u"suites"].keys())
1656 u"tests": pd.Series(
1657 list(data[u"tests"].values()),
1658 index=list(data[u"tests"].keys())
1662 if self._input_data.get(job, None) is None:
1663 self._input_data[job] = pd.Series()
1664 self._input_data[job][str(build_nr)] = build_data
1666 self._cfg.set_input_state(job, build_nr, u"processed")
1668 def process_local_directory(self, local_dir, replace=True):
1669 """Process local directory with XML file(s). The directory is processed
1670 as a 'job' and the XML files in it as builds.
1671 If the given directory contains only sub-directories, these
1672 sub-directories processed as jobs and corresponding XML files as builds
1675 :param local_dir: Local directory to process.
1676 :param replace: If True, the information about jobs and builds is
1677 replaced by the new one, otherwise the new jobs and builds are
1679 :type local_dir: str
1682 if not isdir(local_dir):
1683 raise PresentationError(
1684 f"The directory {local_dir} does not exist."
1687 # Check if the given directory includes only files, or only directories
1688 _, dirnames, filenames = next(walk(local_dir))
1690 if filenames and not dirnames:
1693 # key: dir (job) name, value: list of file names (builds)
1695 local_dir: [join(local_dir, name) for name in filenames]
1698 elif dirnames and not filenames:
1701 # key: dir (job) name, value: list of file names (builds)
1702 local_builds = dict()
1703 for dirname in dirnames:
1705 join(local_dir, dirname, name)
1706 for name in listdir(join(local_dir, dirname))
1707 if isfile(join(local_dir, dirname, name))
1710 local_builds[dirname] = sorted(builds)
1712 elif not filenames and not dirnames:
1713 raise PresentationError(f"The directory {local_dir} is empty.")
1715 raise PresentationError(
1716 f"The directory {local_dir} can include only files or only "
1717 f"directories, not both.\nThe directory {local_dir} includes "
1718 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1722 self._cfg.input = dict()
1724 for job, files in local_builds.items():
1725 for idx, local_file in enumerate(files):
1726 self.process_local_file(local_file, job, idx + 1, replace=False)
1729 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1730 """Return the index of character in the string which is the end of tag.
1732 :param tag_filter: The string where the end of tag is being searched.
1733 :param start: The index where the searching is stated.
1734 :param closer: The character which is the tag closer.
1735 :type tag_filter: str
1738 :returns: The index of the tag closer.
1742 idx_opener = tag_filter.index(closer, start)
1743 return tag_filter.index(closer, idx_opener + 1)
1748 def _condition(tag_filter):
1749 """Create a conditional statement from the given tag filter.
1751 :param tag_filter: Filter based on tags from the element specification.
1752 :type tag_filter: str
1753 :returns: Conditional statement which can be evaluated.
1758 index = InputData._end_of_tag(tag_filter, index)
1762 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1764 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1765 continue_on_error=False):
1766 """Filter required data from the given jobs and builds.
1768 The output data structure is:
1771 - test (or suite) 1 ID:
1777 - test (or suite) n ID:
1784 :param element: Element which will use the filtered data.
1785 :param params: Parameters which will be included in the output. If None,
1786 all parameters are included.
1787 :param data: If not None, this data is used instead of data specified
1789 :param data_set: The set of data to be filtered: tests, suites,
1791 :param continue_on_error: Continue if there is error while reading the
1792 data. The Item will be empty then
1793 :type element: pandas.Series
1797 :type continue_on_error: bool
1798 :returns: Filtered data.
1799 :rtype pandas.Series
1803 if data_set == "suites":
1805 elif element[u"filter"] in (u"all", u"template"):
1808 cond = InputData._condition(element[u"filter"])
1809 logging.debug(f" Filter: {cond}")
1811 logging.error(u" No filter defined.")
1815 params = element.get(u"parameters", None)
1817 params.extend((u"type", u"status"))
1819 data_to_filter = data if data else element[u"data"]
1822 for job, builds in data_to_filter.items():
1823 data[job] = pd.Series()
1824 for build in builds:
1825 data[job][str(build)] = pd.Series()
1828 self.data[job][str(build)][data_set].items())
1830 if continue_on_error:
1834 for test_id, test_data in data_dict.items():
1835 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1836 data[job][str(build)][test_id] = pd.Series()
1838 for param, val in test_data.items():
1839 data[job][str(build)][test_id][param] = val
1841 for param in params:
1843 data[job][str(build)][test_id][param] =\
1846 data[job][str(build)][test_id][param] =\
1850 except (KeyError, IndexError, ValueError) as err:
1852 f"Missing mandatory parameter in the element specification: "
1856 except AttributeError as err:
1857 logging.error(repr(err))
1859 except SyntaxError as err:
1861 f"The filter {cond} is not correct. Check if all tags are "
1862 f"enclosed by apostrophes.\n{repr(err)}"
1866 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1867 continue_on_error=False):
1868 """Filter required data from the given jobs and builds.
1870 The output data structure is:
1873 - test (or suite) 1 ID:
1879 - test (or suite) n ID:
1886 :param element: Element which will use the filtered data.
1887 :param params: Parameters which will be included in the output. If None,
1888 all parameters are included.
1889 :param data_set: The set of data to be filtered: tests, suites,
1891 :param continue_on_error: Continue if there is error while reading the
1892 data. The Item will be empty then
1893 :type element: pandas.Series
1896 :type continue_on_error: bool
1897 :returns: Filtered data.
1898 :rtype pandas.Series
1901 include = element.get(u"include", None)
1903 logging.warning(u"No tests to include, skipping the element.")
1907 params = element.get(u"parameters", None)
1908 if params and u"type" not in params:
1909 params.append(u"type")
1911 cores = element.get(u"core", None)
1915 for test in include:
1916 tests.append(test.format(core=core))
1922 for job, builds in element[u"data"].items():
1923 data[job] = pd.Series()
1924 for build in builds:
1925 data[job][str(build)] = pd.Series()
1928 reg_ex = re.compile(str(test).lower())
1929 for test_id in self.data[job][
1930 str(build)][data_set].keys():
1931 if re.match(reg_ex, str(test_id).lower()):
1932 test_data = self.data[job][
1933 str(build)][data_set][test_id]
1934 data[job][str(build)][test_id] = pd.Series()
1936 for param, val in test_data.items():
1937 data[job][str(build)][test_id]\
1940 for param in params:
1942 data[job][str(build)][
1946 data[job][str(build)][
1947 test_id][param] = u"No Data"
1948 except KeyError as err:
1949 if continue_on_error:
1950 logging.debug(repr(err))
1952 logging.error(repr(err))
1956 except (KeyError, IndexError, ValueError) as err:
1958 f"Missing mandatory parameter in the element "
1959 f"specification: {repr(err)}"
1962 except AttributeError as err:
1963 logging.error(repr(err))
1967 def merge_data(data):
1968 """Merge data from more jobs and builds to a simple data structure.
1970 The output data structure is:
1972 - test (suite) 1 ID:
1978 - test (suite) n ID:
1981 :param data: Data to merge.
1982 :type data: pandas.Series
1983 :returns: Merged data.
1984 :rtype: pandas.Series
1987 logging.info(u" Merging data ...")
1989 merged_data = pd.Series()
1990 for builds in data.values:
1991 for item in builds.values:
1992 for item_id, item_data in item.items():
1993 merged_data[item_id] = item_data
1996 def print_all_oper_data(self):
1997 """Print all operational data to console.
2000 for job in self._input_data.values:
2001 for build in job.values:
2002 for test_id, test_data in build[u"tests"].items():
2004 if test_data.get(u"show-run", None) is None:
2006 for dut_name, data in test_data[u"show-run"].items():
2007 if data.get(u"runtime", None) is None:
2009 runtime = loads(data[u"runtime"])
2011 threads_nr = len(runtime[0][u"clocks"])
2012 except (IndexError, KeyError):
2014 threads = OrderedDict(
2015 {idx: list() for idx in range(threads_nr)})
2016 for item in runtime:
2017 for idx in range(threads_nr):
2018 if item[u"vectors"][idx] > 0:
2019 clocks = item[u"clocks"][idx] / \
2020 item[u"vectors"][idx]
2021 elif item[u"calls"][idx] > 0:
2022 clocks = item[u"clocks"][idx] / \
2024 elif item[u"suspends"][idx] > 0:
2025 clocks = item[u"clocks"][idx] / \
2026 item[u"suspends"][idx]
2030 if item[u"calls"][idx] > 0:
2031 vectors_call = item[u"vectors"][idx] / \
2036 if int(item[u"calls"][idx]) + int(
2037 item[u"vectors"][idx]) + \
2038 int(item[u"suspends"][idx]):
2039 threads[idx].append([
2041 item[u"calls"][idx],
2042 item[u"vectors"][idx],
2043 item[u"suspends"][idx],
2048 print(f"Host IP: {data.get(u'host', '')}, "
2049 f"Socket: {data.get(u'socket', '')}")
2050 for thread_nr, thread in threads.items():
2051 txt_table = prettytable.PrettyTable(
2057 u"Cycles per Packet",
2058 u"Average Vector Size"
2063 txt_table.add_row(row)
2065 if len(thread) == 0:
2068 avg = f", Average Vector Size per Node: " \
2069 f"{(avg / len(thread)):.2f}"
2070 th_name = u"main" if thread_nr == 0 \
2071 else f"worker_{thread_nr}"
2072 print(f"{dut_name}, {th_name}{avg}")
2073 txt_table.float_format = u".2"
2074 txt_table.align = u"r"
2075 txt_table.align[u"Name"] = u"l"
2076 print(f"{txt_table.get_string()}\n")