1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
28 from os import remove, walk, listdir
29 from os.path import isfile, isdir, join
30 from datetime import datetime as dt
31 from datetime import timedelta
32 from json import loads
33 from json.decoder import JSONDecodeError
40 from robot.api import ExecutionResult, ResultVisitor
41 from robot import errors
43 from resources.libraries.python import jumpavg
44 from input_data_files import download_and_unzip_data_file
45 from pal_errors import PresentationError
48 # Separator used in file names
52 class ExecutionChecker(ResultVisitor):
53 """Class to traverse through the test suite structure.
55 The functionality implemented in this class generates a json structure:
61 "generated": "Timestamp",
62 "version": "SUT version",
63 "job": "Jenkins job name",
64 "build": "Information about the build"
67 "Suite long name 1": {
69 "doc": "Suite 1 documentation",
70 "parent": "Suite 1 parent",
71 "level": "Level of the suite in the suite hierarchy"
73 "Suite long name N": {
75 "doc": "Suite N documentation",
76 "parent": "Suite 2 parent",
77 "level": "Level of the suite in the suite hierarchy"
84 "parent": "Name of the parent of the test",
85 "doc": "Test documentation",
86 "msg": "Test message",
87 "conf-history": "DUT1 and DUT2 VAT History",
88 "show-run": "Show Run",
89 "tags": ["tag 1", "tag 2", "tag n"],
91 "status": "PASS" | "FAIL",
137 "parent": "Name of the parent of the test",
138 "doc": "Test documentation",
139 "msg": "Test message",
140 "tags": ["tag 1", "tag 2", "tag n"],
142 "status": "PASS" | "FAIL",
149 "parent": "Name of the parent of the test",
150 "doc": "Test documentation",
151 "msg": "Test message",
152 "tags": ["tag 1", "tag 2", "tag n"],
153 "type": "MRR" | "BMRR",
154 "status": "PASS" | "FAIL",
156 "receive-rate": float,
157 # Average of a list, computed using AvgStdevStats.
158 # In CSIT-1180, replace with List[float].
172 "metadata": { # Optional
173 "version": "VPP version",
174 "job": "Jenkins job name",
175 "build": "Information about the build"
179 "doc": "Suite 1 documentation",
180 "parent": "Suite 1 parent",
181 "level": "Level of the suite in the suite hierarchy"
184 "doc": "Suite N documentation",
185 "parent": "Suite 2 parent",
186 "level": "Level of the suite in the suite hierarchy"
192 "parent": "Name of the parent of the test",
193 "doc": "Test documentation"
194 "msg": "Test message"
195 "tags": ["tag 1", "tag 2", "tag n"],
196 "conf-history": "DUT1 and DUT2 VAT History"
197 "show-run": "Show Run"
198 "status": "PASS" | "FAIL"
206 .. note:: ID is the lowercase full path to the test.
209 REGEX_PLR_RATE = re.compile(
210 r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
211 r'PLRsearch upper bound::?\s(\d+.\d+)'
213 REGEX_NDRPDR_RATE = re.compile(
214 r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
215 r'NDR_UPPER:\s(\d+.\d+).*\n'
216 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
217 r'PDR_UPPER:\s(\d+.\d+)'
219 REGEX_NDRPDR_GBPS = re.compile(
220 r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
221 r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
222 r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
223 r'PDR_UPPER:.*,\s(\d+.\d+)'
225 REGEX_PERF_MSG_INFO = re.compile(
226 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
227 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
228 r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
229 r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
230 r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
232 REGEX_CPS_MSG_INFO = re.compile(
233 r'NDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*\n'
234 r'PDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*'
236 REGEX_PPS_MSG_INFO = re.compile(
237 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
238 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*'
240 REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
242 REGEX_VSAP_MSG_INFO = re.compile(
243 r'Transfer Rate: (\d*.\d*).*\n'
244 r'Latency: (\d*.\d*).*\n'
245 r'Completed requests: (\d*).*\n'
246 r'Failed requests: (\d*).*\n'
247 r'Total data transferred: (\d*).*\n'
248 r'Connection [cr]ps rate:\s*(\d*.\d*)'
251 # Needed for CPS and PPS tests
252 REGEX_NDRPDR_LAT_BASE = re.compile(
253 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
254 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
256 REGEX_NDRPDR_LAT = re.compile(
257 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
258 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
259 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
260 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
261 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
262 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
265 REGEX_VERSION_VPP = re.compile(
266 r"(return STDOUT Version:\s*|"
267 r"VPP Version:\s*|VPP version:\s*)(.*)"
269 REGEX_VERSION_DPDK = re.compile(
270 r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
272 REGEX_TCP = re.compile(
273 r'Total\s(rps|cps|throughput):\s(\d*).*$'
275 REGEX_MRR = re.compile(
276 r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
277 r'tx\s(\d*),\srx\s(\d*)'
279 REGEX_BMRR = re.compile(
280 r'.*trial results.*: \[(.*)\]'
282 REGEX_RECONF_LOSS = re.compile(
283 r'Packets lost due to reconfig: (\d*)'
285 REGEX_RECONF_TIME = re.compile(
286 r'Implied time lost: (\d*.[\de-]*)'
288 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
290 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
292 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
294 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
296 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
298 def __init__(self, metadata, mapping, ignore):
301 :param metadata: Key-value pairs to be included in "metadata" part of
303 :param mapping: Mapping of the old names of test cases to the new
305 :param ignore: List of TCs to be ignored.
311 # Type of message to parse out from the test messages
312 self._msg_type = None
318 self._timestamp = None
320 # Testbed. The testbed is identified by TG node IP address.
323 # Mapping of TCs long names
324 self._mapping = mapping
327 self._ignore = ignore
329 # Number of PAPI History messages found:
331 # 1 - PAPI History of DUT1
332 # 2 - PAPI History of DUT2
333 self._conf_history_lookup_nr = 0
335 self._sh_run_counter = 0
337 # Test ID of currently processed test- the lowercase full path to the
341 # The main data structure
343 u"metadata": OrderedDict(),
344 u"suites": OrderedDict(),
345 u"tests": OrderedDict()
348 # Save the provided metadata
349 for key, val in metadata.items():
350 self._data[u"metadata"][key] = val
352 # Dictionary defining the methods used to parse different types of
355 u"timestamp": self._get_timestamp,
356 u"vpp-version": self._get_vpp_version,
357 u"dpdk-version": self._get_dpdk_version,
358 u"teardown-papi-history": self._get_papi_history,
359 u"test-show-runtime": self._get_show_run,
360 u"testbed": self._get_testbed
365 """Getter - Data parsed from the XML file.
367 :returns: Data parsed from the XML file.
372 def _get_data_from_mrr_test_msg(self, msg):
373 """Get info from message of MRR performance tests.
375 :param msg: Message to be processed.
377 :returns: Processed message or original message if a problem occurs.
381 groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
382 if not groups or groups.lastindex != 1:
383 return u"Test Failed."
386 data = groups.group(1).split(u", ")
387 except (AttributeError, IndexError, ValueError, KeyError):
388 return u"Test Failed."
393 out_str += f"{(float(item) / 1e6):.2f}, "
394 return out_str[:-2] + u"]"
395 except (AttributeError, IndexError, ValueError, KeyError):
396 return u"Test Failed."
398 def _get_data_from_cps_test_msg(self, msg):
399 """Get info from message of NDRPDR CPS tests.
401 :param msg: Message to be processed.
403 :returns: Processed message or "Test Failed." if a problem occurs.
407 groups = re.search(self.REGEX_CPS_MSG_INFO, msg)
408 if not groups or groups.lastindex != 2:
409 return u"Test Failed."
413 f"1. {(float(groups.group(1)) / 1e6):5.2f}\n"
414 f"2. {(float(groups.group(2)) / 1e6):5.2f}"
416 except (AttributeError, IndexError, ValueError, KeyError):
417 return u"Test Failed."
419 def _get_data_from_pps_test_msg(self, msg):
420 """Get info from message of NDRPDR PPS tests.
422 :param msg: Message to be processed.
424 :returns: Processed message or "Test Failed." if a problem occurs.
428 groups = re.search(self.REGEX_PPS_MSG_INFO, msg)
429 if not groups or groups.lastindex != 4:
430 return u"Test Failed."
434 f"1. {(float(groups.group(1)) / 1e6):5.2f} "
435 f"{float(groups.group(2)):5.2f}\n"
436 f"2. {(float(groups.group(3)) / 1e6):5.2f} "
437 f"{float(groups.group(4)):5.2f}"
439 except (AttributeError, IndexError, ValueError, KeyError):
440 return u"Test Failed."
442 def _get_data_from_perf_test_msg(self, msg):
443 """Get info from message of NDRPDR performance tests.
445 :param msg: Message to be processed.
447 :returns: Processed message or "Test Failed." if a problem occurs.
451 groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
452 if not groups or groups.lastindex != 10:
453 return u"Test Failed."
457 u"ndr_low": float(groups.group(1)),
458 u"ndr_low_b": float(groups.group(2)),
459 u"pdr_low": float(groups.group(3)),
460 u"pdr_low_b": float(groups.group(4)),
461 u"pdr_lat_90_1": groups.group(5),
462 u"pdr_lat_90_2": groups.group(6),
463 u"pdr_lat_50_1": groups.group(7),
464 u"pdr_lat_50_2": groups.group(8),
465 u"pdr_lat_10_1": groups.group(9),
466 u"pdr_lat_10_2": groups.group(10),
468 except (AttributeError, IndexError, ValueError, KeyError):
469 return u"Test Failed."
471 def _process_lat(in_str_1, in_str_2):
472 """Extract min, avg, max values from latency string.
474 :param in_str_1: Latency string for one direction produced by robot
476 :param in_str_2: Latency string for second direction produced by
480 :returns: Processed latency string or None if a problem occurs.
483 in_list_1 = in_str_1.split('/', 3)
484 in_list_2 = in_str_2.split('/', 3)
486 if len(in_list_1) != 4 and len(in_list_2) != 4:
489 in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
491 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
492 except hdrh.codec.HdrLengthException:
495 in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
497 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
498 except hdrh.codec.HdrLengthException:
501 if hdr_lat_1 and hdr_lat_2:
503 hdr_lat_1.get_value_at_percentile(50.0),
504 hdr_lat_1.get_value_at_percentile(90.0),
505 hdr_lat_1.get_value_at_percentile(99.0),
506 hdr_lat_2.get_value_at_percentile(50.0),
507 hdr_lat_2.get_value_at_percentile(90.0),
508 hdr_lat_2.get_value_at_percentile(99.0)
518 f"1. {(data[u'ndr_low'] / 1e6):5.2f} "
519 f"{data[u'ndr_low_b']:5.2f}"
520 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f} "
521 f"{data[u'pdr_low_b']:5.2f}"
524 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
525 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
526 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
529 max_len = len(str(max((max(item) for item in latency))))
530 max_len = 4 if max_len < 4 else max_len
532 for idx, lat in enumerate(latency):
537 f"{lat[0]:{max_len}d} "
538 f"{lat[1]:{max_len}d} "
539 f"{lat[2]:{max_len}d} "
540 f"{lat[3]:{max_len}d} "
541 f"{lat[4]:{max_len}d} "
542 f"{lat[5]:{max_len}d} "
547 except (AttributeError, IndexError, ValueError, KeyError):
548 return u"Test Failed."
550 def _get_testbed(self, msg):
551 """Called when extraction of testbed IP is required.
552 The testbed is identified by TG node IP address.
554 :param msg: Message to process.
559 if msg.message.count(u"Setup of TG node") or \
560 msg.message.count(u"Setup of node TG host"):
561 reg_tg_ip = re.compile(
562 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
564 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
565 except (KeyError, ValueError, IndexError, AttributeError):
568 self._data[u"metadata"][u"testbed"] = self._testbed
569 self._msg_type = None
571 def _get_vpp_version(self, msg):
572 """Called when extraction of VPP version is required.
574 :param msg: Message to process.
579 if msg.message.count(u"return STDOUT Version:") or \
580 msg.message.count(u"VPP Version:") or \
581 msg.message.count(u"VPP version:"):
583 re.search(self.REGEX_VERSION_VPP, msg.message).group(2)
585 self._data[u"metadata"][u"version"] = self._version
586 self._msg_type = None
588 def _get_dpdk_version(self, msg):
589 """Called when extraction of DPDK version is required.
591 :param msg: Message to process.
596 if msg.message.count(u"DPDK Version:"):
598 self._version = str(re.search(
599 self.REGEX_VERSION_DPDK, msg.message).group(2))
600 self._data[u"metadata"][u"version"] = self._version
604 self._msg_type = None
606 def _get_timestamp(self, msg):
607 """Called when extraction of timestamp is required.
609 :param msg: Message to process.
614 self._timestamp = msg.timestamp[:14]
615 self._data[u"metadata"][u"generated"] = self._timestamp
616 self._msg_type = None
618 def _get_papi_history(self, msg):
619 """Called when extraction of PAPI command history is required.
621 :param msg: Message to process.
625 if msg.message.count(u"PAPI command history:"):
626 self._conf_history_lookup_nr += 1
627 if self._conf_history_lookup_nr == 1:
628 self._data[u"tests"][self._test_id][u"conf-history"] = str()
630 self._msg_type = None
632 r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} PAPI command history:",
636 ).replace(u'"', u"'")
637 self._data[u"tests"][self._test_id][u"conf-history"] += (
638 f"**DUT{str(self._conf_history_lookup_nr)}:** {text}"
641 def _get_show_run(self, msg):
642 """Called when extraction of VPP operational data (output of CLI command
643 Show Runtime) is required.
645 :param msg: Message to process.
650 if not msg.message.count(u"stats runtime"):
654 if self._sh_run_counter > 1:
657 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
658 self._data[u"tests"][self._test_id][u"show-run"] = dict()
660 groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
664 host = groups.group(1)
665 except (AttributeError, IndexError):
668 sock = groups.group(2)
669 except (AttributeError, IndexError):
672 runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
673 replace(u"'", u'"').replace(u'b"', u'"').
674 replace(u'u"', u'"').split(u":", 1)[1])
677 threads_nr = len(runtime[0][u"clocks"])
678 except (IndexError, KeyError):
681 dut = u"dut{nr}".format(
682 nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
687 # Needed for json converter, enable when 'threads' is gone.
688 # u"runtime": runtime,
689 u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
693 for idx in range(threads_nr):
694 if item[u"vectors"][idx] > 0:
695 clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
696 elif item[u"calls"][idx] > 0:
697 clocks = item[u"clocks"][idx] / item[u"calls"][idx]
698 elif item[u"suspends"][idx] > 0:
699 clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
703 if item[u"calls"][idx] > 0:
704 vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
708 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
709 int(item[u"suspends"][idx]):
710 oper[u"threads"][idx].append([
713 item[u"vectors"][idx],
714 item[u"suspends"][idx],
719 self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
721 def _get_ndrpdr_throughput(self, msg):
722 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
725 :param msg: The test message to be parsed.
727 :returns: Parsed data as a dict and the status (PASS/FAIL).
728 :rtype: tuple(dict, str)
732 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
733 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
736 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
738 if groups is not None:
740 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
741 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
742 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
743 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
745 except (IndexError, ValueError):
748 return throughput, status
750 def _get_ndrpdr_throughput_gbps(self, msg):
751 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
754 :param msg: The test message to be parsed.
756 :returns: Parsed data as a dict and the status (PASS/FAIL).
757 :rtype: tuple(dict, str)
761 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
762 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
765 groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
767 if groups is not None:
769 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
770 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
771 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
772 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
774 except (IndexError, ValueError):
779 def _get_plr_throughput(self, msg):
780 """Get PLRsearch lower bound and PLRsearch upper bound from the test
783 :param msg: The test message to be parsed.
785 :returns: Parsed data as a dict and the status (PASS/FAIL).
786 :rtype: tuple(dict, str)
794 groups = re.search(self.REGEX_PLR_RATE, msg)
796 if groups is not None:
798 throughput[u"LOWER"] = float(groups.group(1))
799 throughput[u"UPPER"] = float(groups.group(2))
801 except (IndexError, ValueError):
804 return throughput, status
806 def _get_ndrpdr_latency(self, msg):
807 """Get LATENCY from the test message.
809 :param msg: The test message to be parsed.
811 :returns: Parsed data as a dict and the status (PASS/FAIL).
812 :rtype: tuple(dict, str)
822 u"direction1": copy.copy(latency_default),
823 u"direction2": copy.copy(latency_default)
826 u"direction1": copy.copy(latency_default),
827 u"direction2": copy.copy(latency_default)
830 u"direction1": copy.copy(latency_default),
831 u"direction2": copy.copy(latency_default)
834 u"direction1": copy.copy(latency_default),
835 u"direction2": copy.copy(latency_default)
838 u"direction1": copy.copy(latency_default),
839 u"direction2": copy.copy(latency_default)
842 u"direction1": copy.copy(latency_default),
843 u"direction2": copy.copy(latency_default)
847 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
849 groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
851 return latency, u"FAIL"
853 def process_latency(in_str):
854 """Return object with parsed latency values.
856 TODO: Define class for the return type.
858 :param in_str: Input string, min/avg/max/hdrh format.
860 :returns: Dict with corresponding keys, except hdrh float values.
862 :throws IndexError: If in_str does not have enough substrings.
863 :throws ValueError: If a substring does not convert to float.
865 in_list = in_str.split('/', 3)
868 u"min": float(in_list[0]),
869 u"avg": float(in_list[1]),
870 u"max": float(in_list[2]),
874 if len(in_list) == 4:
875 rval[u"hdrh"] = str(in_list[3])
880 latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
881 latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
882 latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
883 latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
884 if groups.lastindex == 4:
885 return latency, u"PASS"
886 except (IndexError, ValueError):
890 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
891 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
892 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
893 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
894 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
895 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
896 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
897 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
898 if groups.lastindex == 12:
899 return latency, u"PASS"
900 except (IndexError, ValueError):
903 return latency, u"FAIL"
906 def _get_hoststack_data(msg, tags):
907 """Get data from the hoststack test message.
909 :param msg: The test message to be parsed.
910 :param tags: Test tags.
913 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
914 :rtype: tuple(dict, str)
919 msg = msg.replace(u"'", u'"').replace(u" ", u"")
920 if u"LDPRELOAD" in tags:
924 except JSONDecodeError:
926 elif u"VPPECHO" in tags:
928 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
930 client=loads(msg_lst[0]),
931 server=loads(msg_lst[1])
934 except (JSONDecodeError, IndexError):
937 return result, status
939 def _get_vsap_data(self, msg, tags):
940 """Get data from the vsap test message.
942 :param msg: The test message to be parsed.
943 :param tags: Test tags.
946 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
947 :rtype: tuple(dict, str)
952 groups = re.search(self.REGEX_VSAP_MSG_INFO, msg)
953 if groups is not None:
955 result[u"transfer-rate"] = float(groups.group(1)) * 1e3
956 result[u"latency"] = float(groups.group(2))
957 result[u"completed-requests"] = int(groups.group(3))
958 result[u"failed-requests"] = int(groups.group(4))
959 result[u"bytes-transferred"] = int(groups.group(5))
960 if u"TCP_CPS"in tags:
961 result[u"cps"] = float(groups.group(6))
962 elif u"TCP_RPS" in tags:
963 result[u"rps"] = float(groups.group(6))
965 return result, status
967 except (IndexError, ValueError):
970 return result, status
972 def visit_suite(self, suite):
973 """Implements traversing through the suite and its direct children.
975 :param suite: Suite to process.
979 if self.start_suite(suite) is not False:
980 suite.suites.visit(self)
981 suite.tests.visit(self)
982 self.end_suite(suite)
984 def start_suite(self, suite):
985 """Called when suite starts.
987 :param suite: Suite to process.
993 parent_name = suite.parent.name
994 except AttributeError:
997 self._data[u"suites"][suite.longname.lower().
999 replace(u" ", u"_")] = {
1000 u"name": suite.name.lower(),
1002 u"parent": parent_name,
1003 u"level": len(suite.longname.split(u"."))
1006 suite.keywords.visit(self)
1008 def end_suite(self, suite):
1009 """Called when suite ends.
1011 :param suite: Suite to process.
1016 def visit_test(self, test):
1017 """Implements traversing through the test.
1019 :param test: Test to process.
1023 if self.start_test(test) is not False:
1024 test.keywords.visit(self)
1027 def start_test(self, test):
1028 """Called when test starts.
1030 :param test: Test to process.
1035 self._sh_run_counter = 0
1037 longname_orig = test.longname.lower()
1039 # Check the ignore list
1040 if longname_orig in self._ignore:
1043 tags = [str(tag) for tag in test.tags]
1044 test_result = dict()
1046 # Change the TC long name and name if defined in the mapping table
1047 longname = self._mapping.get(longname_orig, None)
1048 if longname is not None:
1049 name = longname.split(u'.')[-1]
1051 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1055 longname = longname_orig
1056 name = test.name.lower()
1058 # Remove TC number from the TC long name (backward compatibility):
1059 self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1060 # Remove TC number from the TC name (not needed):
1061 test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1063 test_result[u"parent"] = test.parent.name.lower()
1064 test_result[u"tags"] = tags
1065 test_result["doc"] = test.doc
1066 test_result[u"type"] = u""
1067 test_result[u"status"] = test.status
1068 test_result[u"starttime"] = test.starttime
1069 test_result[u"endtime"] = test.endtime
1071 if test.status == u"PASS":
1072 if u"NDRPDR" in tags:
1073 if u"TCP_PPS" in tags or u"UDP_PPS" in tags:
1074 test_result[u"msg"] = self._get_data_from_pps_test_msg(
1076 elif u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1077 test_result[u"msg"] = self._get_data_from_cps_test_msg(
1080 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1082 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1083 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1086 test_result[u"msg"] = test.message
1088 test_result[u"msg"] = test.message
1090 if u"PERFTEST" in tags:
1091 # Replace info about cores (e.g. -1c-) with the info about threads
1092 # and cores (e.g. -1t1c-) in the long test case names and in the
1093 # test case names if necessary.
1094 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1098 for tag in test_result[u"tags"]:
1099 groups = re.search(self.REGEX_TC_TAG, tag)
1105 self._test_id = re.sub(
1106 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1107 self._test_id, count=1
1109 test_result[u"name"] = re.sub(
1110 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1111 test_result["name"], count=1
1114 test_result[u"status"] = u"FAIL"
1115 self._data[u"tests"][self._test_id] = test_result
1117 f"The test {self._test_id} has no or more than one "
1118 f"multi-threading tags.\n"
1119 f"Tags: {test_result[u'tags']}"
1123 if u"DEVICETEST" in tags:
1124 test_result[u"type"] = u"DEVICETEST"
1125 elif u"NDRPDR" in tags:
1126 if u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1127 test_result[u"type"] = u"CPS"
1129 test_result[u"type"] = u"NDRPDR"
1130 if test.status == u"PASS":
1131 test_result[u"throughput"], test_result[u"status"] = \
1132 self._get_ndrpdr_throughput(test.message)
1133 test_result[u"gbps"], test_result[u"status"] = \
1134 self._get_ndrpdr_throughput_gbps(test.message)
1135 test_result[u"latency"], test_result[u"status"] = \
1136 self._get_ndrpdr_latency(test.message)
1137 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1139 test_result[u"type"] = u"MRR"
1141 test_result[u"type"] = u"BMRR"
1142 if test.status == u"PASS":
1143 test_result[u"result"] = dict()
1144 groups = re.search(self.REGEX_BMRR, test.message)
1145 if groups is not None:
1146 items_str = groups.group(1)
1148 float(item.strip().replace(u"'", u""))
1149 for item in items_str.split(",")
1151 # Use whole list in CSIT-1180.
1152 stats = jumpavg.AvgStdevStats.for_runs(items_float)
1153 test_result[u"result"][u"samples"] = items_float
1154 test_result[u"result"][u"receive-rate"] = stats.avg
1155 test_result[u"result"][u"receive-stdev"] = stats.stdev
1157 groups = re.search(self.REGEX_MRR, test.message)
1158 test_result[u"result"][u"receive-rate"] = \
1159 float(groups.group(3)) / float(groups.group(1))
1160 elif u"SOAK" in tags:
1161 test_result[u"type"] = u"SOAK"
1162 if test.status == u"PASS":
1163 test_result[u"throughput"], test_result[u"status"] = \
1164 self._get_plr_throughput(test.message)
1165 elif u"HOSTSTACK" in tags:
1166 test_result[u"type"] = u"HOSTSTACK"
1167 if test.status == u"PASS":
1168 test_result[u"result"], test_result[u"status"] = \
1169 self._get_hoststack_data(test.message, tags)
1170 elif u"LDP_NGINX" in tags:
1171 test_result[u"type"] = u"LDP_NGINX"
1172 test_result[u"result"], test_result[u"status"] = \
1173 self._get_vsap_data(test.message, tags)
1174 # elif u"TCP" in tags: # This might be not used
1175 # test_result[u"type"] = u"TCP"
1176 # if test.status == u"PASS":
1177 # groups = re.search(self.REGEX_TCP, test.message)
1178 # test_result[u"result"] = int(groups.group(2))
1179 elif u"RECONF" in tags:
1180 test_result[u"type"] = u"RECONF"
1181 if test.status == u"PASS":
1182 test_result[u"result"] = None
1184 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1185 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1186 test_result[u"result"] = {
1187 u"loss": int(grps_loss.group(1)),
1188 u"time": float(grps_time.group(1))
1190 except (AttributeError, IndexError, ValueError, TypeError):
1191 test_result[u"status"] = u"FAIL"
1193 test_result[u"status"] = u"FAIL"
1195 self._data[u"tests"][self._test_id] = test_result
1197 def end_test(self, test):
1198 """Called when test ends.
1200 :param test: Test to process.
1205 def visit_keyword(self, keyword):
1206 """Implements traversing through the keyword and its child keywords.
1208 :param keyword: Keyword to process.
1209 :type keyword: Keyword
1212 if self.start_keyword(keyword) is not False:
1213 self.end_keyword(keyword)
1215 def start_keyword(self, keyword):
1216 """Called when keyword starts. Default implementation does nothing.
1218 :param keyword: Keyword to process.
1219 :type keyword: Keyword
1223 if keyword.type == u"setup":
1224 self.visit_setup_kw(keyword)
1225 elif keyword.type == u"teardown":
1226 self.visit_teardown_kw(keyword)
1228 self.visit_test_kw(keyword)
1229 except AttributeError:
1232 def end_keyword(self, keyword):
1233 """Called when keyword ends. Default implementation does nothing.
1235 :param keyword: Keyword to process.
1236 :type keyword: Keyword
1240 def visit_test_kw(self, test_kw):
1241 """Implements traversing through the test keyword and its child
1244 :param test_kw: Keyword to process.
1245 :type test_kw: Keyword
1248 for keyword in test_kw.keywords:
1249 if self.start_test_kw(keyword) is not False:
1250 self.visit_test_kw(keyword)
1251 self.end_test_kw(keyword)
1253 def start_test_kw(self, test_kw):
1254 """Called when test keyword starts. Default implementation does
1257 :param test_kw: Keyword to process.
1258 :type test_kw: Keyword
1261 if test_kw.name.count(u"Show Runtime On All Duts") or \
1262 test_kw.name.count(u"Show Runtime Counters On All Duts") or \
1263 test_kw.name.count(u"Vpp Show Runtime On All Duts"):
1264 self._msg_type = u"test-show-runtime"
1265 self._sh_run_counter += 1
1268 test_kw.messages.visit(self)
1270 def end_test_kw(self, test_kw):
1271 """Called when keyword ends. Default implementation does nothing.
1273 :param test_kw: Keyword to process.
1274 :type test_kw: Keyword
1278 def visit_setup_kw(self, setup_kw):
1279 """Implements traversing through the teardown keyword and its child
1282 :param setup_kw: Keyword to process.
1283 :type setup_kw: Keyword
1286 for keyword in setup_kw.keywords:
1287 if self.start_setup_kw(keyword) is not False:
1288 self.visit_setup_kw(keyword)
1289 self.end_setup_kw(keyword)
1291 def start_setup_kw(self, setup_kw):
1292 """Called when teardown keyword starts. Default implementation does
1295 :param setup_kw: Keyword to process.
1296 :type setup_kw: Keyword
1299 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1300 and not self._version:
1301 self._msg_type = u"vpp-version"
1302 elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1304 self._msg_type = u"dpdk-version"
1305 elif setup_kw.name.count(u"Set Global Variable") \
1306 and not self._timestamp:
1307 self._msg_type = u"timestamp"
1308 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1309 self._msg_type = u"testbed"
1312 setup_kw.messages.visit(self)
1314 def end_setup_kw(self, setup_kw):
1315 """Called when keyword ends. Default implementation does nothing.
1317 :param setup_kw: Keyword to process.
1318 :type setup_kw: Keyword
1322 def visit_teardown_kw(self, teardown_kw):
1323 """Implements traversing through the teardown keyword and its child
1326 :param teardown_kw: Keyword to process.
1327 :type teardown_kw: Keyword
1330 for keyword in teardown_kw.keywords:
1331 if self.start_teardown_kw(keyword) is not False:
1332 self.visit_teardown_kw(keyword)
1333 self.end_teardown_kw(keyword)
1335 def start_teardown_kw(self, teardown_kw):
1336 """Called when teardown keyword starts
1338 :param teardown_kw: Keyword to process.
1339 :type teardown_kw: Keyword
1342 if teardown_kw.name.count(u"Show Papi History On All Duts"):
1343 self._conf_history_lookup_nr = 0
1344 self._msg_type = u"teardown-papi-history"
1345 teardown_kw.messages.visit(self)
1347 def end_teardown_kw(self, teardown_kw):
1348 """Called when keyword ends. Default implementation does nothing.
1350 :param teardown_kw: Keyword to process.
1351 :type teardown_kw: Keyword
1355 def visit_message(self, msg):
1356 """Implements visiting the message.
1358 :param msg: Message to process.
1362 if self.start_message(msg) is not False:
1363 self.end_message(msg)
1365 def start_message(self, msg):
1366 """Called when message starts. Get required information from messages:
1369 :param msg: Message to process.
1374 self.parse_msg[self._msg_type](msg)
1376 def end_message(self, msg):
1377 """Called when message ends. Default implementation does nothing.
1379 :param msg: Message to process.
1388 The data is extracted from output.xml files generated by Jenkins jobs and
1389 stored in pandas' DataFrames.
1395 (as described in ExecutionChecker documentation)
1397 (as described in ExecutionChecker documentation)
1399 (as described in ExecutionChecker documentation)
1402 def __init__(self, spec):
1405 :param spec: Specification.
1406 :type spec: Specification
1413 self._input_data = pd.Series()
1417 """Getter - Input data.
1419 :returns: Input data
1420 :rtype: pandas.Series
1422 return self._input_data
1424 def metadata(self, job, build):
1425 """Getter - metadata
1427 :param job: Job which metadata we want.
1428 :param build: Build which metadata we want.
1432 :rtype: pandas.Series
1434 return self.data[job][build][u"metadata"]
1436 def suites(self, job, build):
1439 :param job: Job which suites we want.
1440 :param build: Build which suites we want.
1444 :rtype: pandas.Series
1446 return self.data[job][str(build)][u"suites"]
1448 def tests(self, job, build):
1451 :param job: Job which tests we want.
1452 :param build: Build which tests we want.
1456 :rtype: pandas.Series
1458 return self.data[job][build][u"tests"]
1460 def _parse_tests(self, job, build):
1461 """Process data from robot output.xml file and return JSON structured
1464 :param job: The name of job which build output data will be processed.
1465 :param build: The build which output data will be processed.
1468 :returns: JSON data structure.
1477 with open(build[u"file-name"], u'r') as data_file:
1479 result = ExecutionResult(data_file)
1480 except errors.DataError as err:
1482 f"Error occurred while parsing output.xml: {repr(err)}"
1485 checker = ExecutionChecker(
1486 metadata, self._cfg.mapping, self._cfg.ignore
1488 result.visit(checker)
1492 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1493 """Download and parse the input data file.
1495 :param pid: PID of the process executing this method.
1496 :param job: Name of the Jenkins job which generated the processed input
1498 :param build: Information about the Jenkins build which generated the
1499 processed input file.
1500 :param repeat: Repeat the download specified number of times if not
1508 logging.info(f"Processing the job/build: {job}: {build[u'build']}")
1515 success = download_and_unzip_data_file(self._cfg, job, build, pid)
1521 f"It is not possible to download the input data file from the "
1522 f"job {job}, build {build[u'build']}, or it is damaged. "
1526 logging.info(f" Processing data from build {build[u'build']}")
1527 data = self._parse_tests(job, build)
1530 f"Input data file from the job {job}, build "
1531 f"{build[u'build']} is damaged. Skipped."
1534 state = u"processed"
1537 remove(build[u"file-name"])
1538 except OSError as err:
1540 f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1543 # If the time-period is defined in the specification file, remove all
1544 # files which are outside the time period.
1546 timeperiod = self._cfg.environment.get(u"time-period", None)
1547 if timeperiod and data:
1549 timeperiod = timedelta(int(timeperiod))
1550 metadata = data.get(u"metadata", None)
1552 generated = metadata.get(u"generated", None)
1554 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1555 if (now - generated) > timeperiod:
1556 # Remove the data and the file:
1561 f" The build {job}/{build[u'build']} is "
1562 f"outdated, will be removed."
1572 def download_and_parse_data(self, repeat=1):
1573 """Download the input data files, parse input data from input files and
1574 store in pandas' Series.
1576 :param repeat: Repeat the download specified number of times if not
1581 logging.info(u"Downloading and parsing input files ...")
1583 for job, builds in self._cfg.input.items():
1584 for build in builds:
1586 result = self._download_and_parse_build(job, build, repeat)
1589 build_nr = result[u"build"][u"build"]
1592 data = result[u"data"]
1593 build_data = pd.Series({
1594 u"metadata": pd.Series(
1595 list(data[u"metadata"].values()),
1596 index=list(data[u"metadata"].keys())
1598 u"suites": pd.Series(
1599 list(data[u"suites"].values()),
1600 index=list(data[u"suites"].keys())
1602 u"tests": pd.Series(
1603 list(data[u"tests"].values()),
1604 index=list(data[u"tests"].keys())
1608 if self._input_data.get(job, None) is None:
1609 self._input_data[job] = pd.Series()
1610 self._input_data[job][str(build_nr)] = build_data
1611 self._cfg.set_input_file_name(
1612 job, build_nr, result[u"build"][u"file-name"]
1614 self._cfg.set_input_state(job, build_nr, result[u"state"])
1617 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1618 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1620 logging.info(u"Done.")
1622 msg = f"Successful downloads from the sources:\n"
1623 for source in self._cfg.environment[u"data-sources"]:
1624 if source[u"successful-downloads"]:
1626 f"{source[u'url']}/{source[u'path']}/"
1627 f"{source[u'file-name']}: "
1628 f"{source[u'successful-downloads']}\n"
1632 def process_local_file(self, local_file, job=u"local", build_nr=1,
1634 """Process local XML file given as a command-line parameter.
1636 :param local_file: The file to process.
1637 :param job: Job name.
1638 :param build_nr: Build number.
1639 :param replace: If True, the information about jobs and builds is
1640 replaced by the new one, otherwise the new jobs and builds are
1642 :type local_file: str
1646 :raises: PresentationError if an error occurs.
1648 if not isfile(local_file):
1649 raise PresentationError(f"The file {local_file} does not exist.")
1652 build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1653 except (IndexError, ValueError):
1658 u"status": u"failed",
1659 u"file-name": local_file
1662 self._cfg.input = dict()
1663 self._cfg.add_build(job, build)
1665 logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1666 data = self._parse_tests(job, build)
1668 raise PresentationError(
1669 f"Error occurred while parsing the file {local_file}"
1672 build_data = pd.Series({
1673 u"metadata": pd.Series(
1674 list(data[u"metadata"].values()),
1675 index=list(data[u"metadata"].keys())
1677 u"suites": pd.Series(
1678 list(data[u"suites"].values()),
1679 index=list(data[u"suites"].keys())
1681 u"tests": pd.Series(
1682 list(data[u"tests"].values()),
1683 index=list(data[u"tests"].keys())
1687 if self._input_data.get(job, None) is None:
1688 self._input_data[job] = pd.Series()
1689 self._input_data[job][str(build_nr)] = build_data
1691 self._cfg.set_input_state(job, build_nr, u"processed")
1693 def process_local_directory(self, local_dir, replace=True):
1694 """Process local directory with XML file(s). The directory is processed
1695 as a 'job' and the XML files in it as builds.
1696 If the given directory contains only sub-directories, these
1697 sub-directories processed as jobs and corresponding XML files as builds
1700 :param local_dir: Local directory to process.
1701 :param replace: If True, the information about jobs and builds is
1702 replaced by the new one, otherwise the new jobs and builds are
1704 :type local_dir: str
1707 if not isdir(local_dir):
1708 raise PresentationError(
1709 f"The directory {local_dir} does not exist."
1712 # Check if the given directory includes only files, or only directories
1713 _, dirnames, filenames = next(walk(local_dir))
1715 if filenames and not dirnames:
1718 # key: dir (job) name, value: list of file names (builds)
1720 local_dir: [join(local_dir, name) for name in filenames]
1723 elif dirnames and not filenames:
1726 # key: dir (job) name, value: list of file names (builds)
1727 local_builds = dict()
1728 for dirname in dirnames:
1730 join(local_dir, dirname, name)
1731 for name in listdir(join(local_dir, dirname))
1732 if isfile(join(local_dir, dirname, name))
1735 local_builds[dirname] = sorted(builds)
1737 elif not filenames and not dirnames:
1738 raise PresentationError(f"The directory {local_dir} is empty.")
1740 raise PresentationError(
1741 f"The directory {local_dir} can include only files or only "
1742 f"directories, not both.\nThe directory {local_dir} includes "
1743 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1747 self._cfg.input = dict()
1749 for job, files in local_builds.items():
1750 for idx, local_file in enumerate(files):
1751 self.process_local_file(local_file, job, idx + 1, replace=False)
1754 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1755 """Return the index of character in the string which is the end of tag.
1757 :param tag_filter: The string where the end of tag is being searched.
1758 :param start: The index where the searching is stated.
1759 :param closer: The character which is the tag closer.
1760 :type tag_filter: str
1763 :returns: The index of the tag closer.
1767 idx_opener = tag_filter.index(closer, start)
1768 return tag_filter.index(closer, idx_opener + 1)
1773 def _condition(tag_filter):
1774 """Create a conditional statement from the given tag filter.
1776 :param tag_filter: Filter based on tags from the element specification.
1777 :type tag_filter: str
1778 :returns: Conditional statement which can be evaluated.
1783 index = InputData._end_of_tag(tag_filter, index)
1787 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1789 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1790 continue_on_error=False):
1791 """Filter required data from the given jobs and builds.
1793 The output data structure is:
1796 - test (or suite) 1 ID:
1802 - test (or suite) n ID:
1809 :param element: Element which will use the filtered data.
1810 :param params: Parameters which will be included in the output. If None,
1811 all parameters are included.
1812 :param data: If not None, this data is used instead of data specified
1814 :param data_set: The set of data to be filtered: tests, suites,
1816 :param continue_on_error: Continue if there is error while reading the
1817 data. The Item will be empty then
1818 :type element: pandas.Series
1822 :type continue_on_error: bool
1823 :returns: Filtered data.
1824 :rtype pandas.Series
1828 if data_set == "suites":
1830 elif element[u"filter"] in (u"all", u"template"):
1833 cond = InputData._condition(element[u"filter"])
1834 logging.debug(f" Filter: {cond}")
1836 logging.error(u" No filter defined.")
1840 params = element.get(u"parameters", None)
1842 params.extend((u"type", u"status"))
1844 data_to_filter = data if data else element[u"data"]
1847 for job, builds in data_to_filter.items():
1848 data[job] = pd.Series()
1849 for build in builds:
1850 data[job][str(build)] = pd.Series()
1853 self.data[job][str(build)][data_set].items())
1855 if continue_on_error:
1859 for test_id, test_data in data_dict.items():
1860 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1861 data[job][str(build)][test_id] = pd.Series()
1863 for param, val in test_data.items():
1864 data[job][str(build)][test_id][param] = val
1866 for param in params:
1868 data[job][str(build)][test_id][param] =\
1871 data[job][str(build)][test_id][param] =\
1875 except (KeyError, IndexError, ValueError) as err:
1877 f"Missing mandatory parameter in the element specification: "
1881 except AttributeError as err:
1882 logging.error(repr(err))
1884 except SyntaxError as err:
1886 f"The filter {cond} is not correct. Check if all tags are "
1887 f"enclosed by apostrophes.\n{repr(err)}"
1891 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1892 continue_on_error=False):
1893 """Filter required data from the given jobs and builds.
1895 The output data structure is:
1898 - test (or suite) 1 ID:
1904 - test (or suite) n ID:
1911 :param element: Element which will use the filtered data.
1912 :param params: Parameters which will be included in the output. If None,
1913 all parameters are included.
1914 :param data_set: The set of data to be filtered: tests, suites,
1916 :param continue_on_error: Continue if there is error while reading the
1917 data. The Item will be empty then
1918 :type element: pandas.Series
1921 :type continue_on_error: bool
1922 :returns: Filtered data.
1923 :rtype pandas.Series
1926 include = element.get(u"include", None)
1928 logging.warning(u"No tests to include, skipping the element.")
1932 params = element.get(u"parameters", None)
1933 if params and u"type" not in params:
1934 params.append(u"type")
1936 cores = element.get(u"core", None)
1940 for test in include:
1941 tests.append(test.format(core=core))
1947 for job, builds in element[u"data"].items():
1948 data[job] = pd.Series()
1949 for build in builds:
1950 data[job][str(build)] = pd.Series()
1953 reg_ex = re.compile(str(test).lower())
1954 for test_id in self.data[job][
1955 str(build)][data_set].keys():
1956 if re.match(reg_ex, str(test_id).lower()):
1957 test_data = self.data[job][
1958 str(build)][data_set][test_id]
1959 data[job][str(build)][test_id] = pd.Series()
1961 for param, val in test_data.items():
1962 data[job][str(build)][test_id]\
1965 for param in params:
1967 data[job][str(build)][
1971 data[job][str(build)][
1972 test_id][param] = u"No Data"
1973 except KeyError as err:
1974 if continue_on_error:
1975 logging.debug(repr(err))
1977 logging.error(repr(err))
1981 except (KeyError, IndexError, ValueError) as err:
1983 f"Missing mandatory parameter in the element "
1984 f"specification: {repr(err)}"
1987 except AttributeError as err:
1988 logging.error(repr(err))
1992 def merge_data(data):
1993 """Merge data from more jobs and builds to a simple data structure.
1995 The output data structure is:
1997 - test (suite) 1 ID:
2003 - test (suite) n ID:
2006 :param data: Data to merge.
2007 :type data: pandas.Series
2008 :returns: Merged data.
2009 :rtype: pandas.Series
2012 logging.info(u" Merging data ...")
2014 merged_data = pd.Series()
2015 for builds in data.values:
2016 for item in builds.values:
2017 for item_id, item_data in item.items():
2018 merged_data[item_id] = item_data
2021 def print_all_oper_data(self):
2022 """Print all operational data to console.
2030 u"Cycles per Packet",
2031 u"Average Vector Size"
2034 for job in self._input_data.values:
2035 for build in job.values:
2036 for test_id, test_data in build[u"tests"].items():
2038 if test_data.get(u"show-run", None) is None:
2040 for dut_name, data in test_data[u"show-run"].items():
2041 if data.get(u"threads", None) is None:
2043 print(f"Host IP: {data.get(u'host', '')}, "
2044 f"Socket: {data.get(u'socket', '')}")
2045 for thread_nr, thread in data[u"threads"].items():
2046 txt_table = prettytable.PrettyTable(tbl_hdr)
2049 txt_table.add_row(row)
2051 if len(thread) == 0:
2054 avg = f", Average Vector Size per Node: " \
2055 f"{(avg / len(thread)):.2f}"
2056 th_name = u"main" if thread_nr == 0 \
2057 else f"worker_{thread_nr}"
2058 print(f"{dut_name}, {th_name}{avg}")
2059 txt_table.float_format = u".2"
2060 txt_table.align = u"r"
2061 txt_table.align[u"Name"] = u"l"
2062 print(f"{txt_table.get_string()}\n")