1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
28 from os import remove, walk, listdir
29 from os.path import isfile, isdir, join
30 from datetime import datetime as dt
31 from datetime import timedelta
32 from json import loads
33 from json.decoder import JSONDecodeError
40 from robot.api import ExecutionResult, ResultVisitor
41 from robot import errors
43 from resources.libraries.python import jumpavg
44 from input_data_files import download_and_unzip_data_file
45 from pal_errors import PresentationError
48 # Separator used in file names
52 class ExecutionChecker(ResultVisitor):
53 """Class to traverse through the test suite structure.
55 The functionality implemented in this class generates a json structure:
61 "generated": "Timestamp",
62 "version": "SUT version",
63 "job": "Jenkins job name",
64 "build": "Information about the build"
67 "Suite long name 1": {
69 "doc": "Suite 1 documentation",
70 "parent": "Suite 1 parent",
71 "level": "Level of the suite in the suite hierarchy"
73 "Suite long name N": {
75 "doc": "Suite N documentation",
76 "parent": "Suite 2 parent",
77 "level": "Level of the suite in the suite hierarchy"
84 "parent": "Name of the parent of the test",
85 "doc": "Test documentation",
86 "msg": "Test message",
87 "conf-history": "DUT1 and DUT2 VAT History",
88 "show-run": "Show Run",
89 "tags": ["tag 1", "tag 2", "tag n"],
91 "status": "PASS" | "FAIL",
137 "parent": "Name of the parent of the test",
138 "doc": "Test documentation",
139 "msg": "Test message",
140 "tags": ["tag 1", "tag 2", "tag n"],
142 "status": "PASS" | "FAIL",
149 "parent": "Name of the parent of the test",
150 "doc": "Test documentation",
151 "msg": "Test message",
152 "tags": ["tag 1", "tag 2", "tag n"],
153 "type": "MRR" | "BMRR",
154 "status": "PASS" | "FAIL",
156 "receive-rate": float,
157 # Average of a list, computed using AvgStdevStats.
158 # In CSIT-1180, replace with List[float].
172 "metadata": { # Optional
173 "version": "VPP version",
174 "job": "Jenkins job name",
175 "build": "Information about the build"
179 "doc": "Suite 1 documentation",
180 "parent": "Suite 1 parent",
181 "level": "Level of the suite in the suite hierarchy"
184 "doc": "Suite N documentation",
185 "parent": "Suite 2 parent",
186 "level": "Level of the suite in the suite hierarchy"
192 "parent": "Name of the parent of the test",
193 "doc": "Test documentation"
194 "msg": "Test message"
195 "tags": ["tag 1", "tag 2", "tag n"],
196 "conf-history": "DUT1 and DUT2 VAT History"
197 "show-run": "Show Run"
198 "status": "PASS" | "FAIL"
206 .. note:: ID is the lowercase full path to the test.
209 REGEX_PLR_RATE = re.compile(
210 r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
211 r'PLRsearch upper bound::?\s(\d+.\d+)'
213 REGEX_NDRPDR_RATE = re.compile(
214 r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
215 r'NDR_UPPER:\s(\d+.\d+).*\n'
216 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
217 r'PDR_UPPER:\s(\d+.\d+)'
219 REGEX_NDRPDR_GBPS = re.compile(
220 r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
221 r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
222 r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
223 r'PDR_UPPER:.*,\s(\d+.\d+)'
225 REGEX_PERF_MSG_INFO = re.compile(
226 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
227 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
228 r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
229 r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
230 r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
232 REGEX_CPS_MSG_INFO = re.compile(
233 r'NDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*\n'
234 r'PDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*'
236 REGEX_PPS_MSG_INFO = re.compile(
237 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
238 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*'
240 REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
242 # Needed for CPS and PPS tests
243 REGEX_NDRPDR_LAT_BASE = re.compile(
244 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
245 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
247 REGEX_NDRPDR_LAT = re.compile(
248 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
249 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
250 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
251 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
252 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
253 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
256 REGEX_VERSION_VPP = re.compile(
257 r"(return STDOUT Version:\s*|"
258 r"VPP Version:\s*|VPP version:\s*)(.*)"
260 REGEX_VERSION_DPDK = re.compile(
261 r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
263 REGEX_TCP = re.compile(
264 r'Total\s(rps|cps|throughput):\s(\d*).*$'
266 REGEX_MRR = re.compile(
267 r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
268 r'tx\s(\d*),\srx\s(\d*)'
270 REGEX_BMRR = re.compile(
271 r'.*trial results.*: \[(.*)\]'
273 REGEX_RECONF_LOSS = re.compile(
274 r'Packets lost due to reconfig: (\d*)'
276 REGEX_RECONF_TIME = re.compile(
277 r'Implied time lost: (\d*.[\de-]*)'
279 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
281 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
283 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
285 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
287 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
289 def __init__(self, metadata, mapping, ignore):
292 :param metadata: Key-value pairs to be included in "metadata" part of
294 :param mapping: Mapping of the old names of test cases to the new
296 :param ignore: List of TCs to be ignored.
302 # Type of message to parse out from the test messages
303 self._msg_type = None
309 self._timestamp = None
311 # Testbed. The testbed is identified by TG node IP address.
314 # Mapping of TCs long names
315 self._mapping = mapping
318 self._ignore = ignore
320 # Number of PAPI History messages found:
322 # 1 - PAPI History of DUT1
323 # 2 - PAPI History of DUT2
324 self._conf_history_lookup_nr = 0
326 self._sh_run_counter = 0
328 # Test ID of currently processed test- the lowercase full path to the
332 # The main data structure
334 u"metadata": OrderedDict(),
335 u"suites": OrderedDict(),
336 u"tests": OrderedDict()
339 # Save the provided metadata
340 for key, val in metadata.items():
341 self._data[u"metadata"][key] = val
343 # Dictionary defining the methods used to parse different types of
346 u"timestamp": self._get_timestamp,
347 u"vpp-version": self._get_vpp_version,
348 u"dpdk-version": self._get_dpdk_version,
349 # TODO: Remove when not needed:
350 u"teardown-vat-history": self._get_vat_history,
351 u"teardown-papi-history": self._get_papi_history,
352 u"test-show-runtime": self._get_show_run,
353 u"testbed": self._get_testbed
358 """Getter - Data parsed from the XML file.
360 :returns: Data parsed from the XML file.
365 def _get_data_from_mrr_test_msg(self, msg):
366 """Get info from message of MRR performance tests.
368 :param msg: Message to be processed.
370 :returns: Processed message or original message if a problem occurs.
374 groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
375 if not groups or groups.lastindex != 1:
376 return u"Test Failed."
379 data = groups.group(1).split(u", ")
380 except (AttributeError, IndexError, ValueError, KeyError):
381 return u"Test Failed."
386 out_str += f"{(float(item) / 1e6):.2f}, "
387 return out_str[:-2] + u"]"
388 except (AttributeError, IndexError, ValueError, KeyError):
389 return u"Test Failed."
391 def _get_data_from_cps_test_msg(self, msg):
392 """Get info from message of NDRPDR CPS tests.
394 :param msg: Message to be processed.
396 :returns: Processed message or "Test Failed." if a problem occurs.
400 groups = re.search(self.REGEX_CPS_MSG_INFO, msg)
401 if not groups or groups.lastindex != 2:
402 return u"Test Failed."
406 f"1. {(float(groups.group(1)) / 1e6):5.2f}\n"
407 f"2. {(float(groups.group(2)) / 1e6):5.2f}"
409 except (AttributeError, IndexError, ValueError, KeyError):
410 return u"Test Failed."
412 def _get_data_from_pps_test_msg(self, msg):
413 """Get info from message of NDRPDR PPS tests.
415 :param msg: Message to be processed.
417 :returns: Processed message or "Test Failed." if a problem occurs.
421 groups = re.search(self.REGEX_PPS_MSG_INFO, msg)
422 if not groups or groups.lastindex != 4:
423 return u"Test Failed."
427 f"1. {(float(groups.group(1)) / 1e6):5.2f} "
428 f"{float(groups.group(2)):5.2f}\n"
429 f"2. {(float(groups.group(3)) / 1e6):5.2f} "
430 f"{float(groups.group(4)):5.2f}"
432 except (AttributeError, IndexError, ValueError, KeyError):
433 return u"Test Failed."
435 def _get_data_from_perf_test_msg(self, msg):
436 """Get info from message of NDRPDR performance tests.
438 :param msg: Message to be processed.
440 :returns: Processed message or "Test Failed." if a problem occurs.
444 groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
445 if not groups or groups.lastindex != 10:
446 return u"Test Failed."
450 u"ndr_low": float(groups.group(1)),
451 u"ndr_low_b": float(groups.group(2)),
452 u"pdr_low": float(groups.group(3)),
453 u"pdr_low_b": float(groups.group(4)),
454 u"pdr_lat_90_1": groups.group(5),
455 u"pdr_lat_90_2": groups.group(6),
456 u"pdr_lat_50_1": groups.group(7),
457 u"pdr_lat_50_2": groups.group(8),
458 u"pdr_lat_10_1": groups.group(9),
459 u"pdr_lat_10_2": groups.group(10),
461 except (AttributeError, IndexError, ValueError, KeyError):
462 return u"Test Failed."
464 def _process_lat(in_str_1, in_str_2):
465 """Extract min, avg, max values from latency string.
467 :param in_str_1: Latency string for one direction produced by robot
469 :param in_str_2: Latency string for second direction produced by
473 :returns: Processed latency string or None if a problem occurs.
476 in_list_1 = in_str_1.split('/', 3)
477 in_list_2 = in_str_2.split('/', 3)
479 if len(in_list_1) != 4 and len(in_list_2) != 4:
482 in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
484 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
485 except hdrh.codec.HdrLengthException:
488 in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
490 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
491 except hdrh.codec.HdrLengthException:
494 if hdr_lat_1 and hdr_lat_2:
496 hdr_lat_1.get_value_at_percentile(50.0),
497 hdr_lat_1.get_value_at_percentile(90.0),
498 hdr_lat_1.get_value_at_percentile(99.0),
499 hdr_lat_2.get_value_at_percentile(50.0),
500 hdr_lat_2.get_value_at_percentile(90.0),
501 hdr_lat_2.get_value_at_percentile(99.0)
511 f"1. {(data[u'ndr_low'] / 1e6):5.2f} "
512 f"{data[u'ndr_low_b']:5.2f}"
513 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f} "
514 f"{data[u'pdr_low_b']:5.2f}"
517 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
518 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
519 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
522 max_len = len(str(max((max(item) for item in latency))))
523 max_len = 4 if max_len < 4 else max_len
525 for idx, lat in enumerate(latency):
530 f"{lat[0]:{max_len}d} "
531 f"{lat[1]:{max_len}d} "
532 f"{lat[2]:{max_len}d} "
533 f"{lat[3]:{max_len}d} "
534 f"{lat[4]:{max_len}d} "
535 f"{lat[5]:{max_len}d} "
540 except (AttributeError, IndexError, ValueError, KeyError):
541 return u"Test Failed."
543 def _get_testbed(self, msg):
544 """Called when extraction of testbed IP is required.
545 The testbed is identified by TG node IP address.
547 :param msg: Message to process.
552 if msg.message.count(u"Setup of TG node") or \
553 msg.message.count(u"Setup of node TG host"):
554 reg_tg_ip = re.compile(
555 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
557 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
558 except (KeyError, ValueError, IndexError, AttributeError):
561 self._data[u"metadata"][u"testbed"] = self._testbed
562 self._msg_type = None
564 def _get_vpp_version(self, msg):
565 """Called when extraction of VPP version is required.
567 :param msg: Message to process.
572 if msg.message.count(u"return STDOUT Version:") or \
573 msg.message.count(u"VPP Version:") or \
574 msg.message.count(u"VPP version:"):
575 self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
577 self._data[u"metadata"][u"version"] = self._version
578 self._msg_type = None
580 def _get_dpdk_version(self, msg):
581 """Called when extraction of DPDK version is required.
583 :param msg: Message to process.
588 if msg.message.count(u"DPDK Version:"):
590 self._version = str(re.search(
591 self.REGEX_VERSION_DPDK, msg.message).group(2))
592 self._data[u"metadata"][u"version"] = self._version
596 self._msg_type = None
598 def _get_timestamp(self, msg):
599 """Called when extraction of timestamp is required.
601 :param msg: Message to process.
606 self._timestamp = msg.timestamp[:14]
607 self._data[u"metadata"][u"generated"] = self._timestamp
608 self._msg_type = None
610 def _get_vat_history(self, msg):
611 """Called when extraction of VAT command history is required.
613 TODO: Remove when not needed.
615 :param msg: Message to process.
619 if msg.message.count(u"VAT command history:"):
620 self._conf_history_lookup_nr += 1
621 if self._conf_history_lookup_nr == 1:
622 self._data[u"tests"][self._test_id][u"conf-history"] = str()
624 self._msg_type = None
625 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
626 r"VAT command history:", u"",
627 msg.message, count=1).replace(u'\n', u' |br| ').\
630 self._data[u"tests"][self._test_id][u"conf-history"] += (
631 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
634 def _get_papi_history(self, msg):
635 """Called when extraction of PAPI command history is required.
637 :param msg: Message to process.
641 if msg.message.count(u"PAPI command history:"):
642 self._conf_history_lookup_nr += 1
643 if self._conf_history_lookup_nr == 1:
644 self._data[u"tests"][self._test_id][u"conf-history"] = str()
646 self._msg_type = None
647 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
648 r"PAPI command history:", u"",
649 msg.message, count=1).replace(u'\n', u' |br| ').\
651 self._data[u"tests"][self._test_id][u"conf-history"] += (
652 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
655 def _get_show_run(self, msg):
656 """Called when extraction of VPP operational data (output of CLI command
657 Show Runtime) is required.
659 :param msg: Message to process.
664 if not msg.message.count(u"stats runtime"):
668 if self._sh_run_counter > 1:
671 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
672 self._data[u"tests"][self._test_id][u"show-run"] = dict()
674 groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
678 host = groups.group(1)
679 except (AttributeError, IndexError):
682 sock = groups.group(2)
683 except (AttributeError, IndexError):
686 runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
687 replace(u"'", u'"').replace(u'b"', u'"').
688 replace(u'u"', u'"').split(u":", 1)[1])
691 threads_nr = len(runtime[0][u"clocks"])
692 except (IndexError, KeyError):
695 dut = u"DUT{nr}".format(
696 nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
701 u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
705 for idx in range(threads_nr):
706 if item[u"vectors"][idx] > 0:
707 clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
708 elif item[u"calls"][idx] > 0:
709 clocks = item[u"clocks"][idx] / item[u"calls"][idx]
710 elif item[u"suspends"][idx] > 0:
711 clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
715 if item[u"calls"][idx] > 0:
716 vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
720 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
721 int(item[u"suspends"][idx]):
722 oper[u"threads"][idx].append([
725 item[u"vectors"][idx],
726 item[u"suspends"][idx],
731 self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
733 def _get_ndrpdr_throughput(self, msg):
734 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
737 :param msg: The test message to be parsed.
739 :returns: Parsed data as a dict and the status (PASS/FAIL).
740 :rtype: tuple(dict, str)
744 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
745 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
748 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
750 if groups is not None:
752 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
753 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
754 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
755 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
757 except (IndexError, ValueError):
760 return throughput, status
762 def _get_ndrpdr_throughput_gbps(self, msg):
763 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
766 :param msg: The test message to be parsed.
768 :returns: Parsed data as a dict and the status (PASS/FAIL).
769 :rtype: tuple(dict, str)
773 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
774 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
777 groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
779 if groups is not None:
781 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
782 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
783 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
784 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
786 except (IndexError, ValueError):
791 def _get_plr_throughput(self, msg):
792 """Get PLRsearch lower bound and PLRsearch upper bound from the test
795 :param msg: The test message to be parsed.
797 :returns: Parsed data as a dict and the status (PASS/FAIL).
798 :rtype: tuple(dict, str)
806 groups = re.search(self.REGEX_PLR_RATE, msg)
808 if groups is not None:
810 throughput[u"LOWER"] = float(groups.group(1))
811 throughput[u"UPPER"] = float(groups.group(2))
813 except (IndexError, ValueError):
816 return throughput, status
818 def _get_ndrpdr_latency(self, msg):
819 """Get LATENCY from the test message.
821 :param msg: The test message to be parsed.
823 :returns: Parsed data as a dict and the status (PASS/FAIL).
824 :rtype: tuple(dict, str)
834 u"direction1": copy.copy(latency_default),
835 u"direction2": copy.copy(latency_default)
838 u"direction1": copy.copy(latency_default),
839 u"direction2": copy.copy(latency_default)
842 u"direction1": copy.copy(latency_default),
843 u"direction2": copy.copy(latency_default)
846 u"direction1": copy.copy(latency_default),
847 u"direction2": copy.copy(latency_default)
850 u"direction1": copy.copy(latency_default),
851 u"direction2": copy.copy(latency_default)
854 u"direction1": copy.copy(latency_default),
855 u"direction2": copy.copy(latency_default)
859 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
861 groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
863 return latency, u"FAIL"
865 def process_latency(in_str):
866 """Return object with parsed latency values.
868 TODO: Define class for the return type.
870 :param in_str: Input string, min/avg/max/hdrh format.
872 :returns: Dict with corresponding keys, except hdrh float values.
874 :throws IndexError: If in_str does not have enough substrings.
875 :throws ValueError: If a substring does not convert to float.
877 in_list = in_str.split('/', 3)
880 u"min": float(in_list[0]),
881 u"avg": float(in_list[1]),
882 u"max": float(in_list[2]),
886 if len(in_list) == 4:
887 rval[u"hdrh"] = str(in_list[3])
892 latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
893 latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
894 latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
895 latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
896 if groups.lastindex == 4:
897 return latency, u"PASS"
898 except (IndexError, ValueError):
902 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
903 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
904 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
905 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
906 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
907 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
908 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
909 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
910 if groups.lastindex == 12:
911 return latency, u"PASS"
912 except (IndexError, ValueError):
915 # TODO: Remove when not needed
916 latency[u"NDR10"] = {
917 u"direction1": copy.copy(latency_default),
918 u"direction2": copy.copy(latency_default)
920 latency[u"NDR50"] = {
921 u"direction1": copy.copy(latency_default),
922 u"direction2": copy.copy(latency_default)
924 latency[u"NDR90"] = {
925 u"direction1": copy.copy(latency_default),
926 u"direction2": copy.copy(latency_default)
929 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
930 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
931 latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
932 latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
933 latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
934 latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
935 latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
936 latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
937 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
938 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
939 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
940 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
941 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
942 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
943 return latency, u"PASS"
944 except (IndexError, ValueError):
947 return latency, u"FAIL"
950 def _get_hoststack_data(msg, tags):
951 """Get data from the hoststack test message.
953 :param msg: The test message to be parsed.
954 :param tags: Test tags.
957 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
958 :rtype: tuple(dict, str)
963 msg = msg.replace(u"'", u'"').replace(u" ", u"")
964 if u"LDPRELOAD" in tags:
968 except JSONDecodeError:
970 elif u"VPPECHO" in tags:
972 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
974 client=loads(msg_lst[0]),
975 server=loads(msg_lst[1])
978 except (JSONDecodeError, IndexError):
981 return result, status
983 def visit_suite(self, suite):
984 """Implements traversing through the suite and its direct children.
986 :param suite: Suite to process.
990 if self.start_suite(suite) is not False:
991 suite.suites.visit(self)
992 suite.tests.visit(self)
993 self.end_suite(suite)
995 def start_suite(self, suite):
996 """Called when suite starts.
998 :param suite: Suite to process.
1004 parent_name = suite.parent.name
1005 except AttributeError:
1008 doc_str = suite.doc.\
1009 replace(u'"', u"'").\
1010 replace(u'\n', u' ').\
1011 replace(u'\r', u'').\
1012 replace(u'*[', u' |br| *[').\
1013 replace(u"*", u"**").\
1014 replace(u' |br| *[', u'*[', 1)
1016 self._data[u"suites"][suite.longname.lower().
1017 replace(u'"', u"'").
1018 replace(u" ", u"_")] = {
1019 u"name": suite.name.lower(),
1021 u"parent": parent_name,
1022 u"level": len(suite.longname.split(u"."))
1025 suite.keywords.visit(self)
1027 def end_suite(self, suite):
1028 """Called when suite ends.
1030 :param suite: Suite to process.
1035 def visit_test(self, test):
1036 """Implements traversing through the test.
1038 :param test: Test to process.
1042 if self.start_test(test) is not False:
1043 test.keywords.visit(self)
1046 def start_test(self, test):
1047 """Called when test starts.
1049 :param test: Test to process.
1054 self._sh_run_counter = 0
1056 longname_orig = test.longname.lower()
1058 # Check the ignore list
1059 if longname_orig in self._ignore:
1062 tags = [str(tag) for tag in test.tags]
1063 test_result = dict()
1065 # Change the TC long name and name if defined in the mapping table
1066 longname = self._mapping.get(longname_orig, None)
1067 if longname is not None:
1068 name = longname.split(u'.')[-1]
1070 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1074 longname = longname_orig
1075 name = test.name.lower()
1077 # Remove TC number from the TC long name (backward compatibility):
1078 self._test_id = re.sub(
1079 self.REGEX_TC_NUMBER, u"", longname.replace(u"snat", u"nat")
1081 # Remove TC number from the TC name (not needed):
1082 test_result[u"name"] = re.sub(
1083 self.REGEX_TC_NUMBER, "", name.replace(u"snat", u"nat")
1086 test_result[u"parent"] = test.parent.name.lower().\
1087 replace(u"snat", u"nat")
1088 test_result[u"tags"] = tags
1089 test_result["doc"] = test.doc.\
1090 replace(u'"', u"'").\
1091 replace(u'\n', u' ').\
1092 replace(u'\r', u'').\
1093 replace(u'[', u' |br| [').\
1094 replace(u' |br| [', u'[', 1)
1095 test_result[u"type"] = u"FUNC"
1096 test_result[u"status"] = test.status
1098 if test.status == u"PASS":
1099 if u"NDRPDR" in tags:
1100 if u"TCP_PPS" in tags or u"UDP_PPS" in tags:
1101 test_result[u"msg"] = self._get_data_from_pps_test_msg(
1102 test.message).replace(u'\n', u' |br| '). \
1103 replace(u'\r', u'').replace(u'"', u"'")
1104 elif u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1105 test_result[u"msg"] = self._get_data_from_cps_test_msg(
1106 test.message).replace(u'\n', u' |br| '). \
1107 replace(u'\r', u'').replace(u'"', u"'")
1109 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1110 test.message).replace(u'\n', u' |br| ').\
1111 replace(u'\r', u'').replace(u'"', u"'")
1112 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1113 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1114 test.message).replace(u'\n', u' |br| ').\
1115 replace(u'\r', u'').replace(u'"', u"'")
1117 test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\
1118 replace(u'\r', u'').replace(u'"', u"'")
1120 test_result[u"msg"] = u"Test Failed."
1122 if u"PERFTEST" in tags:
1123 # Replace info about cores (e.g. -1c-) with the info about threads
1124 # and cores (e.g. -1t1c-) in the long test case names and in the
1125 # test case names if necessary.
1126 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1130 for tag in test_result[u"tags"]:
1131 groups = re.search(self.REGEX_TC_TAG, tag)
1137 self._test_id = re.sub(
1138 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1139 self._test_id, count=1
1141 test_result[u"name"] = re.sub(
1142 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1143 test_result["name"], count=1
1146 test_result[u"status"] = u"FAIL"
1147 self._data[u"tests"][self._test_id] = test_result
1149 f"The test {self._test_id} has no or more than one "
1150 f"multi-threading tags.\n"
1151 f"Tags: {test_result[u'tags']}"
1155 if test.status == u"PASS":
1156 if u"DEVICETEST" in tags:
1157 test_result[u"type"] = u"DEVICETEST"
1158 elif u"NDRPDR" in tags:
1159 if u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1160 test_result[u"type"] = u"CPS"
1162 test_result[u"type"] = u"NDRPDR"
1163 test_result[u"throughput"], test_result[u"status"] = \
1164 self._get_ndrpdr_throughput(test.message)
1165 test_result[u"gbps"], test_result[u"status"] = \
1166 self._get_ndrpdr_throughput_gbps(test.message)
1167 test_result[u"latency"], test_result[u"status"] = \
1168 self._get_ndrpdr_latency(test.message)
1169 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1171 test_result[u"type"] = u"MRR"
1173 test_result[u"type"] = u"BMRR"
1175 test_result[u"result"] = dict()
1176 groups = re.search(self.REGEX_BMRR, test.message)
1177 if groups is not None:
1178 items_str = groups.group(1)
1180 float(item.strip().replace(u"'", u""))
1181 for item in items_str.split(",")
1183 # Use whole list in CSIT-1180.
1184 stats = jumpavg.AvgStdevStats.for_runs(items_float)
1185 test_result[u"result"][u"receive-rate"] = stats.avg
1186 test_result[u"result"][u"receive-stdev"] = stats.stdev
1188 groups = re.search(self.REGEX_MRR, test.message)
1189 test_result[u"result"][u"receive-rate"] = \
1190 float(groups.group(3)) / float(groups.group(1))
1191 elif u"SOAK" in tags:
1192 test_result[u"type"] = u"SOAK"
1193 test_result[u"throughput"], test_result[u"status"] = \
1194 self._get_plr_throughput(test.message)
1195 elif u"HOSTSTACK" in tags:
1196 test_result[u"type"] = u"HOSTSTACK"
1197 test_result[u"result"], test_result[u"status"] = \
1198 self._get_hoststack_data(test.message, tags)
1199 elif u"TCP" in tags:
1200 test_result[u"type"] = u"TCP"
1201 groups = re.search(self.REGEX_TCP, test.message)
1202 test_result[u"result"] = int(groups.group(2))
1203 elif u"RECONF" in tags:
1204 test_result[u"type"] = u"RECONF"
1205 test_result[u"result"] = None
1207 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1208 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1209 test_result[u"result"] = {
1210 u"loss": int(grps_loss.group(1)),
1211 u"time": float(grps_time.group(1))
1213 except (AttributeError, IndexError, ValueError, TypeError):
1214 test_result[u"status"] = u"FAIL"
1216 test_result[u"status"] = u"FAIL"
1217 self._data[u"tests"][self._test_id] = test_result
1220 self._data[u"tests"][self._test_id] = test_result
1222 def end_test(self, test):
1223 """Called when test ends.
1225 :param test: Test to process.
1230 def visit_keyword(self, keyword):
1231 """Implements traversing through the keyword and its child keywords.
1233 :param keyword: Keyword to process.
1234 :type keyword: Keyword
1237 if self.start_keyword(keyword) is not False:
1238 self.end_keyword(keyword)
1240 def start_keyword(self, keyword):
1241 """Called when keyword starts. Default implementation does nothing.
1243 :param keyword: Keyword to process.
1244 :type keyword: Keyword
1248 if keyword.type == u"setup":
1249 self.visit_setup_kw(keyword)
1250 elif keyword.type == u"teardown":
1251 self.visit_teardown_kw(keyword)
1253 self.visit_test_kw(keyword)
1254 except AttributeError:
1257 def end_keyword(self, keyword):
1258 """Called when keyword ends. Default implementation does nothing.
1260 :param keyword: Keyword to process.
1261 :type keyword: Keyword
1265 def visit_test_kw(self, test_kw):
1266 """Implements traversing through the test keyword and its child
1269 :param test_kw: Keyword to process.
1270 :type test_kw: Keyword
1273 for keyword in test_kw.keywords:
1274 if self.start_test_kw(keyword) is not False:
1275 self.visit_test_kw(keyword)
1276 self.end_test_kw(keyword)
1278 def start_test_kw(self, test_kw):
1279 """Called when test keyword starts. Default implementation does
1282 :param test_kw: Keyword to process.
1283 :type test_kw: Keyword
1286 if test_kw.name.count(u"Show Runtime On All Duts") or \
1287 test_kw.name.count(u"Show Runtime Counters On All Duts") or \
1288 test_kw.name.count(u"Vpp Show Runtime On All Duts"):
1289 self._msg_type = u"test-show-runtime"
1290 self._sh_run_counter += 1
1293 test_kw.messages.visit(self)
1295 def end_test_kw(self, test_kw):
1296 """Called when keyword ends. Default implementation does nothing.
1298 :param test_kw: Keyword to process.
1299 :type test_kw: Keyword
1303 def visit_setup_kw(self, setup_kw):
1304 """Implements traversing through the teardown keyword and its child
1307 :param setup_kw: Keyword to process.
1308 :type setup_kw: Keyword
1311 for keyword in setup_kw.keywords:
1312 if self.start_setup_kw(keyword) is not False:
1313 self.visit_setup_kw(keyword)
1314 self.end_setup_kw(keyword)
1316 def start_setup_kw(self, setup_kw):
1317 """Called when teardown keyword starts. Default implementation does
1320 :param setup_kw: Keyword to process.
1321 :type setup_kw: Keyword
1324 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1325 and not self._version:
1326 self._msg_type = u"vpp-version"
1327 elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1329 self._msg_type = u"dpdk-version"
1330 elif setup_kw.name.count(u"Set Global Variable") \
1331 and not self._timestamp:
1332 self._msg_type = u"timestamp"
1333 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1334 self._msg_type = u"testbed"
1337 setup_kw.messages.visit(self)
1339 def end_setup_kw(self, setup_kw):
1340 """Called when keyword ends. Default implementation does nothing.
1342 :param setup_kw: Keyword to process.
1343 :type setup_kw: Keyword
1347 def visit_teardown_kw(self, teardown_kw):
1348 """Implements traversing through the teardown keyword and its child
1351 :param teardown_kw: Keyword to process.
1352 :type teardown_kw: Keyword
1355 for keyword in teardown_kw.keywords:
1356 if self.start_teardown_kw(keyword) is not False:
1357 self.visit_teardown_kw(keyword)
1358 self.end_teardown_kw(keyword)
1360 def start_teardown_kw(self, teardown_kw):
1361 """Called when teardown keyword starts
1363 :param teardown_kw: Keyword to process.
1364 :type teardown_kw: Keyword
1368 if teardown_kw.name.count(u"Show Vat History On All Duts"):
1369 # TODO: Remove when not needed:
1370 self._conf_history_lookup_nr = 0
1371 self._msg_type = u"teardown-vat-history"
1372 teardown_kw.messages.visit(self)
1373 elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1374 self._conf_history_lookup_nr = 0
1375 self._msg_type = u"teardown-papi-history"
1376 teardown_kw.messages.visit(self)
1378 def end_teardown_kw(self, teardown_kw):
1379 """Called when keyword ends. Default implementation does nothing.
1381 :param teardown_kw: Keyword to process.
1382 :type teardown_kw: Keyword
1386 def visit_message(self, msg):
1387 """Implements visiting the message.
1389 :param msg: Message to process.
1393 if self.start_message(msg) is not False:
1394 self.end_message(msg)
1396 def start_message(self, msg):
1397 """Called when message starts. Get required information from messages:
1400 :param msg: Message to process.
1405 self.parse_msg[self._msg_type](msg)
1407 def end_message(self, msg):
1408 """Called when message ends. Default implementation does nothing.
1410 :param msg: Message to process.
1419 The data is extracted from output.xml files generated by Jenkins jobs and
1420 stored in pandas' DataFrames.
1426 (as described in ExecutionChecker documentation)
1428 (as described in ExecutionChecker documentation)
1430 (as described in ExecutionChecker documentation)
1433 def __init__(self, spec):
1436 :param spec: Specification.
1437 :type spec: Specification
1444 self._input_data = pd.Series()
1448 """Getter - Input data.
1450 :returns: Input data
1451 :rtype: pandas.Series
1453 return self._input_data
1455 def metadata(self, job, build):
1456 """Getter - metadata
1458 :param job: Job which metadata we want.
1459 :param build: Build which metadata we want.
1463 :rtype: pandas.Series
1465 return self.data[job][build][u"metadata"]
1467 def suites(self, job, build):
1470 :param job: Job which suites we want.
1471 :param build: Build which suites we want.
1475 :rtype: pandas.Series
1477 return self.data[job][str(build)][u"suites"]
1479 def tests(self, job, build):
1482 :param job: Job which tests we want.
1483 :param build: Build which tests we want.
1487 :rtype: pandas.Series
1489 return self.data[job][build][u"tests"]
1491 def _parse_tests(self, job, build):
1492 """Process data from robot output.xml file and return JSON structured
1495 :param job: The name of job which build output data will be processed.
1496 :param build: The build which output data will be processed.
1499 :returns: JSON data structure.
1508 with open(build[u"file-name"], u'r') as data_file:
1510 result = ExecutionResult(data_file)
1511 except errors.DataError as err:
1513 f"Error occurred while parsing output.xml: {repr(err)}"
1516 checker = ExecutionChecker(metadata, self._cfg.mapping,
1518 result.visit(checker)
1522 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1523 """Download and parse the input data file.
1525 :param pid: PID of the process executing this method.
1526 :param job: Name of the Jenkins job which generated the processed input
1528 :param build: Information about the Jenkins build which generated the
1529 processed input file.
1530 :param repeat: Repeat the download specified number of times if not
1538 logging.info(f" Processing the job/build: {job}: {build[u'build']}")
1545 success = download_and_unzip_data_file(self._cfg, job, build, pid)
1551 f"It is not possible to download the input data file from the "
1552 f"job {job}, build {build[u'build']}, or it is damaged. "
1556 logging.info(f" Processing data from build {build[u'build']}")
1557 data = self._parse_tests(job, build)
1560 f"Input data file from the job {job}, build "
1561 f"{build[u'build']} is damaged. Skipped."
1564 state = u"processed"
1567 remove(build[u"file-name"])
1568 except OSError as err:
1570 f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1573 # If the time-period is defined in the specification file, remove all
1574 # files which are outside the time period.
1576 timeperiod = self._cfg.input.get(u"time-period", None)
1577 if timeperiod and data:
1579 timeperiod = timedelta(int(timeperiod))
1580 metadata = data.get(u"metadata", None)
1582 generated = metadata.get(u"generated", None)
1584 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1585 if (now - generated) > timeperiod:
1586 # Remove the data and the file:
1591 f" The build {job}/{build[u'build']} is "
1592 f"outdated, will be removed."
1594 logging.info(u" Done.")
1604 def download_and_parse_data(self, repeat=1):
1605 """Download the input data files, parse input data from input files and
1606 store in pandas' Series.
1608 :param repeat: Repeat the download specified number of times if not
1613 logging.info(u"Downloading and parsing input files ...")
1615 for job, builds in self._cfg.builds.items():
1616 for build in builds:
1618 result = self._download_and_parse_build(job, build, repeat)
1621 build_nr = result[u"build"][u"build"]
1624 data = result[u"data"]
1625 build_data = pd.Series({
1626 u"metadata": pd.Series(
1627 list(data[u"metadata"].values()),
1628 index=list(data[u"metadata"].keys())
1630 u"suites": pd.Series(
1631 list(data[u"suites"].values()),
1632 index=list(data[u"suites"].keys())
1634 u"tests": pd.Series(
1635 list(data[u"tests"].values()),
1636 index=list(data[u"tests"].keys())
1640 if self._input_data.get(job, None) is None:
1641 self._input_data[job] = pd.Series()
1642 self._input_data[job][str(build_nr)] = build_data
1644 self._cfg.set_input_file_name(
1645 job, build_nr, result[u"build"][u"file-name"])
1647 self._cfg.set_input_state(job, build_nr, result[u"state"])
1650 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1651 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1653 logging.info(u"Done.")
1655 def process_local_file(self, local_file, job=u"local", build_nr=1,
1657 """Process local XML file given as a command-line parameter.
1659 :param local_file: The file to process.
1660 :param job: Job name.
1661 :param build_nr: Build number.
1662 :param replace: If True, the information about jobs and builds is
1663 replaced by the new one, otherwise the new jobs and builds are
1665 :type local_file: str
1669 :raises: PresentationError if an error occurs.
1671 if not isfile(local_file):
1672 raise PresentationError(f"The file {local_file} does not exist.")
1675 build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1676 except (IndexError, ValueError):
1681 u"status": u"failed",
1682 u"file-name": local_file
1685 self._cfg.builds = dict()
1686 self._cfg.add_build(job, build)
1688 logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1689 data = self._parse_tests(job, build)
1691 raise PresentationError(
1692 f"Error occurred while parsing the file {local_file}"
1695 build_data = pd.Series({
1696 u"metadata": pd.Series(
1697 list(data[u"metadata"].values()),
1698 index=list(data[u"metadata"].keys())
1700 u"suites": pd.Series(
1701 list(data[u"suites"].values()),
1702 index=list(data[u"suites"].keys())
1704 u"tests": pd.Series(
1705 list(data[u"tests"].values()),
1706 index=list(data[u"tests"].keys())
1710 if self._input_data.get(job, None) is None:
1711 self._input_data[job] = pd.Series()
1712 self._input_data[job][str(build_nr)] = build_data
1714 self._cfg.set_input_state(job, build_nr, u"processed")
1716 def process_local_directory(self, local_dir, replace=True):
1717 """Process local directory with XML file(s). The directory is processed
1718 as a 'job' and the XML files in it as builds.
1719 If the given directory contains only sub-directories, these
1720 sub-directories processed as jobs and corresponding XML files as builds
1723 :param local_dir: Local directory to process.
1724 :param replace: If True, the information about jobs and builds is
1725 replaced by the new one, otherwise the new jobs and builds are
1727 :type local_dir: str
1730 if not isdir(local_dir):
1731 raise PresentationError(
1732 f"The directory {local_dir} does not exist."
1735 # Check if the given directory includes only files, or only directories
1736 _, dirnames, filenames = next(walk(local_dir))
1738 if filenames and not dirnames:
1741 # key: dir (job) name, value: list of file names (builds)
1743 local_dir: [join(local_dir, name) for name in filenames]
1746 elif dirnames and not filenames:
1749 # key: dir (job) name, value: list of file names (builds)
1750 local_builds = dict()
1751 for dirname in dirnames:
1753 join(local_dir, dirname, name)
1754 for name in listdir(join(local_dir, dirname))
1755 if isfile(join(local_dir, dirname, name))
1758 local_builds[dirname] = sorted(builds)
1760 elif not filenames and not dirnames:
1761 raise PresentationError(f"The directory {local_dir} is empty.")
1763 raise PresentationError(
1764 f"The directory {local_dir} can include only files or only "
1765 f"directories, not both.\nThe directory {local_dir} includes "
1766 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1770 self._cfg.builds = dict()
1772 for job, files in local_builds.items():
1773 for idx, local_file in enumerate(files):
1774 self.process_local_file(local_file, job, idx + 1, replace=False)
1777 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1778 """Return the index of character in the string which is the end of tag.
1780 :param tag_filter: The string where the end of tag is being searched.
1781 :param start: The index where the searching is stated.
1782 :param closer: The character which is the tag closer.
1783 :type tag_filter: str
1786 :returns: The index of the tag closer.
1790 idx_opener = tag_filter.index(closer, start)
1791 return tag_filter.index(closer, idx_opener + 1)
1796 def _condition(tag_filter):
1797 """Create a conditional statement from the given tag filter.
1799 :param tag_filter: Filter based on tags from the element specification.
1800 :type tag_filter: str
1801 :returns: Conditional statement which can be evaluated.
1806 index = InputData._end_of_tag(tag_filter, index)
1810 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1812 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1813 continue_on_error=False):
1814 """Filter required data from the given jobs and builds.
1816 The output data structure is:
1819 - test (or suite) 1 ID:
1825 - test (or suite) n ID:
1832 :param element: Element which will use the filtered data.
1833 :param params: Parameters which will be included in the output. If None,
1834 all parameters are included.
1835 :param data: If not None, this data is used instead of data specified
1837 :param data_set: The set of data to be filtered: tests, suites,
1839 :param continue_on_error: Continue if there is error while reading the
1840 data. The Item will be empty then
1841 :type element: pandas.Series
1845 :type continue_on_error: bool
1846 :returns: Filtered data.
1847 :rtype pandas.Series
1851 if data_set == "suites":
1853 elif element[u"filter"] in (u"all", u"template"):
1856 cond = InputData._condition(element[u"filter"])
1857 logging.debug(f" Filter: {cond}")
1859 logging.error(u" No filter defined.")
1863 params = element.get(u"parameters", None)
1865 params.append(u"type")
1867 data_to_filter = data if data else element[u"data"]
1870 for job, builds in data_to_filter.items():
1871 data[job] = pd.Series()
1872 for build in builds:
1873 data[job][str(build)] = pd.Series()
1876 self.data[job][str(build)][data_set].items())
1878 if continue_on_error:
1882 for test_id, test_data in data_dict.items():
1883 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1884 data[job][str(build)][test_id] = pd.Series()
1886 for param, val in test_data.items():
1887 data[job][str(build)][test_id][param] = val
1889 for param in params:
1891 data[job][str(build)][test_id][param] =\
1894 data[job][str(build)][test_id][param] =\
1898 except (KeyError, IndexError, ValueError) as err:
1900 f"Missing mandatory parameter in the element specification: "
1904 except AttributeError as err:
1905 logging.error(repr(err))
1907 except SyntaxError as err:
1909 f"The filter {cond} is not correct. Check if all tags are "
1910 f"enclosed by apostrophes.\n{repr(err)}"
1914 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1915 continue_on_error=False):
1916 """Filter required data from the given jobs and builds.
1918 The output data structure is:
1921 - test (or suite) 1 ID:
1927 - test (or suite) n ID:
1934 :param element: Element which will use the filtered data.
1935 :param params: Parameters which will be included in the output. If None,
1936 all parameters are included.
1937 :param data_set: The set of data to be filtered: tests, suites,
1939 :param continue_on_error: Continue if there is error while reading the
1940 data. The Item will be empty then
1941 :type element: pandas.Series
1944 :type continue_on_error: bool
1945 :returns: Filtered data.
1946 :rtype pandas.Series
1949 include = element.get(u"include", None)
1951 logging.warning(u"No tests to include, skipping the element.")
1955 params = element.get(u"parameters", None)
1957 params.append(u"type")
1961 for job, builds in element[u"data"].items():
1962 data[job] = pd.Series()
1963 for build in builds:
1964 data[job][str(build)] = pd.Series()
1965 for test in include:
1967 reg_ex = re.compile(str(test).lower())
1968 for test_id in self.data[job][
1969 str(build)][data_set].keys():
1970 if re.match(reg_ex, str(test_id).lower()):
1971 test_data = self.data[job][
1972 str(build)][data_set][test_id]
1973 data[job][str(build)][test_id] = pd.Series()
1975 for param, val in test_data.items():
1976 data[job][str(build)][test_id]\
1979 for param in params:
1981 data[job][str(build)][
1985 data[job][str(build)][
1986 test_id][param] = u"No Data"
1987 except KeyError as err:
1988 if continue_on_error:
1989 logging.debug(repr(err))
1991 logging.error(repr(err))
1995 except (KeyError, IndexError, ValueError) as err:
1997 f"Missing mandatory parameter in the element "
1998 f"specification: {repr(err)}"
2001 except AttributeError as err:
2002 logging.error(repr(err))
2006 def merge_data(data):
2007 """Merge data from more jobs and builds to a simple data structure.
2009 The output data structure is:
2011 - test (suite) 1 ID:
2017 - test (suite) n ID:
2020 :param data: Data to merge.
2021 :type data: pandas.Series
2022 :returns: Merged data.
2023 :rtype: pandas.Series
2026 logging.info(u" Merging data ...")
2028 merged_data = pd.Series()
2029 for builds in data.values:
2030 for item in builds.values:
2031 for item_id, item_data in item.items():
2032 merged_data[item_id] = item_data
2035 def print_all_oper_data(self):
2036 """Print all operational data to console.
2044 u"Cycles per Packet",
2045 u"Average Vector Size"
2048 for job in self._input_data.values:
2049 for build in job.values:
2050 for test_id, test_data in build[u"tests"].items():
2052 if test_data.get(u"show-run", None) is None:
2054 for dut_name, data in test_data[u"show-run"].items():
2055 if data.get(u"threads", None) is None:
2057 print(f"Host IP: {data.get(u'host', '')}, "
2058 f"Socket: {data.get(u'socket', '')}")
2059 for thread_nr, thread in data[u"threads"].items():
2060 txt_table = prettytable.PrettyTable(tbl_hdr)
2063 txt_table.add_row(row)
2065 if len(thread) == 0:
2068 avg = f", Average Vector Size per Node: " \
2069 f"{(avg / len(thread)):.2f}"
2070 th_name = u"main" if thread_nr == 0 \
2071 else f"worker_{thread_nr}"
2072 print(f"{dut_name}, {th_name}{avg}")
2073 txt_table.float_format = u".2"
2074 txt_table.align = u"r"
2075 txt_table.align[u"Name"] = u"l"
2076 print(f"{txt_table.get_string()}\n")