1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
28 from os import remove, walk, listdir
29 from os.path import isfile, isdir, join
30 from datetime import datetime as dt
31 from datetime import timedelta
32 from json import loads
33 from json.decoder import JSONDecodeError
40 from robot.api import ExecutionResult, ResultVisitor
41 from robot import errors
43 from resources.libraries.python import jumpavg
44 from input_data_files import download_and_unzip_data_file
45 from pal_errors import PresentationError
48 # Separator used in file names
52 class ExecutionChecker(ResultVisitor):
53 """Class to traverse through the test suite structure.
55 The functionality implemented in this class generates a json structure:
61 "generated": "Timestamp",
62 "version": "SUT version",
63 "job": "Jenkins job name",
64 "build": "Information about the build"
67 "Suite long name 1": {
69 "doc": "Suite 1 documentation",
70 "parent": "Suite 1 parent",
71 "level": "Level of the suite in the suite hierarchy"
73 "Suite long name N": {
75 "doc": "Suite N documentation",
76 "parent": "Suite 2 parent",
77 "level": "Level of the suite in the suite hierarchy"
84 "parent": "Name of the parent of the test",
85 "doc": "Test documentation",
86 "msg": "Test message",
87 "conf-history": "DUT1 and DUT2 VAT History",
88 "show-run": "Show Run",
89 "tags": ["tag 1", "tag 2", "tag n"],
91 "status": "PASS" | "FAIL",
137 "parent": "Name of the parent of the test",
138 "doc": "Test documentation",
139 "msg": "Test message",
140 "tags": ["tag 1", "tag 2", "tag n"],
142 "status": "PASS" | "FAIL",
149 "parent": "Name of the parent of the test",
150 "doc": "Test documentation",
151 "msg": "Test message",
152 "tags": ["tag 1", "tag 2", "tag n"],
153 "type": "MRR" | "BMRR",
154 "status": "PASS" | "FAIL",
156 "receive-rate": float,
157 # Average of a list, computed using AvgStdevStats.
158 # In CSIT-1180, replace with List[float].
172 "metadata": { # Optional
173 "version": "VPP version",
174 "job": "Jenkins job name",
175 "build": "Information about the build"
179 "doc": "Suite 1 documentation",
180 "parent": "Suite 1 parent",
181 "level": "Level of the suite in the suite hierarchy"
184 "doc": "Suite N documentation",
185 "parent": "Suite 2 parent",
186 "level": "Level of the suite in the suite hierarchy"
192 "parent": "Name of the parent of the test",
193 "doc": "Test documentation"
194 "msg": "Test message"
195 "tags": ["tag 1", "tag 2", "tag n"],
196 "conf-history": "DUT1 and DUT2 VAT History"
197 "show-run": "Show Run"
198 "status": "PASS" | "FAIL"
206 .. note:: ID is the lowercase full path to the test.
209 REGEX_PLR_RATE = re.compile(
210 r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
211 r'PLRsearch upper bound::?\s(\d+.\d+)'
213 REGEX_NDRPDR_RATE = re.compile(
214 r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
215 r'NDR_UPPER:\s(\d+.\d+).*\n'
216 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
217 r'PDR_UPPER:\s(\d+.\d+)'
219 REGEX_NDRPDR_GBPS = re.compile(
220 r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
221 r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
222 r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
223 r'PDR_UPPER:.*,\s(\d+.\d+)'
225 REGEX_PERF_MSG_INFO = re.compile(
226 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
227 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
228 r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
229 r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
230 r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
232 REGEX_CPS_MSG_INFO = re.compile(
233 r'NDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*\n'
234 r'PDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*'
236 REGEX_PPS_MSG_INFO = re.compile(
237 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
238 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*'
240 REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
242 # Needed for CPS and PPS tests
243 REGEX_NDRPDR_LAT_BASE = re.compile(
244 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
245 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
247 REGEX_NDRPDR_LAT = re.compile(
248 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
249 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
250 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
251 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
252 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
253 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
256 REGEX_VERSION_VPP = re.compile(
257 r"(return STDOUT Version:\s*|"
258 r"VPP Version:\s*|VPP version:\s*)(.*)"
260 REGEX_VERSION_DPDK = re.compile(
261 r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
263 REGEX_TCP = re.compile(
264 r'Total\s(rps|cps|throughput):\s(\d*).*$'
266 REGEX_MRR = re.compile(
267 r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
268 r'tx\s(\d*),\srx\s(\d*)'
270 REGEX_BMRR = re.compile(
271 r'.*trial results.*: \[(.*)\]'
273 REGEX_RECONF_LOSS = re.compile(
274 r'Packets lost due to reconfig: (\d*)'
276 REGEX_RECONF_TIME = re.compile(
277 r'Implied time lost: (\d*.[\de-]*)'
279 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
281 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
283 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
285 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
287 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
289 def __init__(self, metadata, mapping, ignore):
292 :param metadata: Key-value pairs to be included in "metadata" part of
294 :param mapping: Mapping of the old names of test cases to the new
296 :param ignore: List of TCs to be ignored.
302 # Type of message to parse out from the test messages
303 self._msg_type = None
309 self._timestamp = None
311 # Testbed. The testbed is identified by TG node IP address.
314 # Mapping of TCs long names
315 self._mapping = mapping
318 self._ignore = ignore
320 # Number of PAPI History messages found:
322 # 1 - PAPI History of DUT1
323 # 2 - PAPI History of DUT2
324 self._conf_history_lookup_nr = 0
326 self._sh_run_counter = 0
328 # Test ID of currently processed test- the lowercase full path to the
332 # The main data structure
334 u"metadata": OrderedDict(),
335 u"suites": OrderedDict(),
336 u"tests": OrderedDict()
339 # Save the provided metadata
340 for key, val in metadata.items():
341 self._data[u"metadata"][key] = val
343 # Dictionary defining the methods used to parse different types of
346 u"timestamp": self._get_timestamp,
347 u"vpp-version": self._get_vpp_version,
348 u"dpdk-version": self._get_dpdk_version,
349 # TODO: Remove when not needed:
350 u"teardown-vat-history": self._get_vat_history,
351 u"teardown-papi-history": self._get_papi_history,
352 u"test-show-runtime": self._get_show_run,
353 u"testbed": self._get_testbed
358 """Getter - Data parsed from the XML file.
360 :returns: Data parsed from the XML file.
365 def _get_data_from_mrr_test_msg(self, msg):
366 """Get info from message of MRR performance tests.
368 :param msg: Message to be processed.
370 :returns: Processed message or original message if a problem occurs.
374 groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
375 if not groups or groups.lastindex != 1:
376 return u"Test Failed."
379 data = groups.group(1).split(u", ")
380 except (AttributeError, IndexError, ValueError, KeyError):
381 return u"Test Failed."
386 out_str += f"{(float(item) / 1e6):.2f}, "
387 return out_str[:-2] + u"]"
388 except (AttributeError, IndexError, ValueError, KeyError):
389 return u"Test Failed."
391 def _get_data_from_cps_test_msg(self, msg):
392 """Get info from message of NDRPDR CPS tests.
394 :param msg: Message to be processed.
396 :returns: Processed message or "Test Failed." if a problem occurs.
400 groups = re.search(self.REGEX_CPS_MSG_INFO, msg)
401 if not groups or groups.lastindex != 2:
402 return u"Test Failed."
406 f"1. {(float(groups.group(1)) / 1e6):5.2f}\n"
407 f"2. {(float(groups.group(2)) / 1e6):5.2f}"
409 except (AttributeError, IndexError, ValueError, KeyError):
410 return u"Test Failed."
412 def _get_data_from_pps_test_msg(self, msg):
413 """Get info from message of NDRPDR PPS tests.
415 :param msg: Message to be processed.
417 :returns: Processed message or "Test Failed." if a problem occurs.
421 groups = re.search(self.REGEX_PPS_MSG_INFO, msg)
422 if not groups or groups.lastindex != 4:
423 return u"Test Failed."
427 f"1. {(float(groups.group(1)) / 1e6):5.2f} "
428 f"{float(groups.group(2)):5.2f}\n"
429 f"2. {(float(groups.group(3)) / 1e6):5.2f} "
430 f"{float(groups.group(4)):5.2f}"
432 except (AttributeError, IndexError, ValueError, KeyError):
433 return u"Test Failed."
435 def _get_data_from_perf_test_msg(self, msg):
436 """Get info from message of NDRPDR performance tests.
438 :param msg: Message to be processed.
440 :returns: Processed message or "Test Failed." if a problem occurs.
444 groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
445 if not groups or groups.lastindex != 10:
446 return u"Test Failed."
450 u"ndr_low": float(groups.group(1)),
451 u"ndr_low_b": float(groups.group(2)),
452 u"pdr_low": float(groups.group(3)),
453 u"pdr_low_b": float(groups.group(4)),
454 u"pdr_lat_90_1": groups.group(5),
455 u"pdr_lat_90_2": groups.group(6),
456 u"pdr_lat_50_1": groups.group(7),
457 u"pdr_lat_50_2": groups.group(8),
458 u"pdr_lat_10_1": groups.group(9),
459 u"pdr_lat_10_2": groups.group(10),
461 except (AttributeError, IndexError, ValueError, KeyError):
462 return u"Test Failed."
464 def _process_lat(in_str_1, in_str_2):
465 """Extract min, avg, max values from latency string.
467 :param in_str_1: Latency string for one direction produced by robot
469 :param in_str_2: Latency string for second direction produced by
473 :returns: Processed latency string or None if a problem occurs.
476 in_list_1 = in_str_1.split('/', 3)
477 in_list_2 = in_str_2.split('/', 3)
479 if len(in_list_1) != 4 and len(in_list_2) != 4:
482 in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
484 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
485 except hdrh.codec.HdrLengthException:
488 in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
490 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
491 except hdrh.codec.HdrLengthException:
494 if hdr_lat_1 and hdr_lat_2:
496 hdr_lat_1.get_value_at_percentile(50.0),
497 hdr_lat_1.get_value_at_percentile(90.0),
498 hdr_lat_1.get_value_at_percentile(99.0),
499 hdr_lat_2.get_value_at_percentile(50.0),
500 hdr_lat_2.get_value_at_percentile(90.0),
501 hdr_lat_2.get_value_at_percentile(99.0)
511 f"1. {(data[u'ndr_low'] / 1e6):5.2f} "
512 f"{data[u'ndr_low_b']:5.2f}"
513 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f} "
514 f"{data[u'pdr_low_b']:5.2f}"
517 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
518 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
519 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
522 max_len = len(str(max((max(item) for item in latency))))
523 max_len = 4 if max_len < 4 else max_len
525 for idx, lat in enumerate(latency):
530 f"{lat[0]:{max_len}d} "
531 f"{lat[1]:{max_len}d} "
532 f"{lat[2]:{max_len}d} "
533 f"{lat[3]:{max_len}d} "
534 f"{lat[4]:{max_len}d} "
535 f"{lat[5]:{max_len}d} "
540 except (AttributeError, IndexError, ValueError, KeyError):
541 return u"Test Failed."
543 def _get_testbed(self, msg):
544 """Called when extraction of testbed IP is required.
545 The testbed is identified by TG node IP address.
547 :param msg: Message to process.
552 if msg.message.count(u"Setup of TG node") or \
553 msg.message.count(u"Setup of node TG host"):
554 reg_tg_ip = re.compile(
555 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
557 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
558 except (KeyError, ValueError, IndexError, AttributeError):
561 self._data[u"metadata"][u"testbed"] = self._testbed
562 self._msg_type = None
564 def _get_vpp_version(self, msg):
565 """Called when extraction of VPP version is required.
567 :param msg: Message to process.
572 if msg.message.count(u"return STDOUT Version:") or \
573 msg.message.count(u"VPP Version:") or \
574 msg.message.count(u"VPP version:"):
575 self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
577 self._data[u"metadata"][u"version"] = self._version
578 self._msg_type = None
580 def _get_dpdk_version(self, msg):
581 """Called when extraction of DPDK version is required.
583 :param msg: Message to process.
588 if msg.message.count(u"DPDK Version:"):
590 self._version = str(re.search(
591 self.REGEX_VERSION_DPDK, msg.message).group(2))
592 self._data[u"metadata"][u"version"] = self._version
596 self._msg_type = None
598 def _get_timestamp(self, msg):
599 """Called when extraction of timestamp is required.
601 :param msg: Message to process.
606 self._timestamp = msg.timestamp[:14]
607 self._data[u"metadata"][u"generated"] = self._timestamp
608 self._msg_type = None
610 def _get_vat_history(self, msg):
611 """Called when extraction of VAT command history is required.
613 TODO: Remove when not needed.
615 :param msg: Message to process.
619 if msg.message.count(u"VAT command history:"):
620 self._conf_history_lookup_nr += 1
621 if self._conf_history_lookup_nr == 1:
622 self._data[u"tests"][self._test_id][u"conf-history"] = str()
624 self._msg_type = None
625 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
626 r"VAT command history:", u"",
627 msg.message, count=1).replace(u'\n', u' |br| ').\
630 self._data[u"tests"][self._test_id][u"conf-history"] += (
631 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
634 def _get_papi_history(self, msg):
635 """Called when extraction of PAPI command history is required.
637 :param msg: Message to process.
641 if msg.message.count(u"PAPI command history:"):
642 self._conf_history_lookup_nr += 1
643 if self._conf_history_lookup_nr == 1:
644 self._data[u"tests"][self._test_id][u"conf-history"] = str()
646 self._msg_type = None
647 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
648 r"PAPI command history:", u"",
649 msg.message, count=1).replace(u'\n', u' |br| ').\
651 self._data[u"tests"][self._test_id][u"conf-history"] += (
652 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
655 def _get_show_run(self, msg):
656 """Called when extraction of VPP operational data (output of CLI command
657 Show Runtime) is required.
659 :param msg: Message to process.
664 if not msg.message.count(u"stats runtime"):
668 if self._sh_run_counter > 1:
671 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
672 self._data[u"tests"][self._test_id][u"show-run"] = dict()
674 groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
678 host = groups.group(1)
679 except (AttributeError, IndexError):
682 sock = groups.group(2)
683 except (AttributeError, IndexError):
686 runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
687 replace(u"'", u'"').replace(u'b"', u'"').
688 replace(u'u"', u'"').split(u":", 1)[1])
691 threads_nr = len(runtime[0][u"clocks"])
692 except (IndexError, KeyError):
695 dut = u"DUT{nr}".format(
696 nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
701 u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
705 for idx in range(threads_nr):
706 if item[u"vectors"][idx] > 0:
707 clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
708 elif item[u"calls"][idx] > 0:
709 clocks = item[u"clocks"][idx] / item[u"calls"][idx]
710 elif item[u"suspends"][idx] > 0:
711 clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
715 if item[u"calls"][idx] > 0:
716 vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
720 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
721 int(item[u"suspends"][idx]):
722 oper[u"threads"][idx].append([
725 item[u"vectors"][idx],
726 item[u"suspends"][idx],
731 self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
733 def _get_ndrpdr_throughput(self, msg):
734 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
737 :param msg: The test message to be parsed.
739 :returns: Parsed data as a dict and the status (PASS/FAIL).
740 :rtype: tuple(dict, str)
744 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
745 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
748 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
750 if groups is not None:
752 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
753 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
754 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
755 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
757 except (IndexError, ValueError):
760 return throughput, status
762 def _get_ndrpdr_throughput_gbps(self, msg):
763 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
766 :param msg: The test message to be parsed.
768 :returns: Parsed data as a dict and the status (PASS/FAIL).
769 :rtype: tuple(dict, str)
773 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
774 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
777 groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
779 if groups is not None:
781 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
782 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
783 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
784 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
786 except (IndexError, ValueError):
791 def _get_plr_throughput(self, msg):
792 """Get PLRsearch lower bound and PLRsearch upper bound from the test
795 :param msg: The test message to be parsed.
797 :returns: Parsed data as a dict and the status (PASS/FAIL).
798 :rtype: tuple(dict, str)
806 groups = re.search(self.REGEX_PLR_RATE, msg)
808 if groups is not None:
810 throughput[u"LOWER"] = float(groups.group(1))
811 throughput[u"UPPER"] = float(groups.group(2))
813 except (IndexError, ValueError):
816 return throughput, status
818 def _get_ndrpdr_latency(self, msg):
819 """Get LATENCY from the test message.
821 :param msg: The test message to be parsed.
823 :returns: Parsed data as a dict and the status (PASS/FAIL).
824 :rtype: tuple(dict, str)
834 u"direction1": copy.copy(latency_default),
835 u"direction2": copy.copy(latency_default)
838 u"direction1": copy.copy(latency_default),
839 u"direction2": copy.copy(latency_default)
842 u"direction1": copy.copy(latency_default),
843 u"direction2": copy.copy(latency_default)
846 u"direction1": copy.copy(latency_default),
847 u"direction2": copy.copy(latency_default)
850 u"direction1": copy.copy(latency_default),
851 u"direction2": copy.copy(latency_default)
854 u"direction1": copy.copy(latency_default),
855 u"direction2": copy.copy(latency_default)
859 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
861 groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
863 return latency, u"FAIL"
865 def process_latency(in_str):
866 """Return object with parsed latency values.
868 TODO: Define class for the return type.
870 :param in_str: Input string, min/avg/max/hdrh format.
872 :returns: Dict with corresponding keys, except hdrh float values.
874 :throws IndexError: If in_str does not have enough substrings.
875 :throws ValueError: If a substring does not convert to float.
877 in_list = in_str.split('/', 3)
880 u"min": float(in_list[0]),
881 u"avg": float(in_list[1]),
882 u"max": float(in_list[2]),
886 if len(in_list) == 4:
887 rval[u"hdrh"] = str(in_list[3])
892 latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
893 latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
894 latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
895 latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
896 if groups.lastindex == 4:
897 return latency, u"PASS"
898 except (IndexError, ValueError):
902 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
903 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
904 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
905 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
906 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
907 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
908 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
909 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
910 if groups.lastindex == 12:
911 return latency, u"PASS"
912 except (IndexError, ValueError):
915 # TODO: Remove when not needed
916 latency[u"NDR10"] = {
917 u"direction1": copy.copy(latency_default),
918 u"direction2": copy.copy(latency_default)
920 latency[u"NDR50"] = {
921 u"direction1": copy.copy(latency_default),
922 u"direction2": copy.copy(latency_default)
924 latency[u"NDR90"] = {
925 u"direction1": copy.copy(latency_default),
926 u"direction2": copy.copy(latency_default)
929 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
930 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
931 latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
932 latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
933 latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
934 latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
935 latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
936 latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
937 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
938 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
939 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
940 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
941 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
942 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
943 return latency, u"PASS"
944 except (IndexError, ValueError):
947 return latency, u"FAIL"
950 def _get_hoststack_data(msg, tags):
951 """Get data from the hoststack test message.
953 :param msg: The test message to be parsed.
954 :param tags: Test tags.
957 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
958 :rtype: tuple(dict, str)
963 msg = msg.replace(u"'", u'"').replace(u" ", u"")
964 if u"LDPRELOAD" in tags:
968 except JSONDecodeError:
970 elif u"VPPECHO" in tags:
972 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
974 client=loads(msg_lst[0]),
975 server=loads(msg_lst[1])
978 except (JSONDecodeError, IndexError):
981 return result, status
983 def visit_suite(self, suite):
984 """Implements traversing through the suite and its direct children.
986 :param suite: Suite to process.
990 if self.start_suite(suite) is not False:
991 suite.suites.visit(self)
992 suite.tests.visit(self)
993 self.end_suite(suite)
995 def start_suite(self, suite):
996 """Called when suite starts.
998 :param suite: Suite to process.
1004 parent_name = suite.parent.name
1005 except AttributeError:
1008 doc_str = suite.doc.\
1009 replace(u'"', u"'").\
1010 replace(u'\n', u' ').\
1011 replace(u'\r', u'').\
1012 replace(u'*[', u' |br| *[').\
1013 replace(u"*", u"**").\
1014 replace(u' |br| *[', u'*[', 1)
1016 self._data[u"suites"][suite.longname.lower().
1017 replace(u'"', u"'").
1018 replace(u" ", u"_")] = {
1019 u"name": suite.name.lower(),
1021 u"parent": parent_name,
1022 u"level": len(suite.longname.split(u"."))
1025 suite.keywords.visit(self)
1027 def end_suite(self, suite):
1028 """Called when suite ends.
1030 :param suite: Suite to process.
1035 def visit_test(self, test):
1036 """Implements traversing through the test.
1038 :param test: Test to process.
1042 if self.start_test(test) is not False:
1043 test.keywords.visit(self)
1046 def start_test(self, test):
1047 """Called when test starts.
1049 :param test: Test to process.
1054 self._sh_run_counter = 0
1056 longname_orig = test.longname.lower()
1058 # Check the ignore list
1059 if longname_orig in self._ignore:
1062 tags = [str(tag) for tag in test.tags]
1063 test_result = dict()
1065 # Change the TC long name and name if defined in the mapping table
1066 longname = self._mapping.get(longname_orig, None)
1067 if longname is not None:
1068 name = longname.split(u'.')[-1]
1070 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1074 longname = longname_orig
1075 name = test.name.lower()
1077 # Remove TC number from the TC long name (backward compatibility):
1078 self._test_id = re.sub(
1079 self.REGEX_TC_NUMBER, u"", longname.replace(u"snat", u"nat")
1081 # Remove TC number from the TC name (not needed):
1082 test_result[u"name"] = re.sub(
1083 self.REGEX_TC_NUMBER, "", name.replace(u"snat", u"nat")
1086 test_result[u"parent"] = test.parent.name.lower().\
1087 replace(u"snat", u"nat")
1088 test_result[u"tags"] = tags
1089 test_result["doc"] = test.doc.\
1090 replace(u'"', u"'").\
1091 replace(u'\n', u' ').\
1092 replace(u'\r', u'').\
1093 replace(u'[', u' |br| [').\
1094 replace(u' |br| [', u'[', 1)
1095 test_result[u"type"] = u"FUNC"
1096 test_result[u"status"] = test.status
1098 if test.status == u"PASS":
1099 if u"NDRPDR" in tags:
1100 if u"TCP_PPS" in tags or u"UDP_PPS" in tags:
1101 test_result[u"msg"] = self._get_data_from_pps_test_msg(
1102 test.message).replace(u'\n', u' |br| '). \
1103 replace(u'\r', u'').replace(u'"', u"'")
1104 elif u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1105 test_result[u"msg"] = self._get_data_from_cps_test_msg(
1106 test.message).replace(u'\n', u' |br| '). \
1107 replace(u'\r', u'').replace(u'"', u"'")
1109 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1110 test.message).replace(u'\n', u' |br| ').\
1111 replace(u'\r', u'').replace(u'"', u"'")
1112 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1113 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1114 test.message).replace(u'\n', u' |br| ').\
1115 replace(u'\r', u'').replace(u'"', u"'")
1117 test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\
1118 replace(u'\r', u'').replace(u'"', u"'")
1120 test_result[u"msg"] = u"Test Failed."
1122 if u"PERFTEST" in tags:
1123 # Replace info about cores (e.g. -1c-) with the info about threads
1124 # and cores (e.g. -1t1c-) in the long test case names and in the
1125 # test case names if necessary.
1126 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1130 for tag in test_result[u"tags"]:
1131 groups = re.search(self.REGEX_TC_TAG, tag)
1137 self._test_id = re.sub(
1138 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1139 self._test_id, count=1
1141 test_result[u"name"] = re.sub(
1142 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1143 test_result["name"], count=1
1146 test_result[u"status"] = u"FAIL"
1147 self._data[u"tests"][self._test_id] = test_result
1149 f"The test {self._test_id} has no or more than one "
1150 f"multi-threading tags.\n"
1151 f"Tags: {test_result[u'tags']}"
1155 if test.status == u"PASS":
1156 if u"DEVICETEST" in tags:
1157 test_result[u"type"] = u"DEVICETEST"
1158 elif u"NDRPDR" in tags:
1159 if u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1160 test_result[u"type"] = u"CPS"
1162 test_result[u"type"] = u"NDRPDR"
1163 test_result[u"throughput"], test_result[u"status"] = \
1164 self._get_ndrpdr_throughput(test.message)
1165 test_result[u"gbps"], test_result[u"status"] = \
1166 self._get_ndrpdr_throughput_gbps(test.message)
1167 test_result[u"latency"], test_result[u"status"] = \
1168 self._get_ndrpdr_latency(test.message)
1169 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1171 test_result[u"type"] = u"MRR"
1173 test_result[u"type"] = u"BMRR"
1175 test_result[u"result"] = dict()
1176 groups = re.search(self.REGEX_BMRR, test.message)
1177 if groups is not None:
1178 items_str = groups.group(1)
1180 float(item.strip().replace(u"'", u""))
1181 for item in items_str.split(",")
1183 # Use whole list in CSIT-1180.
1184 stats = jumpavg.AvgStdevStats.for_runs(items_float)
1185 test_result[u"result"][u"samples"] = items_float
1186 test_result[u"result"][u"receive-rate"] = stats.avg
1187 test_result[u"result"][u"receive-stdev"] = stats.stdev
1189 groups = re.search(self.REGEX_MRR, test.message)
1190 test_result[u"result"][u"receive-rate"] = \
1191 float(groups.group(3)) / float(groups.group(1))
1192 elif u"SOAK" in tags:
1193 test_result[u"type"] = u"SOAK"
1194 test_result[u"throughput"], test_result[u"status"] = \
1195 self._get_plr_throughput(test.message)
1196 elif u"HOSTSTACK" in tags:
1197 test_result[u"type"] = u"HOSTSTACK"
1198 test_result[u"result"], test_result[u"status"] = \
1199 self._get_hoststack_data(test.message, tags)
1200 elif u"TCP" in tags:
1201 test_result[u"type"] = u"TCP"
1202 groups = re.search(self.REGEX_TCP, test.message)
1203 test_result[u"result"] = int(groups.group(2))
1204 elif u"RECONF" in tags:
1205 test_result[u"type"] = u"RECONF"
1206 test_result[u"result"] = None
1208 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1209 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1210 test_result[u"result"] = {
1211 u"loss": int(grps_loss.group(1)),
1212 u"time": float(grps_time.group(1))
1214 except (AttributeError, IndexError, ValueError, TypeError):
1215 test_result[u"status"] = u"FAIL"
1217 test_result[u"status"] = u"FAIL"
1218 self._data[u"tests"][self._test_id] = test_result
1221 self._data[u"tests"][self._test_id] = test_result
1223 def end_test(self, test):
1224 """Called when test ends.
1226 :param test: Test to process.
1231 def visit_keyword(self, keyword):
1232 """Implements traversing through the keyword and its child keywords.
1234 :param keyword: Keyword to process.
1235 :type keyword: Keyword
1238 if self.start_keyword(keyword) is not False:
1239 self.end_keyword(keyword)
1241 def start_keyword(self, keyword):
1242 """Called when keyword starts. Default implementation does nothing.
1244 :param keyword: Keyword to process.
1245 :type keyword: Keyword
1249 if keyword.type == u"setup":
1250 self.visit_setup_kw(keyword)
1251 elif keyword.type == u"teardown":
1252 self.visit_teardown_kw(keyword)
1254 self.visit_test_kw(keyword)
1255 except AttributeError:
1258 def end_keyword(self, keyword):
1259 """Called when keyword ends. Default implementation does nothing.
1261 :param keyword: Keyword to process.
1262 :type keyword: Keyword
1266 def visit_test_kw(self, test_kw):
1267 """Implements traversing through the test keyword and its child
1270 :param test_kw: Keyword to process.
1271 :type test_kw: Keyword
1274 for keyword in test_kw.keywords:
1275 if self.start_test_kw(keyword) is not False:
1276 self.visit_test_kw(keyword)
1277 self.end_test_kw(keyword)
1279 def start_test_kw(self, test_kw):
1280 """Called when test keyword starts. Default implementation does
1283 :param test_kw: Keyword to process.
1284 :type test_kw: Keyword
1287 if test_kw.name.count(u"Show Runtime On All Duts") or \
1288 test_kw.name.count(u"Show Runtime Counters On All Duts") or \
1289 test_kw.name.count(u"Vpp Show Runtime On All Duts"):
1290 self._msg_type = u"test-show-runtime"
1291 self._sh_run_counter += 1
1294 test_kw.messages.visit(self)
1296 def end_test_kw(self, test_kw):
1297 """Called when keyword ends. Default implementation does nothing.
1299 :param test_kw: Keyword to process.
1300 :type test_kw: Keyword
1304 def visit_setup_kw(self, setup_kw):
1305 """Implements traversing through the teardown keyword and its child
1308 :param setup_kw: Keyword to process.
1309 :type setup_kw: Keyword
1312 for keyword in setup_kw.keywords:
1313 if self.start_setup_kw(keyword) is not False:
1314 self.visit_setup_kw(keyword)
1315 self.end_setup_kw(keyword)
1317 def start_setup_kw(self, setup_kw):
1318 """Called when teardown keyword starts. Default implementation does
1321 :param setup_kw: Keyword to process.
1322 :type setup_kw: Keyword
1325 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1326 and not self._version:
1327 self._msg_type = u"vpp-version"
1328 elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1330 self._msg_type = u"dpdk-version"
1331 elif setup_kw.name.count(u"Set Global Variable") \
1332 and not self._timestamp:
1333 self._msg_type = u"timestamp"
1334 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1335 self._msg_type = u"testbed"
1338 setup_kw.messages.visit(self)
1340 def end_setup_kw(self, setup_kw):
1341 """Called when keyword ends. Default implementation does nothing.
1343 :param setup_kw: Keyword to process.
1344 :type setup_kw: Keyword
1348 def visit_teardown_kw(self, teardown_kw):
1349 """Implements traversing through the teardown keyword and its child
1352 :param teardown_kw: Keyword to process.
1353 :type teardown_kw: Keyword
1356 for keyword in teardown_kw.keywords:
1357 if self.start_teardown_kw(keyword) is not False:
1358 self.visit_teardown_kw(keyword)
1359 self.end_teardown_kw(keyword)
1361 def start_teardown_kw(self, teardown_kw):
1362 """Called when teardown keyword starts
1364 :param teardown_kw: Keyword to process.
1365 :type teardown_kw: Keyword
1369 if teardown_kw.name.count(u"Show Vat History On All Duts"):
1370 # TODO: Remove when not needed:
1371 self._conf_history_lookup_nr = 0
1372 self._msg_type = u"teardown-vat-history"
1373 teardown_kw.messages.visit(self)
1374 elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1375 self._conf_history_lookup_nr = 0
1376 self._msg_type = u"teardown-papi-history"
1377 teardown_kw.messages.visit(self)
1379 def end_teardown_kw(self, teardown_kw):
1380 """Called when keyword ends. Default implementation does nothing.
1382 :param teardown_kw: Keyword to process.
1383 :type teardown_kw: Keyword
1387 def visit_message(self, msg):
1388 """Implements visiting the message.
1390 :param msg: Message to process.
1394 if self.start_message(msg) is not False:
1395 self.end_message(msg)
1397 def start_message(self, msg):
1398 """Called when message starts. Get required information from messages:
1401 :param msg: Message to process.
1406 self.parse_msg[self._msg_type](msg)
1408 def end_message(self, msg):
1409 """Called when message ends. Default implementation does nothing.
1411 :param msg: Message to process.
1420 The data is extracted from output.xml files generated by Jenkins jobs and
1421 stored in pandas' DataFrames.
1427 (as described in ExecutionChecker documentation)
1429 (as described in ExecutionChecker documentation)
1431 (as described in ExecutionChecker documentation)
1434 def __init__(self, spec):
1437 :param spec: Specification.
1438 :type spec: Specification
1445 self._input_data = pd.Series()
1449 """Getter - Input data.
1451 :returns: Input data
1452 :rtype: pandas.Series
1454 return self._input_data
1456 def metadata(self, job, build):
1457 """Getter - metadata
1459 :param job: Job which metadata we want.
1460 :param build: Build which metadata we want.
1464 :rtype: pandas.Series
1466 return self.data[job][build][u"metadata"]
1468 def suites(self, job, build):
1471 :param job: Job which suites we want.
1472 :param build: Build which suites we want.
1476 :rtype: pandas.Series
1478 return self.data[job][str(build)][u"suites"]
1480 def tests(self, job, build):
1483 :param job: Job which tests we want.
1484 :param build: Build which tests we want.
1488 :rtype: pandas.Series
1490 return self.data[job][build][u"tests"]
1492 def _parse_tests(self, job, build):
1493 """Process data from robot output.xml file and return JSON structured
1496 :param job: The name of job which build output data will be processed.
1497 :param build: The build which output data will be processed.
1500 :returns: JSON data structure.
1509 with open(build[u"file-name"], u'r') as data_file:
1511 result = ExecutionResult(data_file)
1512 except errors.DataError as err:
1514 f"Error occurred while parsing output.xml: {repr(err)}"
1517 checker = ExecutionChecker(metadata, self._cfg.mapping,
1519 result.visit(checker)
1523 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1524 """Download and parse the input data file.
1526 :param pid: PID of the process executing this method.
1527 :param job: Name of the Jenkins job which generated the processed input
1529 :param build: Information about the Jenkins build which generated the
1530 processed input file.
1531 :param repeat: Repeat the download specified number of times if not
1539 logging.info(f" Processing the job/build: {job}: {build[u'build']}")
1546 success = download_and_unzip_data_file(self._cfg, job, build, pid)
1552 f"It is not possible to download the input data file from the "
1553 f"job {job}, build {build[u'build']}, or it is damaged. "
1557 logging.info(f" Processing data from build {build[u'build']}")
1558 data = self._parse_tests(job, build)
1561 f"Input data file from the job {job}, build "
1562 f"{build[u'build']} is damaged. Skipped."
1565 state = u"processed"
1568 remove(build[u"file-name"])
1569 except OSError as err:
1571 f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1574 # If the time-period is defined in the specification file, remove all
1575 # files which are outside the time period.
1577 timeperiod = self._cfg.input.get(u"time-period", None)
1578 if timeperiod and data:
1580 timeperiod = timedelta(int(timeperiod))
1581 metadata = data.get(u"metadata", None)
1583 generated = metadata.get(u"generated", None)
1585 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1586 if (now - generated) > timeperiod:
1587 # Remove the data and the file:
1592 f" The build {job}/{build[u'build']} is "
1593 f"outdated, will be removed."
1595 logging.info(u" Done.")
1605 def download_and_parse_data(self, repeat=1):
1606 """Download the input data files, parse input data from input files and
1607 store in pandas' Series.
1609 :param repeat: Repeat the download specified number of times if not
1614 logging.info(u"Downloading and parsing input files ...")
1616 for job, builds in self._cfg.builds.items():
1617 for build in builds:
1619 result = self._download_and_parse_build(job, build, repeat)
1622 build_nr = result[u"build"][u"build"]
1625 data = result[u"data"]
1626 build_data = pd.Series({
1627 u"metadata": pd.Series(
1628 list(data[u"metadata"].values()),
1629 index=list(data[u"metadata"].keys())
1631 u"suites": pd.Series(
1632 list(data[u"suites"].values()),
1633 index=list(data[u"suites"].keys())
1635 u"tests": pd.Series(
1636 list(data[u"tests"].values()),
1637 index=list(data[u"tests"].keys())
1641 if self._input_data.get(job, None) is None:
1642 self._input_data[job] = pd.Series()
1643 self._input_data[job][str(build_nr)] = build_data
1645 self._cfg.set_input_file_name(
1646 job, build_nr, result[u"build"][u"file-name"])
1648 self._cfg.set_input_state(job, build_nr, result[u"state"])
1651 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1652 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1654 logging.info(u"Done.")
1656 def process_local_file(self, local_file, job=u"local", build_nr=1,
1658 """Process local XML file given as a command-line parameter.
1660 :param local_file: The file to process.
1661 :param job: Job name.
1662 :param build_nr: Build number.
1663 :param replace: If True, the information about jobs and builds is
1664 replaced by the new one, otherwise the new jobs and builds are
1666 :type local_file: str
1670 :raises: PresentationError if an error occurs.
1672 if not isfile(local_file):
1673 raise PresentationError(f"The file {local_file} does not exist.")
1676 build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1677 except (IndexError, ValueError):
1682 u"status": u"failed",
1683 u"file-name": local_file
1686 self._cfg.builds = dict()
1687 self._cfg.add_build(job, build)
1689 logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1690 data = self._parse_tests(job, build)
1692 raise PresentationError(
1693 f"Error occurred while parsing the file {local_file}"
1696 build_data = pd.Series({
1697 u"metadata": pd.Series(
1698 list(data[u"metadata"].values()),
1699 index=list(data[u"metadata"].keys())
1701 u"suites": pd.Series(
1702 list(data[u"suites"].values()),
1703 index=list(data[u"suites"].keys())
1705 u"tests": pd.Series(
1706 list(data[u"tests"].values()),
1707 index=list(data[u"tests"].keys())
1711 if self._input_data.get(job, None) is None:
1712 self._input_data[job] = pd.Series()
1713 self._input_data[job][str(build_nr)] = build_data
1715 self._cfg.set_input_state(job, build_nr, u"processed")
1717 def process_local_directory(self, local_dir, replace=True):
1718 """Process local directory with XML file(s). The directory is processed
1719 as a 'job' and the XML files in it as builds.
1720 If the given directory contains only sub-directories, these
1721 sub-directories processed as jobs and corresponding XML files as builds
1724 :param local_dir: Local directory to process.
1725 :param replace: If True, the information about jobs and builds is
1726 replaced by the new one, otherwise the new jobs and builds are
1728 :type local_dir: str
1731 if not isdir(local_dir):
1732 raise PresentationError(
1733 f"The directory {local_dir} does not exist."
1736 # Check if the given directory includes only files, or only directories
1737 _, dirnames, filenames = next(walk(local_dir))
1739 if filenames and not dirnames:
1742 # key: dir (job) name, value: list of file names (builds)
1744 local_dir: [join(local_dir, name) for name in filenames]
1747 elif dirnames and not filenames:
1750 # key: dir (job) name, value: list of file names (builds)
1751 local_builds = dict()
1752 for dirname in dirnames:
1754 join(local_dir, dirname, name)
1755 for name in listdir(join(local_dir, dirname))
1756 if isfile(join(local_dir, dirname, name))
1759 local_builds[dirname] = sorted(builds)
1761 elif not filenames and not dirnames:
1762 raise PresentationError(f"The directory {local_dir} is empty.")
1764 raise PresentationError(
1765 f"The directory {local_dir} can include only files or only "
1766 f"directories, not both.\nThe directory {local_dir} includes "
1767 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1771 self._cfg.builds = dict()
1773 for job, files in local_builds.items():
1774 for idx, local_file in enumerate(files):
1775 self.process_local_file(local_file, job, idx + 1, replace=False)
1778 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1779 """Return the index of character in the string which is the end of tag.
1781 :param tag_filter: The string where the end of tag is being searched.
1782 :param start: The index where the searching is stated.
1783 :param closer: The character which is the tag closer.
1784 :type tag_filter: str
1787 :returns: The index of the tag closer.
1791 idx_opener = tag_filter.index(closer, start)
1792 return tag_filter.index(closer, idx_opener + 1)
1797 def _condition(tag_filter):
1798 """Create a conditional statement from the given tag filter.
1800 :param tag_filter: Filter based on tags from the element specification.
1801 :type tag_filter: str
1802 :returns: Conditional statement which can be evaluated.
1807 index = InputData._end_of_tag(tag_filter, index)
1811 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1813 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1814 continue_on_error=False):
1815 """Filter required data from the given jobs and builds.
1817 The output data structure is:
1820 - test (or suite) 1 ID:
1826 - test (or suite) n ID:
1833 :param element: Element which will use the filtered data.
1834 :param params: Parameters which will be included in the output. If None,
1835 all parameters are included.
1836 :param data: If not None, this data is used instead of data specified
1838 :param data_set: The set of data to be filtered: tests, suites,
1840 :param continue_on_error: Continue if there is error while reading the
1841 data. The Item will be empty then
1842 :type element: pandas.Series
1846 :type continue_on_error: bool
1847 :returns: Filtered data.
1848 :rtype pandas.Series
1852 if data_set == "suites":
1854 elif element[u"filter"] in (u"all", u"template"):
1857 cond = InputData._condition(element[u"filter"])
1858 logging.debug(f" Filter: {cond}")
1860 logging.error(u" No filter defined.")
1864 params = element.get(u"parameters", None)
1866 params.append(u"type")
1868 data_to_filter = data if data else element[u"data"]
1871 for job, builds in data_to_filter.items():
1872 data[job] = pd.Series()
1873 for build in builds:
1874 data[job][str(build)] = pd.Series()
1877 self.data[job][str(build)][data_set].items())
1879 if continue_on_error:
1883 for test_id, test_data in data_dict.items():
1884 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1885 data[job][str(build)][test_id] = pd.Series()
1887 for param, val in test_data.items():
1888 data[job][str(build)][test_id][param] = val
1890 for param in params:
1892 data[job][str(build)][test_id][param] =\
1895 data[job][str(build)][test_id][param] =\
1899 except (KeyError, IndexError, ValueError) as err:
1901 f"Missing mandatory parameter in the element specification: "
1905 except AttributeError as err:
1906 logging.error(repr(err))
1908 except SyntaxError as err:
1910 f"The filter {cond} is not correct. Check if all tags are "
1911 f"enclosed by apostrophes.\n{repr(err)}"
1915 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1916 continue_on_error=False):
1917 """Filter required data from the given jobs and builds.
1919 The output data structure is:
1922 - test (or suite) 1 ID:
1928 - test (or suite) n ID:
1935 :param element: Element which will use the filtered data.
1936 :param params: Parameters which will be included in the output. If None,
1937 all parameters are included.
1938 :param data_set: The set of data to be filtered: tests, suites,
1940 :param continue_on_error: Continue if there is error while reading the
1941 data. The Item will be empty then
1942 :type element: pandas.Series
1945 :type continue_on_error: bool
1946 :returns: Filtered data.
1947 :rtype pandas.Series
1950 include = element.get(u"include", None)
1952 logging.warning(u"No tests to include, skipping the element.")
1956 params = element.get(u"parameters", None)
1958 params.append(u"type")
1962 for job, builds in element[u"data"].items():
1963 data[job] = pd.Series()
1964 for build in builds:
1965 data[job][str(build)] = pd.Series()
1966 for test in include:
1968 reg_ex = re.compile(str(test).lower())
1969 for test_id in self.data[job][
1970 str(build)][data_set].keys():
1971 if re.match(reg_ex, str(test_id).lower()):
1972 test_data = self.data[job][
1973 str(build)][data_set][test_id]
1974 data[job][str(build)][test_id] = pd.Series()
1976 for param, val in test_data.items():
1977 data[job][str(build)][test_id]\
1980 for param in params:
1982 data[job][str(build)][
1986 data[job][str(build)][
1987 test_id][param] = u"No Data"
1988 except KeyError as err:
1989 if continue_on_error:
1990 logging.debug(repr(err))
1992 logging.error(repr(err))
1996 except (KeyError, IndexError, ValueError) as err:
1998 f"Missing mandatory parameter in the element "
1999 f"specification: {repr(err)}"
2002 except AttributeError as err:
2003 logging.error(repr(err))
2007 def merge_data(data):
2008 """Merge data from more jobs and builds to a simple data structure.
2010 The output data structure is:
2012 - test (suite) 1 ID:
2018 - test (suite) n ID:
2021 :param data: Data to merge.
2022 :type data: pandas.Series
2023 :returns: Merged data.
2024 :rtype: pandas.Series
2027 logging.info(u" Merging data ...")
2029 merged_data = pd.Series()
2030 for builds in data.values:
2031 for item in builds.values:
2032 for item_id, item_data in item.items():
2033 merged_data[item_id] = item_data
2036 def print_all_oper_data(self):
2037 """Print all operational data to console.
2045 u"Cycles per Packet",
2046 u"Average Vector Size"
2049 for job in self._input_data.values:
2050 for build in job.values:
2051 for test_id, test_data in build[u"tests"].items():
2053 if test_data.get(u"show-run", None) is None:
2055 for dut_name, data in test_data[u"show-run"].items():
2056 if data.get(u"threads", None) is None:
2058 print(f"Host IP: {data.get(u'host', '')}, "
2059 f"Socket: {data.get(u'socket', '')}")
2060 for thread_nr, thread in data[u"threads"].items():
2061 txt_table = prettytable.PrettyTable(tbl_hdr)
2064 txt_table.add_row(row)
2066 if len(thread) == 0:
2069 avg = f", Average Vector Size per Node: " \
2070 f"{(avg / len(thread)):.2f}"
2071 th_name = u"main" if thread_nr == 0 \
2072 else f"worker_{thread_nr}"
2073 print(f"{dut_name}, {th_name}{avg}")
2074 txt_table.float_format = u".2"
2075 txt_table.align = u"r"
2076 txt_table.align[u"Name"] = u"l"
2077 print(f"{txt_table.get_string()}\n")