1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
29 from datetime import datetime as dt
30 from datetime import timedelta
31 from json import loads
38 from robot.api import ExecutionResult, ResultVisitor
39 from robot import errors
41 from resources.libraries.python import jumpavg
42 from input_data_files import download_and_unzip_data_file
45 # Separator used in file names
49 class ExecutionChecker(ResultVisitor):
50 """Class to traverse through the test suite structure.
52 The functionality implemented in this class generates a json structure:
58 "generated": "Timestamp",
59 "version": "SUT version",
60 "job": "Jenkins job name",
61 "build": "Information about the build"
64 "Suite long name 1": {
66 "doc": "Suite 1 documentation",
67 "parent": "Suite 1 parent",
68 "level": "Level of the suite in the suite hierarchy"
70 "Suite long name N": {
72 "doc": "Suite N documentation",
73 "parent": "Suite 2 parent",
74 "level": "Level of the suite in the suite hierarchy"
81 "parent": "Name of the parent of the test",
82 "doc": "Test documentation",
83 "msg": "Test message",
84 "conf-history": "DUT1 and DUT2 VAT History",
85 "show-run": "Show Run",
86 "tags": ["tag 1", "tag 2", "tag n"],
88 "status": "PASS" | "FAIL",
134 "parent": "Name of the parent of the test",
135 "doc": "Test documentation",
136 "msg": "Test message",
137 "tags": ["tag 1", "tag 2", "tag n"],
139 "status": "PASS" | "FAIL",
146 "parent": "Name of the parent of the test",
147 "doc": "Test documentation",
148 "msg": "Test message",
149 "tags": ["tag 1", "tag 2", "tag n"],
150 "type": "MRR" | "BMRR",
151 "status": "PASS" | "FAIL",
153 "receive-rate": float,
154 # Average of a list, computed using AvgStdevStats.
155 # In CSIT-1180, replace with List[float].
169 "metadata": { # Optional
170 "version": "VPP version",
171 "job": "Jenkins job name",
172 "build": "Information about the build"
176 "doc": "Suite 1 documentation",
177 "parent": "Suite 1 parent",
178 "level": "Level of the suite in the suite hierarchy"
181 "doc": "Suite N documentation",
182 "parent": "Suite 2 parent",
183 "level": "Level of the suite in the suite hierarchy"
189 "parent": "Name of the parent of the test",
190 "doc": "Test documentation"
191 "msg": "Test message"
192 "tags": ["tag 1", "tag 2", "tag n"],
193 "conf-history": "DUT1 and DUT2 VAT History"
194 "show-run": "Show Run"
195 "status": "PASS" | "FAIL"
203 .. note:: ID is the lowercase full path to the test.
206 REGEX_PLR_RATE = re.compile(
207 r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
208 r'PLRsearch upper bound::?\s(\d+.\d+)'
210 REGEX_NDRPDR_RATE = re.compile(
211 r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
212 r'NDR_UPPER:\s(\d+.\d+).*\n'
213 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
214 r'PDR_UPPER:\s(\d+.\d+)'
216 REGEX_PERF_MSG_INFO = re.compile(
217 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
218 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
219 r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
220 r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
221 r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
223 REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
225 # TODO: Remove when not needed
226 REGEX_NDRPDR_LAT_BASE = re.compile(
227 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
228 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
230 REGEX_NDRPDR_LAT = re.compile(
231 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
232 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
233 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
234 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
235 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
236 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
238 # TODO: Remove when not needed
239 REGEX_NDRPDR_LAT_LONG = re.compile(
240 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
241 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
242 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
243 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
244 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
245 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
246 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
247 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
248 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
250 REGEX_VERSION_VPP = re.compile(
251 r"(return STDOUT Version:\s*|"
252 r"VPP Version:\s*|VPP version:\s*)(.*)"
254 REGEX_VERSION_DPDK = re.compile(
255 r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
257 REGEX_TCP = re.compile(
258 r'Total\s(rps|cps|throughput):\s(\d*).*$'
260 REGEX_MRR = re.compile(
261 r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
262 r'tx\s(\d*),\srx\s(\d*)'
264 REGEX_BMRR = re.compile(
265 r'Maximum Receive Rate trial results'
266 r' in packets per second: \[(.*)\]'
268 REGEX_RECONF_LOSS = re.compile(
269 r'Packets lost due to reconfig: (\d*)'
271 REGEX_RECONF_TIME = re.compile(
272 r'Implied time lost: (\d*.[\de-]*)'
274 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
276 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
278 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
280 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
282 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
284 def __init__(self, metadata, mapping, ignore):
287 :param metadata: Key-value pairs to be included in "metadata" part of
289 :param mapping: Mapping of the old names of test cases to the new
291 :param ignore: List of TCs to be ignored.
297 # Type of message to parse out from the test messages
298 self._msg_type = None
304 self._timestamp = None
306 # Testbed. The testbed is identified by TG node IP address.
309 # Mapping of TCs long names
310 self._mapping = mapping
313 self._ignore = ignore
315 # Number of PAPI History messages found:
317 # 1 - PAPI History of DUT1
318 # 2 - PAPI History of DUT2
319 self._conf_history_lookup_nr = 0
321 self._sh_run_counter = 0
323 # Test ID of currently processed test- the lowercase full path to the
327 # The main data structure
329 u"metadata": OrderedDict(),
330 u"suites": OrderedDict(),
331 u"tests": OrderedDict()
334 # Save the provided metadata
335 for key, val in metadata.items():
336 self._data[u"metadata"][key] = val
338 # Dictionary defining the methods used to parse different types of
341 u"timestamp": self._get_timestamp,
342 u"vpp-version": self._get_vpp_version,
343 u"dpdk-version": self._get_dpdk_version,
344 # TODO: Remove when not needed:
345 u"teardown-vat-history": self._get_vat_history,
346 u"teardown-papi-history": self._get_papi_history,
347 u"test-show-runtime": self._get_show_run,
348 u"testbed": self._get_testbed
353 """Getter - Data parsed from the XML file.
355 :returns: Data parsed from the XML file.
360 def _get_data_from_mrr_test_msg(self, msg):
361 """Get info from message of MRR performance tests.
363 :param msg: Message to be processed.
365 :returns: Processed message or original message if a problem occurs.
369 groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
370 if not groups or groups.lastindex != 1:
371 return u"Test Failed."
374 data = groups.group(1).split(u", ")
375 except (AttributeError, IndexError, ValueError, KeyError):
376 return u"Test Failed."
381 out_str += f"{(float(item) / 1e6):.2f}, "
382 return out_str[:-2] + u"]"
383 except (AttributeError, IndexError, ValueError, KeyError):
384 return u"Test Failed."
386 def _get_data_from_perf_test_msg(self, msg):
387 """Get info from message of NDRPDR performance tests.
389 :param msg: Message to be processed.
391 :returns: Processed message or original message if a problem occurs.
395 groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
396 if not groups or groups.lastindex != 10:
397 return u"Test Failed."
401 u"ndr_low": float(groups.group(1)),
402 u"ndr_low_b": float(groups.group(2)),
403 u"pdr_low": float(groups.group(3)),
404 u"pdr_low_b": float(groups.group(4)),
405 u"pdr_lat_90_1": groups.group(5),
406 u"pdr_lat_90_2": groups.group(6),
407 u"pdr_lat_50_1": groups.group(7),
408 u"pdr_lat_50_2": groups.group(8),
409 u"pdr_lat_10_1": groups.group(9),
410 u"pdr_lat_10_2": groups.group(10),
412 except (AttributeError, IndexError, ValueError, KeyError):
413 return u"Test Failed."
415 def _process_lat(in_str_1, in_str_2):
416 """Extract min, avg, max values from latency string.
418 :param in_str_1: Latency string for one direction produced by robot
420 :param in_str_2: Latency string for second direction produced by
424 :returns: Processed latency string or None if a problem occurs.
427 in_list_1 = in_str_1.split('/', 3)
428 in_list_2 = in_str_2.split('/', 3)
430 if len(in_list_1) != 4 and len(in_list_2) != 4:
433 in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
435 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
436 except hdrh.codec.HdrLengthException:
439 in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
441 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
442 except hdrh.codec.HdrLengthException:
445 if hdr_lat_1 and hdr_lat_2:
447 hdr_lat_1.get_value_at_percentile(50.0),
448 hdr_lat_1.get_value_at_percentile(90.0),
449 hdr_lat_1.get_value_at_percentile(99.0),
450 hdr_lat_2.get_value_at_percentile(50.0),
451 hdr_lat_2.get_value_at_percentile(90.0),
452 hdr_lat_2.get_value_at_percentile(99.0)
462 f"1. {(data[u'ndr_low'] / 1e6):5.2f} "
463 f"{data[u'ndr_low_b']:.2f}"
464 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f} "
465 f"{data[u'pdr_low_b']:.2f}"
468 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
469 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
470 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
473 max_len = len(str(max((max(item) for item in latency))))
474 max_len = 4 if max_len < 4 else max_len
476 for idx, lat in enumerate(latency):
481 f"{lat[0]:{max_len}d} "
482 f"{lat[1]:{max_len}d} "
483 f"{lat[2]:{max_len}d} "
484 f"{lat[3]:{max_len}d} "
485 f"{lat[4]:{max_len}d} "
486 f"{lat[5]:{max_len}d} "
491 except (AttributeError, IndexError, ValueError, KeyError):
492 return u"Test Failed."
494 def _get_testbed(self, msg):
495 """Called when extraction of testbed IP is required.
496 The testbed is identified by TG node IP address.
498 :param msg: Message to process.
503 if msg.message.count(u"Setup of TG node") or \
504 msg.message.count(u"Setup of node TG host"):
505 reg_tg_ip = re.compile(
506 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
508 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
509 except (KeyError, ValueError, IndexError, AttributeError):
512 self._data[u"metadata"][u"testbed"] = self._testbed
513 self._msg_type = None
515 def _get_vpp_version(self, msg):
516 """Called when extraction of VPP version is required.
518 :param msg: Message to process.
523 if msg.message.count(u"return STDOUT Version:") or \
524 msg.message.count(u"VPP Version:") or \
525 msg.message.count(u"VPP version:"):
526 self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
528 self._data[u"metadata"][u"version"] = self._version
529 self._msg_type = None
531 def _get_dpdk_version(self, msg):
532 """Called when extraction of DPDK version is required.
534 :param msg: Message to process.
539 if msg.message.count(u"DPDK Version:"):
541 self._version = str(re.search(
542 self.REGEX_VERSION_DPDK, msg.message).group(2))
543 self._data[u"metadata"][u"version"] = self._version
547 self._msg_type = None
549 def _get_timestamp(self, msg):
550 """Called when extraction of timestamp is required.
552 :param msg: Message to process.
557 self._timestamp = msg.timestamp[:14]
558 self._data[u"metadata"][u"generated"] = self._timestamp
559 self._msg_type = None
561 def _get_vat_history(self, msg):
562 """Called when extraction of VAT command history is required.
564 TODO: Remove when not needed.
566 :param msg: Message to process.
570 if msg.message.count(u"VAT command history:"):
571 self._conf_history_lookup_nr += 1
572 if self._conf_history_lookup_nr == 1:
573 self._data[u"tests"][self._test_id][u"conf-history"] = str()
575 self._msg_type = None
576 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
577 r"VAT command history:", u"",
578 msg.message, count=1).replace(u'\n', u' |br| ').\
581 self._data[u"tests"][self._test_id][u"conf-history"] += (
582 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
585 def _get_papi_history(self, msg):
586 """Called when extraction of PAPI command history is required.
588 :param msg: Message to process.
592 if msg.message.count(u"PAPI command history:"):
593 self._conf_history_lookup_nr += 1
594 if self._conf_history_lookup_nr == 1:
595 self._data[u"tests"][self._test_id][u"conf-history"] = str()
597 self._msg_type = None
598 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
599 r"PAPI command history:", u"",
600 msg.message, count=1).replace(u'\n', u' |br| ').\
602 self._data[u"tests"][self._test_id][u"conf-history"] += (
603 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
606 def _get_show_run(self, msg):
607 """Called when extraction of VPP operational data (output of CLI command
608 Show Runtime) is required.
610 :param msg: Message to process.
615 if not msg.message.count(u"stats runtime"):
619 if self._sh_run_counter > 1:
622 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
623 self._data[u"tests"][self._test_id][u"show-run"] = dict()
625 groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
629 host = groups.group(1)
630 except (AttributeError, IndexError):
633 sock = groups.group(2)
634 except (AttributeError, IndexError):
637 runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
638 replace(u"'", u'"').replace(u'b"', u'"').
639 replace(u'u"', u'"').split(u":", 1)[1])
642 threads_nr = len(runtime[0][u"clocks"])
643 except (IndexError, KeyError):
646 dut = u"DUT{nr}".format(
647 nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
652 u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
656 for idx in range(threads_nr):
657 if item[u"vectors"][idx] > 0:
658 clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
659 elif item[u"calls"][idx] > 0:
660 clocks = item[u"clocks"][idx] / item[u"calls"][idx]
661 elif item[u"suspends"][idx] > 0:
662 clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
666 if item[u"calls"][idx] > 0:
667 vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
671 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
672 int(item[u"suspends"][idx]):
673 oper[u"threads"][idx].append([
676 item[u"vectors"][idx],
677 item[u"suspends"][idx],
682 self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
684 def _get_ndrpdr_throughput(self, msg):
685 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
688 :param msg: The test message to be parsed.
690 :returns: Parsed data as a dict and the status (PASS/FAIL).
691 :rtype: tuple(dict, str)
695 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
696 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
699 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
701 if groups is not None:
703 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
704 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
705 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
706 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
708 except (IndexError, ValueError):
711 return throughput, status
713 def _get_plr_throughput(self, msg):
714 """Get PLRsearch lower bound and PLRsearch upper bound from the test
717 :param msg: The test message to be parsed.
719 :returns: Parsed data as a dict and the status (PASS/FAIL).
720 :rtype: tuple(dict, str)
728 groups = re.search(self.REGEX_PLR_RATE, msg)
730 if groups is not None:
732 throughput[u"LOWER"] = float(groups.group(1))
733 throughput[u"UPPER"] = float(groups.group(2))
735 except (IndexError, ValueError):
738 return throughput, status
740 def _get_ndrpdr_latency(self, msg):
741 """Get LATENCY from the test message.
743 :param msg: The test message to be parsed.
745 :returns: Parsed data as a dict and the status (PASS/FAIL).
746 :rtype: tuple(dict, str)
756 u"direction1": copy.copy(latency_default),
757 u"direction2": copy.copy(latency_default)
760 u"direction1": copy.copy(latency_default),
761 u"direction2": copy.copy(latency_default)
764 u"direction1": copy.copy(latency_default),
765 u"direction2": copy.copy(latency_default)
768 u"direction1": copy.copy(latency_default),
769 u"direction2": copy.copy(latency_default)
772 u"direction1": copy.copy(latency_default),
773 u"direction2": copy.copy(latency_default)
776 u"direction1": copy.copy(latency_default),
777 u"direction2": copy.copy(latency_default)
781 # TODO: Rewrite when long and base are not needed
782 groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
784 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
786 groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
788 return latency, u"FAIL"
790 def process_latency(in_str):
791 """Return object with parsed latency values.
793 TODO: Define class for the return type.
795 :param in_str: Input string, min/avg/max/hdrh format.
797 :returns: Dict with corresponding keys, except hdrh float values.
799 :throws IndexError: If in_str does not have enough substrings.
800 :throws ValueError: If a substring does not convert to float.
802 in_list = in_str.split('/', 3)
805 u"min": float(in_list[0]),
806 u"avg": float(in_list[1]),
807 u"max": float(in_list[2]),
811 if len(in_list) == 4:
812 rval[u"hdrh"] = str(in_list[3])
817 latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
818 latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
819 latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
820 latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
821 if groups.lastindex == 4:
822 return latency, u"PASS"
823 except (IndexError, ValueError):
827 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
828 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
829 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
830 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
831 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
832 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
833 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
834 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
835 if groups.lastindex == 12:
836 return latency, u"PASS"
837 except (IndexError, ValueError):
840 # TODO: Remove when not needed
841 latency[u"NDR10"] = {
842 u"direction1": copy.copy(latency_default),
843 u"direction2": copy.copy(latency_default)
845 latency[u"NDR50"] = {
846 u"direction1": copy.copy(latency_default),
847 u"direction2": copy.copy(latency_default)
849 latency[u"NDR90"] = {
850 u"direction1": copy.copy(latency_default),
851 u"direction2": copy.copy(latency_default)
854 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
855 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
856 latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
857 latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
858 latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
859 latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
860 latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
861 latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
862 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
863 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
864 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
865 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
866 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
867 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
868 return latency, u"PASS"
869 except (IndexError, ValueError):
872 return latency, u"FAIL"
874 def visit_suite(self, suite):
875 """Implements traversing through the suite and its direct children.
877 :param suite: Suite to process.
881 if self.start_suite(suite) is not False:
882 suite.suites.visit(self)
883 suite.tests.visit(self)
884 self.end_suite(suite)
886 def start_suite(self, suite):
887 """Called when suite starts.
889 :param suite: Suite to process.
895 parent_name = suite.parent.name
896 except AttributeError:
899 doc_str = suite.doc.\
900 replace(u'"', u"'").\
901 replace(u'\n', u' ').\
902 replace(u'\r', u'').\
903 replace(u'*[', u' |br| *[').\
904 replace(u"*", u"**").\
905 replace(u' |br| *[', u'*[', 1)
907 self._data[u"suites"][suite.longname.lower().
909 replace(u" ", u"_")] = {
910 u"name": suite.name.lower(),
912 u"parent": parent_name,
913 u"level": len(suite.longname.split(u"."))
916 suite.keywords.visit(self)
918 def end_suite(self, suite):
919 """Called when suite ends.
921 :param suite: Suite to process.
926 def visit_test(self, test):
927 """Implements traversing through the test.
929 :param test: Test to process.
933 if self.start_test(test) is not False:
934 test.keywords.visit(self)
937 def start_test(self, test):
938 """Called when test starts.
940 :param test: Test to process.
945 self._sh_run_counter = 0
947 longname_orig = test.longname.lower()
949 # Check the ignore list
950 if longname_orig in self._ignore:
953 tags = [str(tag) for tag in test.tags]
956 # Change the TC long name and name if defined in the mapping table
957 longname = self._mapping.get(longname_orig, None)
958 if longname is not None:
959 name = longname.split(u'.')[-1]
961 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
965 longname = longname_orig
966 name = test.name.lower()
968 # Remove TC number from the TC long name (backward compatibility):
969 self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
970 # Remove TC number from the TC name (not needed):
971 test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
973 test_result[u"parent"] = test.parent.name.lower()
974 test_result[u"tags"] = tags
975 test_result["doc"] = test.doc.\
976 replace(u'"', u"'").\
977 replace(u'\n', u' ').\
978 replace(u'\r', u'').\
979 replace(u'[', u' |br| [').\
980 replace(u' |br| [', u'[', 1)
981 test_result[u"type"] = u"FUNC"
982 test_result[u"status"] = test.status
984 if test.status == u"PASS":
985 if u"NDRPDR" in tags:
986 test_result[u"msg"] = self._get_data_from_perf_test_msg(
987 test.message).replace(u'\n', u' |br| ').\
988 replace(u'\r', u'').replace(u'"', u"'")
989 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
990 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
991 test.message).replace(u'\n', u' |br| ').\
992 replace(u'\r', u'').replace(u'"', u"'")
994 test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\
995 replace(u'\r', u'').replace(u'"', u"'")
997 test_result[u"msg"] = u"Test Failed."
999 if u"PERFTEST" in tags:
1000 # Replace info about cores (e.g. -1c-) with the info about threads
1001 # and cores (e.g. -1t1c-) in the long test case names and in the
1002 # test case names if necessary.
1003 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1007 for tag in test_result[u"tags"]:
1008 groups = re.search(self.REGEX_TC_TAG, tag)
1014 self._test_id = re.sub(
1015 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1016 self._test_id, count=1
1018 test_result[u"name"] = re.sub(
1019 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1020 test_result["name"], count=1
1023 test_result[u"status"] = u"FAIL"
1024 self._data[u"tests"][self._test_id] = test_result
1026 f"The test {self._test_id} has no or more than one "
1027 f"multi-threading tags.\n"
1028 f"Tags: {test_result[u'tags']}"
1032 if test.status == u"PASS":
1033 if u"NDRPDR" in tags:
1034 test_result[u"type"] = u"NDRPDR"
1035 test_result[u"throughput"], test_result[u"status"] = \
1036 self._get_ndrpdr_throughput(test.message)
1037 test_result[u"latency"], test_result[u"status"] = \
1038 self._get_ndrpdr_latency(test.message)
1039 elif u"SOAK" in tags:
1040 test_result[u"type"] = u"SOAK"
1041 test_result[u"throughput"], test_result[u"status"] = \
1042 self._get_plr_throughput(test.message)
1043 elif u"TCP" in tags:
1044 test_result[u"type"] = u"TCP"
1045 groups = re.search(self.REGEX_TCP, test.message)
1046 test_result[u"result"] = int(groups.group(2))
1047 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1049 test_result[u"type"] = u"MRR"
1051 test_result[u"type"] = u"BMRR"
1053 test_result[u"result"] = dict()
1054 groups = re.search(self.REGEX_BMRR, test.message)
1055 if groups is not None:
1056 items_str = groups.group(1)
1057 items_float = [float(item.strip()) for item
1058 in items_str.split(",")]
1059 # Use whole list in CSIT-1180.
1060 stats = jumpavg.AvgStdevStats.for_runs(items_float)
1061 test_result[u"result"][u"receive-rate"] = stats.avg
1063 groups = re.search(self.REGEX_MRR, test.message)
1064 test_result[u"result"][u"receive-rate"] = \
1065 float(groups.group(3)) / float(groups.group(1))
1066 elif u"RECONF" in tags:
1067 test_result[u"type"] = u"RECONF"
1068 test_result[u"result"] = None
1070 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1071 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1072 test_result[u"result"] = {
1073 u"loss": int(grps_loss.group(1)),
1074 u"time": float(grps_time.group(1))
1076 except (AttributeError, IndexError, ValueError, TypeError):
1077 test_result[u"status"] = u"FAIL"
1078 elif u"DEVICETEST" in tags:
1079 test_result[u"type"] = u"DEVICETEST"
1081 test_result[u"status"] = u"FAIL"
1082 self._data[u"tests"][self._test_id] = test_result
1085 self._data[u"tests"][self._test_id] = test_result
1087 def end_test(self, test):
1088 """Called when test ends.
1090 :param test: Test to process.
1095 def visit_keyword(self, keyword):
1096 """Implements traversing through the keyword and its child keywords.
1098 :param keyword: Keyword to process.
1099 :type keyword: Keyword
1102 if self.start_keyword(keyword) is not False:
1103 self.end_keyword(keyword)
1105 def start_keyword(self, keyword):
1106 """Called when keyword starts. Default implementation does nothing.
1108 :param keyword: Keyword to process.
1109 :type keyword: Keyword
1113 if keyword.type == u"setup":
1114 self.visit_setup_kw(keyword)
1115 elif keyword.type == u"teardown":
1116 self.visit_teardown_kw(keyword)
1118 self.visit_test_kw(keyword)
1119 except AttributeError:
1122 def end_keyword(self, keyword):
1123 """Called when keyword ends. Default implementation does nothing.
1125 :param keyword: Keyword to process.
1126 :type keyword: Keyword
1130 def visit_test_kw(self, test_kw):
1131 """Implements traversing through the test keyword and its child
1134 :param test_kw: Keyword to process.
1135 :type test_kw: Keyword
1138 for keyword in test_kw.keywords:
1139 if self.start_test_kw(keyword) is not False:
1140 self.visit_test_kw(keyword)
1141 self.end_test_kw(keyword)
1143 def start_test_kw(self, test_kw):
1144 """Called when test keyword starts. Default implementation does
1147 :param test_kw: Keyword to process.
1148 :type test_kw: Keyword
1151 if test_kw.name.count(u"Show Runtime On All Duts") or \
1152 test_kw.name.count(u"Show Runtime Counters On All Duts"):
1153 self._msg_type = u"test-show-runtime"
1154 self._sh_run_counter += 1
1155 elif test_kw.name.count(u"Install Dpdk Test") and not self._version:
1156 self._msg_type = u"dpdk-version"
1159 test_kw.messages.visit(self)
1161 def end_test_kw(self, test_kw):
1162 """Called when keyword ends. Default implementation does nothing.
1164 :param test_kw: Keyword to process.
1165 :type test_kw: Keyword
1169 def visit_setup_kw(self, setup_kw):
1170 """Implements traversing through the teardown keyword and its child
1173 :param setup_kw: Keyword to process.
1174 :type setup_kw: Keyword
1177 for keyword in setup_kw.keywords:
1178 if self.start_setup_kw(keyword) is not False:
1179 self.visit_setup_kw(keyword)
1180 self.end_setup_kw(keyword)
1182 def start_setup_kw(self, setup_kw):
1183 """Called when teardown keyword starts. Default implementation does
1186 :param setup_kw: Keyword to process.
1187 :type setup_kw: Keyword
1190 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1191 and not self._version:
1192 self._msg_type = u"vpp-version"
1193 elif setup_kw.name.count(u"Set Global Variable") \
1194 and not self._timestamp:
1195 self._msg_type = u"timestamp"
1196 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1197 self._msg_type = u"testbed"
1200 setup_kw.messages.visit(self)
1202 def end_setup_kw(self, setup_kw):
1203 """Called when keyword ends. Default implementation does nothing.
1205 :param setup_kw: Keyword to process.
1206 :type setup_kw: Keyword
1210 def visit_teardown_kw(self, teardown_kw):
1211 """Implements traversing through the teardown keyword and its child
1214 :param teardown_kw: Keyword to process.
1215 :type teardown_kw: Keyword
1218 for keyword in teardown_kw.keywords:
1219 if self.start_teardown_kw(keyword) is not False:
1220 self.visit_teardown_kw(keyword)
1221 self.end_teardown_kw(keyword)
1223 def start_teardown_kw(self, teardown_kw):
1224 """Called when teardown keyword starts
1226 :param teardown_kw: Keyword to process.
1227 :type teardown_kw: Keyword
1231 if teardown_kw.name.count(u"Show Vat History On All Duts"):
1232 # TODO: Remove when not needed:
1233 self._conf_history_lookup_nr = 0
1234 self._msg_type = u"teardown-vat-history"
1235 teardown_kw.messages.visit(self)
1236 elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1237 self._conf_history_lookup_nr = 0
1238 self._msg_type = u"teardown-papi-history"
1239 teardown_kw.messages.visit(self)
1241 def end_teardown_kw(self, teardown_kw):
1242 """Called when keyword ends. Default implementation does nothing.
1244 :param teardown_kw: Keyword to process.
1245 :type teardown_kw: Keyword
1249 def visit_message(self, msg):
1250 """Implements visiting the message.
1252 :param msg: Message to process.
1256 if self.start_message(msg) is not False:
1257 self.end_message(msg)
1259 def start_message(self, msg):
1260 """Called when message starts. Get required information from messages:
1263 :param msg: Message to process.
1269 self.parse_msg[self._msg_type](msg)
1271 def end_message(self, msg):
1272 """Called when message ends. Default implementation does nothing.
1274 :param msg: Message to process.
1283 The data is extracted from output.xml files generated by Jenkins jobs and
1284 stored in pandas' DataFrames.
1290 (as described in ExecutionChecker documentation)
1292 (as described in ExecutionChecker documentation)
1294 (as described in ExecutionChecker documentation)
1297 def __init__(self, spec):
1300 :param spec: Specification.
1301 :type spec: Specification
1308 self._input_data = pd.Series()
1312 """Getter - Input data.
1314 :returns: Input data
1315 :rtype: pandas.Series
1317 return self._input_data
1319 def metadata(self, job, build):
1320 """Getter - metadata
1322 :param job: Job which metadata we want.
1323 :param build: Build which metadata we want.
1327 :rtype: pandas.Series
1330 return self.data[job][build][u"metadata"]
1332 def suites(self, job, build):
1335 :param job: Job which suites we want.
1336 :param build: Build which suites we want.
1340 :rtype: pandas.Series
1343 return self.data[job][str(build)][u"suites"]
1345 def tests(self, job, build):
1348 :param job: Job which tests we want.
1349 :param build: Build which tests we want.
1353 :rtype: pandas.Series
1356 return self.data[job][build][u"tests"]
1358 def _parse_tests(self, job, build, log):
1359 """Process data from robot output.xml file and return JSON structured
1362 :param job: The name of job which build output data will be processed.
1363 :param build: The build which output data will be processed.
1364 :param log: List of log messages.
1367 :type log: list of tuples (severity, msg)
1368 :returns: JSON data structure.
1377 with open(build[u"file-name"], u'r') as data_file:
1379 result = ExecutionResult(data_file)
1380 except errors.DataError as err:
1382 (u"ERROR", f"Error occurred while parsing output.xml: "
1386 checker = ExecutionChecker(metadata, self._cfg.mapping,
1388 result.visit(checker)
1392 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1393 """Download and parse the input data file.
1395 :param pid: PID of the process executing this method.
1396 :param job: Name of the Jenkins job which generated the processed input
1398 :param build: Information about the Jenkins build which generated the
1399 processed input file.
1400 :param repeat: Repeat the download specified number of times if not
1411 (u"INFO", f" Processing the job/build: {job}: {build[u'build']}")
1419 success = download_and_unzip_data_file(self._cfg, job, build, pid,
1427 f"It is not possible to download the input data file from the "
1428 f"job {job}, build {build[u'build']}, or it is damaged. "
1434 f" Processing data from the build {build[u'build']} ...")
1436 data = self._parse_tests(job, build, logs)
1440 f"Input data file from the job {job}, build "
1441 f"{build[u'build']} is damaged. Skipped.")
1444 state = u"processed"
1447 remove(build[u"file-name"])
1448 except OSError as err:
1450 ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
1454 # If the time-period is defined in the specification file, remove all
1455 # files which are outside the time period.
1456 timeperiod = self._cfg.input.get(u"time-period", None)
1457 if timeperiod and data:
1459 timeperiod = timedelta(int(timeperiod))
1460 metadata = data.get(u"metadata", None)
1462 generated = metadata.get(u"generated", None)
1464 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1465 if (now - generated) > timeperiod:
1466 # Remove the data and the file:
1471 f" The build {job}/{build[u'build']} is "
1472 f"outdated, will be removed.")
1474 logs.append((u"INFO", u" Done."))
1476 for level, line in logs:
1477 if level == u"INFO":
1479 elif level == u"ERROR":
1481 elif level == u"DEBUG":
1483 elif level == u"CRITICAL":
1484 logging.critical(line)
1485 elif level == u"WARNING":
1486 logging.warning(line)
1488 return {u"data": data, u"state": state, u"job": job, u"build": build}
1490 def download_and_parse_data(self, repeat=1):
1491 """Download the input data files, parse input data from input files and
1492 store in pandas' Series.
1494 :param repeat: Repeat the download specified number of times if not
1499 logging.info(u"Downloading and parsing input files ...")
1501 for job, builds in self._cfg.builds.items():
1502 for build in builds:
1504 result = self._download_and_parse_build(job, build, repeat)
1505 build_nr = result[u"build"][u"build"]
1508 data = result[u"data"]
1509 build_data = pd.Series({
1510 u"metadata": pd.Series(
1511 list(data[u"metadata"].values()),
1512 index=list(data[u"metadata"].keys())
1514 u"suites": pd.Series(
1515 list(data[u"suites"].values()),
1516 index=list(data[u"suites"].keys())
1518 u"tests": pd.Series(
1519 list(data[u"tests"].values()),
1520 index=list(data[u"tests"].keys())
1524 if self._input_data.get(job, None) is None:
1525 self._input_data[job] = pd.Series()
1526 self._input_data[job][str(build_nr)] = build_data
1528 self._cfg.set_input_file_name(
1529 job, build_nr, result[u"build"][u"file-name"])
1531 self._cfg.set_input_state(job, build_nr, result[u"state"])
1534 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1535 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1537 logging.info(u"Done.")
1540 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1541 """Return the index of character in the string which is the end of tag.
1543 :param tag_filter: The string where the end of tag is being searched.
1544 :param start: The index where the searching is stated.
1545 :param closer: The character which is the tag closer.
1546 :type tag_filter: str
1549 :returns: The index of the tag closer.
1554 idx_opener = tag_filter.index(closer, start)
1555 return tag_filter.index(closer, idx_opener + 1)
1560 def _condition(tag_filter):
1561 """Create a conditional statement from the given tag filter.
1563 :param tag_filter: Filter based on tags from the element specification.
1564 :type tag_filter: str
1565 :returns: Conditional statement which can be evaluated.
1571 index = InputData._end_of_tag(tag_filter, index)
1575 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1577 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1578 continue_on_error=False):
1579 """Filter required data from the given jobs and builds.
1581 The output data structure is:
1585 - test (or suite) 1 ID:
1591 - test (or suite) n ID:
1598 :param element: Element which will use the filtered data.
1599 :param params: Parameters which will be included in the output. If None,
1600 all parameters are included.
1601 :param data: If not None, this data is used instead of data specified
1603 :param data_set: The set of data to be filtered: tests, suites,
1605 :param continue_on_error: Continue if there is error while reading the
1606 data. The Item will be empty then
1607 :type element: pandas.Series
1611 :type continue_on_error: bool
1612 :returns: Filtered data.
1613 :rtype pandas.Series
1617 if data_set == "suites":
1619 elif element[u"filter"] in (u"all", u"template"):
1622 cond = InputData._condition(element[u"filter"])
1623 logging.debug(f" Filter: {cond}")
1625 logging.error(u" No filter defined.")
1629 params = element.get(u"parameters", None)
1631 params.append(u"type")
1633 data_to_filter = data if data else element[u"data"]
1636 for job, builds in data_to_filter.items():
1637 data[job] = pd.Series()
1638 for build in builds:
1639 data[job][str(build)] = pd.Series()
1642 self.data[job][str(build)][data_set].items())
1644 if continue_on_error:
1648 for test_id, test_data in data_dict.items():
1649 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1650 data[job][str(build)][test_id] = pd.Series()
1652 for param, val in test_data.items():
1653 data[job][str(build)][test_id][param] = val
1655 for param in params:
1657 data[job][str(build)][test_id][param] =\
1660 data[job][str(build)][test_id][param] =\
1664 except (KeyError, IndexError, ValueError) as err:
1666 f"Missing mandatory parameter in the element specification: "
1670 except AttributeError as err:
1671 logging.error(repr(err))
1673 except SyntaxError as err:
1675 f"The filter {cond} is not correct. Check if all tags are "
1676 f"enclosed by apostrophes.\n{repr(err)}"
1680 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1681 continue_on_error=False):
1682 """Filter required data from the given jobs and builds.
1684 The output data structure is:
1688 - test (or suite) 1 ID:
1694 - test (or suite) n ID:
1701 :param element: Element which will use the filtered data.
1702 :param params: Parameters which will be included in the output. If None,
1703 all parameters are included.
1704 :param data_set: The set of data to be filtered: tests, suites,
1706 :param continue_on_error: Continue if there is error while reading the
1707 data. The Item will be empty then
1708 :type element: pandas.Series
1711 :type continue_on_error: bool
1712 :returns: Filtered data.
1713 :rtype pandas.Series
1716 include = element.get(u"include", None)
1718 logging.warning(u"No tests to include, skipping the element.")
1722 params = element.get(u"parameters", None)
1724 params.append(u"type")
1728 for job, builds in element[u"data"].items():
1729 data[job] = pd.Series()
1730 for build in builds:
1731 data[job][str(build)] = pd.Series()
1732 for test in include:
1734 reg_ex = re.compile(str(test).lower())
1735 for test_id in self.data[job][
1736 str(build)][data_set].keys():
1737 if re.match(reg_ex, str(test_id).lower()):
1738 test_data = self.data[job][
1739 str(build)][data_set][test_id]
1740 data[job][str(build)][test_id] = pd.Series()
1742 for param, val in test_data.items():
1743 data[job][str(build)][test_id]\
1746 for param in params:
1748 data[job][str(build)][
1752 data[job][str(build)][
1753 test_id][param] = u"No Data"
1754 except KeyError as err:
1755 logging.error(repr(err))
1756 if continue_on_error:
1761 except (KeyError, IndexError, ValueError) as err:
1763 f"Missing mandatory parameter in the element "
1764 f"specification: {repr(err)}"
1767 except AttributeError as err:
1768 logging.error(repr(err))
1772 def merge_data(data):
1773 """Merge data from more jobs and builds to a simple data structure.
1775 The output data structure is:
1777 - test (suite) 1 ID:
1783 - test (suite) n ID:
1786 :param data: Data to merge.
1787 :type data: pandas.Series
1788 :returns: Merged data.
1789 :rtype: pandas.Series
1792 logging.info(u" Merging data ...")
1794 merged_data = pd.Series()
1795 for builds in data.values:
1796 for item in builds.values:
1797 for item_id, item_data in item.items():
1798 merged_data[item_id] = item_data
1802 def print_all_oper_data(self):
1803 """Print all operational data to console.
1811 u"Cycles per Packet",
1812 u"Average Vector Size"
1815 for job in self._input_data.values:
1816 for build in job.values:
1817 for test_id, test_data in build[u"tests"].items():
1819 if test_data.get(u"show-run", None) is None:
1821 for dut_name, data in test_data[u"show-run"].items():
1822 if data.get(u"threads", None) is None:
1824 print(f"Host IP: {data.get(u'host', '')}, "
1825 f"Socket: {data.get(u'socket', '')}")
1826 for thread_nr, thread in data[u"threads"].items():
1827 txt_table = prettytable.PrettyTable(tbl_hdr)
1830 txt_table.add_row(row)
1832 if len(thread) == 0:
1835 avg = f", Average Vector Size per Node: " \
1836 f"{(avg / len(thread)):.2f}"
1837 th_name = u"main" if thread_nr == 0 \
1838 else f"worker_{thread_nr}"
1839 print(f"{dut_name}, {th_name}{avg}")
1840 txt_table.float_format = u".2"
1841 txt_table.align = u"r"
1842 txt_table.align[u"Name"] = u"l"
1843 print(f"{txt_table.get_string()}\n")