1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
28 from os import remove, walk, listdir
29 from os.path import isfile, isdir, join
30 from datetime import datetime as dt
31 from datetime import timedelta
32 from json import loads
33 from json.decoder import JSONDecodeError
40 from robot.api import ExecutionResult, ResultVisitor
41 from robot import errors
43 from resources.libraries.python import jumpavg
44 from input_data_files import download_and_unzip_data_file
45 from pal_errors import PresentationError
48 # Separator used in file names
52 class ExecutionChecker(ResultVisitor):
53 """Class to traverse through the test suite structure.
55 The functionality implemented in this class generates a json structure:
61 "generated": "Timestamp",
62 "version": "SUT version",
63 "job": "Jenkins job name",
64 "build": "Information about the build"
67 "Suite long name 1": {
69 "doc": "Suite 1 documentation",
70 "parent": "Suite 1 parent",
71 "level": "Level of the suite in the suite hierarchy"
73 "Suite long name N": {
75 "doc": "Suite N documentation",
76 "parent": "Suite 2 parent",
77 "level": "Level of the suite in the suite hierarchy"
84 "parent": "Name of the parent of the test",
85 "doc": "Test documentation",
86 "msg": "Test message",
87 "conf-history": "DUT1 and DUT2 VAT History",
88 "show-run": "Show Run",
89 "tags": ["tag 1", "tag 2", "tag n"],
91 "status": "PASS" | "FAIL",
137 "parent": "Name of the parent of the test",
138 "doc": "Test documentation",
139 "msg": "Test message",
140 "tags": ["tag 1", "tag 2", "tag n"],
142 "status": "PASS" | "FAIL",
149 "parent": "Name of the parent of the test",
150 "doc": "Test documentation",
151 "msg": "Test message",
152 "tags": ["tag 1", "tag 2", "tag n"],
153 "type": "MRR" | "BMRR",
154 "status": "PASS" | "FAIL",
156 "receive-rate": float,
157 # Average of a list, computed using AvgStdevStats.
158 # In CSIT-1180, replace with List[float].
172 "metadata": { # Optional
173 "version": "VPP version",
174 "job": "Jenkins job name",
175 "build": "Information about the build"
179 "doc": "Suite 1 documentation",
180 "parent": "Suite 1 parent",
181 "level": "Level of the suite in the suite hierarchy"
184 "doc": "Suite N documentation",
185 "parent": "Suite 2 parent",
186 "level": "Level of the suite in the suite hierarchy"
192 "parent": "Name of the parent of the test",
193 "doc": "Test documentation"
194 "msg": "Test message"
195 "tags": ["tag 1", "tag 2", "tag n"],
196 "conf-history": "DUT1 and DUT2 VAT History"
197 "show-run": "Show Run"
198 "status": "PASS" | "FAIL"
206 .. note:: ID is the lowercase full path to the test.
209 REGEX_PLR_RATE = re.compile(
210 r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
211 r'PLRsearch upper bound::?\s(\d+.\d+)'
213 REGEX_NDRPDR_RATE = re.compile(
214 r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
215 r'NDR_UPPER:\s(\d+.\d+).*\n'
216 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
217 r'PDR_UPPER:\s(\d+.\d+)'
219 REGEX_PERF_MSG_INFO = re.compile(
220 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
221 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
222 r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
223 r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
224 r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
226 REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
228 # TODO: Remove when not needed
229 REGEX_NDRPDR_LAT_BASE = re.compile(
230 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
231 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
233 REGEX_NDRPDR_LAT = re.compile(
234 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
235 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
236 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
237 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
238 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
239 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
241 # TODO: Remove when not needed
242 REGEX_NDRPDR_LAT_LONG = re.compile(
243 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
244 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
245 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
246 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
247 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
248 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
249 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
250 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
251 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
253 REGEX_VERSION_VPP = re.compile(
254 r"(return STDOUT Version:\s*|"
255 r"VPP Version:\s*|VPP version:\s*)(.*)"
257 REGEX_VERSION_DPDK = re.compile(
258 r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
260 REGEX_TCP = re.compile(
261 r'Total\s(rps|cps|throughput):\s(\d*).*$'
263 REGEX_MRR = re.compile(
264 r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
265 r'tx\s(\d*),\srx\s(\d*)'
267 REGEX_BMRR = re.compile(
268 r'Maximum Receive Rate trial results'
269 r' in packets per second: \[(.*)\]'
271 REGEX_RECONF_LOSS = re.compile(
272 r'Packets lost due to reconfig: (\d*)'
274 REGEX_RECONF_TIME = re.compile(
275 r'Implied time lost: (\d*.[\de-]*)'
277 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
279 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
281 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
283 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
285 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
287 def __init__(self, metadata, mapping, ignore):
290 :param metadata: Key-value pairs to be included in "metadata" part of
292 :param mapping: Mapping of the old names of test cases to the new
294 :param ignore: List of TCs to be ignored.
300 # Type of message to parse out from the test messages
301 self._msg_type = None
307 self._timestamp = None
309 # Testbed. The testbed is identified by TG node IP address.
312 # Mapping of TCs long names
313 self._mapping = mapping
316 self._ignore = ignore
318 # Number of PAPI History messages found:
320 # 1 - PAPI History of DUT1
321 # 2 - PAPI History of DUT2
322 self._conf_history_lookup_nr = 0
324 self._sh_run_counter = 0
326 # Test ID of currently processed test- the lowercase full path to the
330 # The main data structure
332 u"metadata": OrderedDict(),
333 u"suites": OrderedDict(),
334 u"tests": OrderedDict()
337 # Save the provided metadata
338 for key, val in metadata.items():
339 self._data[u"metadata"][key] = val
341 # Dictionary defining the methods used to parse different types of
344 u"timestamp": self._get_timestamp,
345 u"vpp-version": self._get_vpp_version,
346 u"dpdk-version": self._get_dpdk_version,
347 # TODO: Remove when not needed:
348 u"teardown-vat-history": self._get_vat_history,
349 u"teardown-papi-history": self._get_papi_history,
350 u"test-show-runtime": self._get_show_run,
351 u"testbed": self._get_testbed
356 """Getter - Data parsed from the XML file.
358 :returns: Data parsed from the XML file.
363 def _get_data_from_mrr_test_msg(self, msg):
364 """Get info from message of MRR performance tests.
366 :param msg: Message to be processed.
368 :returns: Processed message or original message if a problem occurs.
372 groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
373 if not groups or groups.lastindex != 1:
374 return u"Test Failed."
377 data = groups.group(1).split(u", ")
378 except (AttributeError, IndexError, ValueError, KeyError):
379 return u"Test Failed."
384 out_str += f"{(float(item) / 1e6):.2f}, "
385 return out_str[:-2] + u"]"
386 except (AttributeError, IndexError, ValueError, KeyError):
387 return u"Test Failed."
389 def _get_data_from_perf_test_msg(self, msg):
390 """Get info from message of NDRPDR performance tests.
392 :param msg: Message to be processed.
394 :returns: Processed message or original message if a problem occurs.
398 groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
399 if not groups or groups.lastindex != 10:
400 return u"Test Failed."
404 u"ndr_low": float(groups.group(1)),
405 u"ndr_low_b": float(groups.group(2)),
406 u"pdr_low": float(groups.group(3)),
407 u"pdr_low_b": float(groups.group(4)),
408 u"pdr_lat_90_1": groups.group(5),
409 u"pdr_lat_90_2": groups.group(6),
410 u"pdr_lat_50_1": groups.group(7),
411 u"pdr_lat_50_2": groups.group(8),
412 u"pdr_lat_10_1": groups.group(9),
413 u"pdr_lat_10_2": groups.group(10),
415 except (AttributeError, IndexError, ValueError, KeyError):
416 return u"Test Failed."
418 def _process_lat(in_str_1, in_str_2):
419 """Extract min, avg, max values from latency string.
421 :param in_str_1: Latency string for one direction produced by robot
423 :param in_str_2: Latency string for second direction produced by
427 :returns: Processed latency string or None if a problem occurs.
430 in_list_1 = in_str_1.split('/', 3)
431 in_list_2 = in_str_2.split('/', 3)
433 if len(in_list_1) != 4 and len(in_list_2) != 4:
436 in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
438 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
439 except hdrh.codec.HdrLengthException:
442 in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
444 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
445 except hdrh.codec.HdrLengthException:
448 if hdr_lat_1 and hdr_lat_2:
450 hdr_lat_1.get_value_at_percentile(50.0),
451 hdr_lat_1.get_value_at_percentile(90.0),
452 hdr_lat_1.get_value_at_percentile(99.0),
453 hdr_lat_2.get_value_at_percentile(50.0),
454 hdr_lat_2.get_value_at_percentile(90.0),
455 hdr_lat_2.get_value_at_percentile(99.0)
465 f"1. {(data[u'ndr_low'] / 1e6):5.2f} "
466 f"{data[u'ndr_low_b']:5.2f}"
467 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f} "
468 f"{data[u'pdr_low_b']:5.2f}"
471 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
472 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
473 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
476 max_len = len(str(max((max(item) for item in latency))))
477 max_len = 4 if max_len < 4 else max_len
479 for idx, lat in enumerate(latency):
484 f"{lat[0]:{max_len}d} "
485 f"{lat[1]:{max_len}d} "
486 f"{lat[2]:{max_len}d} "
487 f"{lat[3]:{max_len}d} "
488 f"{lat[4]:{max_len}d} "
489 f"{lat[5]:{max_len}d} "
494 except (AttributeError, IndexError, ValueError, KeyError):
495 return u"Test Failed."
497 def _get_testbed(self, msg):
498 """Called when extraction of testbed IP is required.
499 The testbed is identified by TG node IP address.
501 :param msg: Message to process.
506 if msg.message.count(u"Setup of TG node") or \
507 msg.message.count(u"Setup of node TG host"):
508 reg_tg_ip = re.compile(
509 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
511 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
512 except (KeyError, ValueError, IndexError, AttributeError):
515 self._data[u"metadata"][u"testbed"] = self._testbed
516 self._msg_type = None
518 def _get_vpp_version(self, msg):
519 """Called when extraction of VPP version is required.
521 :param msg: Message to process.
526 if msg.message.count(u"return STDOUT Version:") or \
527 msg.message.count(u"VPP Version:") or \
528 msg.message.count(u"VPP version:"):
529 self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
531 self._data[u"metadata"][u"version"] = self._version
532 self._msg_type = None
534 def _get_dpdk_version(self, msg):
535 """Called when extraction of DPDK version is required.
537 :param msg: Message to process.
542 if msg.message.count(u"DPDK Version:"):
544 self._version = str(re.search(
545 self.REGEX_VERSION_DPDK, msg.message).group(2))
546 self._data[u"metadata"][u"version"] = self._version
550 self._msg_type = None
552 def _get_timestamp(self, msg):
553 """Called when extraction of timestamp is required.
555 :param msg: Message to process.
560 self._timestamp = msg.timestamp[:14]
561 self._data[u"metadata"][u"generated"] = self._timestamp
562 self._msg_type = None
564 def _get_vat_history(self, msg):
565 """Called when extraction of VAT command history is required.
567 TODO: Remove when not needed.
569 :param msg: Message to process.
573 if msg.message.count(u"VAT command history:"):
574 self._conf_history_lookup_nr += 1
575 if self._conf_history_lookup_nr == 1:
576 self._data[u"tests"][self._test_id][u"conf-history"] = str()
578 self._msg_type = None
579 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
580 r"VAT command history:", u"",
581 msg.message, count=1).replace(u'\n', u' |br| ').\
584 self._data[u"tests"][self._test_id][u"conf-history"] += (
585 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
588 def _get_papi_history(self, msg):
589 """Called when extraction of PAPI command history is required.
591 :param msg: Message to process.
595 if msg.message.count(u"PAPI command history:"):
596 self._conf_history_lookup_nr += 1
597 if self._conf_history_lookup_nr == 1:
598 self._data[u"tests"][self._test_id][u"conf-history"] = str()
600 self._msg_type = None
601 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
602 r"PAPI command history:", u"",
603 msg.message, count=1).replace(u'\n', u' |br| ').\
605 self._data[u"tests"][self._test_id][u"conf-history"] += (
606 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
609 def _get_show_run(self, msg):
610 """Called when extraction of VPP operational data (output of CLI command
611 Show Runtime) is required.
613 :param msg: Message to process.
618 if not msg.message.count(u"stats runtime"):
622 if self._sh_run_counter > 1:
625 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
626 self._data[u"tests"][self._test_id][u"show-run"] = dict()
628 groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
632 host = groups.group(1)
633 except (AttributeError, IndexError):
636 sock = groups.group(2)
637 except (AttributeError, IndexError):
640 runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
641 replace(u"'", u'"').replace(u'b"', u'"').
642 replace(u'u"', u'"').split(u":", 1)[1])
645 threads_nr = len(runtime[0][u"clocks"])
646 except (IndexError, KeyError):
649 dut = u"DUT{nr}".format(
650 nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
655 u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
659 for idx in range(threads_nr):
660 if item[u"vectors"][idx] > 0:
661 clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
662 elif item[u"calls"][idx] > 0:
663 clocks = item[u"clocks"][idx] / item[u"calls"][idx]
664 elif item[u"suspends"][idx] > 0:
665 clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
669 if item[u"calls"][idx] > 0:
670 vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
674 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
675 int(item[u"suspends"][idx]):
676 oper[u"threads"][idx].append([
679 item[u"vectors"][idx],
680 item[u"suspends"][idx],
685 self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
687 def _get_ndrpdr_throughput(self, msg):
688 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
691 :param msg: The test message to be parsed.
693 :returns: Parsed data as a dict and the status (PASS/FAIL).
694 :rtype: tuple(dict, str)
698 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
699 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
702 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
704 if groups is not None:
706 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
707 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
708 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
709 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
711 except (IndexError, ValueError):
714 return throughput, status
716 def _get_plr_throughput(self, msg):
717 """Get PLRsearch lower bound and PLRsearch upper bound from the test
720 :param msg: The test message to be parsed.
722 :returns: Parsed data as a dict and the status (PASS/FAIL).
723 :rtype: tuple(dict, str)
731 groups = re.search(self.REGEX_PLR_RATE, msg)
733 if groups is not None:
735 throughput[u"LOWER"] = float(groups.group(1))
736 throughput[u"UPPER"] = float(groups.group(2))
738 except (IndexError, ValueError):
741 return throughput, status
743 def _get_ndrpdr_latency(self, msg):
744 """Get LATENCY from the test message.
746 :param msg: The test message to be parsed.
748 :returns: Parsed data as a dict and the status (PASS/FAIL).
749 :rtype: tuple(dict, str)
759 u"direction1": copy.copy(latency_default),
760 u"direction2": copy.copy(latency_default)
763 u"direction1": copy.copy(latency_default),
764 u"direction2": copy.copy(latency_default)
767 u"direction1": copy.copy(latency_default),
768 u"direction2": copy.copy(latency_default)
771 u"direction1": copy.copy(latency_default),
772 u"direction2": copy.copy(latency_default)
775 u"direction1": copy.copy(latency_default),
776 u"direction2": copy.copy(latency_default)
779 u"direction1": copy.copy(latency_default),
780 u"direction2": copy.copy(latency_default)
784 # TODO: Rewrite when long and base are not needed
785 groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
787 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
789 groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
791 return latency, u"FAIL"
793 def process_latency(in_str):
794 """Return object with parsed latency values.
796 TODO: Define class for the return type.
798 :param in_str: Input string, min/avg/max/hdrh format.
800 :returns: Dict with corresponding keys, except hdrh float values.
802 :throws IndexError: If in_str does not have enough substrings.
803 :throws ValueError: If a substring does not convert to float.
805 in_list = in_str.split('/', 3)
808 u"min": float(in_list[0]),
809 u"avg": float(in_list[1]),
810 u"max": float(in_list[2]),
814 if len(in_list) == 4:
815 rval[u"hdrh"] = str(in_list[3])
820 latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
821 latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
822 latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
823 latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
824 if groups.lastindex == 4:
825 return latency, u"PASS"
826 except (IndexError, ValueError):
830 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
831 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
832 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
833 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
834 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
835 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
836 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
837 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
838 if groups.lastindex == 12:
839 return latency, u"PASS"
840 except (IndexError, ValueError):
843 # TODO: Remove when not needed
844 latency[u"NDR10"] = {
845 u"direction1": copy.copy(latency_default),
846 u"direction2": copy.copy(latency_default)
848 latency[u"NDR50"] = {
849 u"direction1": copy.copy(latency_default),
850 u"direction2": copy.copy(latency_default)
852 latency[u"NDR90"] = {
853 u"direction1": copy.copy(latency_default),
854 u"direction2": copy.copy(latency_default)
857 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
858 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
859 latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
860 latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
861 latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
862 latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
863 latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
864 latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
865 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
866 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
867 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
868 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
869 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
870 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
871 return latency, u"PASS"
872 except (IndexError, ValueError):
875 return latency, u"FAIL"
878 def _get_hoststack_data(msg, tags):
879 """Get data from the hoststack test message.
881 :param msg: The test message to be parsed.
882 :param tags: Test tags.
885 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
886 :rtype: tuple(dict, str)
891 msg = msg.replace(u"'", u'"').replace(u" ", u"")
892 if u"LDPRELOAD" in tags:
896 except JSONDecodeError:
898 elif u"VPPECHO" in tags:
900 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
902 client=loads(msg_lst[0]),
903 server=loads(msg_lst[1])
906 except (JSONDecodeError, IndexError):
909 return result, status
911 def visit_suite(self, suite):
912 """Implements traversing through the suite and its direct children.
914 :param suite: Suite to process.
918 if self.start_suite(suite) is not False:
919 suite.suites.visit(self)
920 suite.tests.visit(self)
921 self.end_suite(suite)
923 def start_suite(self, suite):
924 """Called when suite starts.
926 :param suite: Suite to process.
932 parent_name = suite.parent.name
933 except AttributeError:
936 doc_str = suite.doc.\
937 replace(u'"', u"'").\
938 replace(u'\n', u' ').\
939 replace(u'\r', u'').\
940 replace(u'*[', u' |br| *[').\
941 replace(u"*", u"**").\
942 replace(u' |br| *[', u'*[', 1)
944 self._data[u"suites"][suite.longname.lower().
946 replace(u" ", u"_")] = {
947 u"name": suite.name.lower(),
949 u"parent": parent_name,
950 u"level": len(suite.longname.split(u"."))
953 suite.keywords.visit(self)
955 def end_suite(self, suite):
956 """Called when suite ends.
958 :param suite: Suite to process.
963 def visit_test(self, test):
964 """Implements traversing through the test.
966 :param test: Test to process.
970 if self.start_test(test) is not False:
971 test.keywords.visit(self)
974 def start_test(self, test):
975 """Called when test starts.
977 :param test: Test to process.
982 self._sh_run_counter = 0
984 longname_orig = test.longname.lower()
986 # Check the ignore list
987 if longname_orig in self._ignore:
990 tags = [str(tag) for tag in test.tags]
993 # Change the TC long name and name if defined in the mapping table
994 longname = self._mapping.get(longname_orig, None)
995 if longname is not None:
996 name = longname.split(u'.')[-1]
998 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1002 longname = longname_orig
1003 name = test.name.lower()
1005 # Remove TC number from the TC long name (backward compatibility):
1006 self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1007 # Remove TC number from the TC name (not needed):
1008 test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1010 test_result[u"parent"] = test.parent.name.lower()
1011 test_result[u"tags"] = tags
1012 test_result["doc"] = test.doc.\
1013 replace(u'"', u"'").\
1014 replace(u'\n', u' ').\
1015 replace(u'\r', u'').\
1016 replace(u'[', u' |br| [').\
1017 replace(u' |br| [', u'[', 1)
1018 test_result[u"type"] = u"FUNC"
1019 test_result[u"status"] = test.status
1021 if test.status == u"PASS":
1022 if u"NDRPDR" in tags:
1023 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1024 test.message).replace(u'\n', u' |br| ').\
1025 replace(u'\r', u'').replace(u'"', u"'")
1026 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1027 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1028 test.message).replace(u'\n', u' |br| ').\
1029 replace(u'\r', u'').replace(u'"', u"'")
1031 test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\
1032 replace(u'\r', u'').replace(u'"', u"'")
1034 test_result[u"msg"] = u"Test Failed."
1036 if u"PERFTEST" in tags:
1037 # Replace info about cores (e.g. -1c-) with the info about threads
1038 # and cores (e.g. -1t1c-) in the long test case names and in the
1039 # test case names if necessary.
1040 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1044 for tag in test_result[u"tags"]:
1045 groups = re.search(self.REGEX_TC_TAG, tag)
1051 self._test_id = re.sub(
1052 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1053 self._test_id, count=1
1055 test_result[u"name"] = re.sub(
1056 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1057 test_result["name"], count=1
1060 test_result[u"status"] = u"FAIL"
1061 self._data[u"tests"][self._test_id] = test_result
1063 f"The test {self._test_id} has no or more than one "
1064 f"multi-threading tags.\n"
1065 f"Tags: {test_result[u'tags']}"
1069 if test.status == u"PASS":
1070 if u"NDRPDR" in tags:
1071 test_result[u"type"] = u"NDRPDR"
1072 test_result[u"throughput"], test_result[u"status"] = \
1073 self._get_ndrpdr_throughput(test.message)
1074 test_result[u"latency"], test_result[u"status"] = \
1075 self._get_ndrpdr_latency(test.message)
1076 elif u"SOAK" in tags:
1077 test_result[u"type"] = u"SOAK"
1078 test_result[u"throughput"], test_result[u"status"] = \
1079 self._get_plr_throughput(test.message)
1080 elif u"HOSTSTACK" in tags:
1081 test_result[u"type"] = u"HOSTSTACK"
1082 test_result[u"result"], test_result[u"status"] = \
1083 self._get_hoststack_data(test.message, tags)
1084 elif u"TCP" in tags:
1085 test_result[u"type"] = u"TCP"
1086 groups = re.search(self.REGEX_TCP, test.message)
1087 test_result[u"result"] = int(groups.group(2))
1088 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1090 test_result[u"type"] = u"MRR"
1092 test_result[u"type"] = u"BMRR"
1094 test_result[u"result"] = dict()
1095 groups = re.search(self.REGEX_BMRR, test.message)
1096 if groups is not None:
1097 items_str = groups.group(1)
1098 items_float = [float(item.strip()) for item
1099 in items_str.split(",")]
1100 # Use whole list in CSIT-1180.
1101 stats = jumpavg.AvgStdevStats.for_runs(items_float)
1102 test_result[u"result"][u"receive-rate"] = stats.avg
1103 test_result[u"result"][u"receive-stdev"] = stats.stdev
1105 groups = re.search(self.REGEX_MRR, test.message)
1106 test_result[u"result"][u"receive-rate"] = \
1107 float(groups.group(3)) / float(groups.group(1))
1108 elif u"RECONF" in tags:
1109 test_result[u"type"] = u"RECONF"
1110 test_result[u"result"] = None
1112 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1113 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1114 test_result[u"result"] = {
1115 u"loss": int(grps_loss.group(1)),
1116 u"time": float(grps_time.group(1))
1118 except (AttributeError, IndexError, ValueError, TypeError):
1119 test_result[u"status"] = u"FAIL"
1120 elif u"DEVICETEST" in tags:
1121 test_result[u"type"] = u"DEVICETEST"
1123 test_result[u"status"] = u"FAIL"
1124 self._data[u"tests"][self._test_id] = test_result
1127 self._data[u"tests"][self._test_id] = test_result
1129 def end_test(self, test):
1130 """Called when test ends.
1132 :param test: Test to process.
1137 def visit_keyword(self, keyword):
1138 """Implements traversing through the keyword and its child keywords.
1140 :param keyword: Keyword to process.
1141 :type keyword: Keyword
1144 if self.start_keyword(keyword) is not False:
1145 self.end_keyword(keyword)
1147 def start_keyword(self, keyword):
1148 """Called when keyword starts. Default implementation does nothing.
1150 :param keyword: Keyword to process.
1151 :type keyword: Keyword
1155 if keyword.type == u"setup":
1156 self.visit_setup_kw(keyword)
1157 elif keyword.type == u"teardown":
1158 self.visit_teardown_kw(keyword)
1160 self.visit_test_kw(keyword)
1161 except AttributeError:
1164 def end_keyword(self, keyword):
1165 """Called when keyword ends. Default implementation does nothing.
1167 :param keyword: Keyword to process.
1168 :type keyword: Keyword
1172 def visit_test_kw(self, test_kw):
1173 """Implements traversing through the test keyword and its child
1176 :param test_kw: Keyword to process.
1177 :type test_kw: Keyword
1180 for keyword in test_kw.keywords:
1181 if self.start_test_kw(keyword) is not False:
1182 self.visit_test_kw(keyword)
1183 self.end_test_kw(keyword)
1185 def start_test_kw(self, test_kw):
1186 """Called when test keyword starts. Default implementation does
1189 :param test_kw: Keyword to process.
1190 :type test_kw: Keyword
1193 if test_kw.name.count(u"Show Runtime On All Duts") or \
1194 test_kw.name.count(u"Show Runtime Counters On All Duts"):
1195 self._msg_type = u"test-show-runtime"
1196 self._sh_run_counter += 1
1197 elif test_kw.name.count(u"Install Dpdk Test On All Duts") and \
1199 self._msg_type = u"dpdk-version"
1202 test_kw.messages.visit(self)
1204 def end_test_kw(self, test_kw):
1205 """Called when keyword ends. Default implementation does nothing.
1207 :param test_kw: Keyword to process.
1208 :type test_kw: Keyword
1212 def visit_setup_kw(self, setup_kw):
1213 """Implements traversing through the teardown keyword and its child
1216 :param setup_kw: Keyword to process.
1217 :type setup_kw: Keyword
1220 for keyword in setup_kw.keywords:
1221 if self.start_setup_kw(keyword) is not False:
1222 self.visit_setup_kw(keyword)
1223 self.end_setup_kw(keyword)
1225 def start_setup_kw(self, setup_kw):
1226 """Called when teardown keyword starts. Default implementation does
1229 :param setup_kw: Keyword to process.
1230 :type setup_kw: Keyword
1233 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1234 and not self._version:
1235 self._msg_type = u"vpp-version"
1236 elif setup_kw.name.count(u"Set Global Variable") \
1237 and not self._timestamp:
1238 self._msg_type = u"timestamp"
1239 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1240 self._msg_type = u"testbed"
1243 setup_kw.messages.visit(self)
1245 def end_setup_kw(self, setup_kw):
1246 """Called when keyword ends. Default implementation does nothing.
1248 :param setup_kw: Keyword to process.
1249 :type setup_kw: Keyword
1253 def visit_teardown_kw(self, teardown_kw):
1254 """Implements traversing through the teardown keyword and its child
1257 :param teardown_kw: Keyword to process.
1258 :type teardown_kw: Keyword
1261 for keyword in teardown_kw.keywords:
1262 if self.start_teardown_kw(keyword) is not False:
1263 self.visit_teardown_kw(keyword)
1264 self.end_teardown_kw(keyword)
1266 def start_teardown_kw(self, teardown_kw):
1267 """Called when teardown keyword starts
1269 :param teardown_kw: Keyword to process.
1270 :type teardown_kw: Keyword
1274 if teardown_kw.name.count(u"Show Vat History On All Duts"):
1275 # TODO: Remove when not needed:
1276 self._conf_history_lookup_nr = 0
1277 self._msg_type = u"teardown-vat-history"
1278 teardown_kw.messages.visit(self)
1279 elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1280 self._conf_history_lookup_nr = 0
1281 self._msg_type = u"teardown-papi-history"
1282 teardown_kw.messages.visit(self)
1284 def end_teardown_kw(self, teardown_kw):
1285 """Called when keyword ends. Default implementation does nothing.
1287 :param teardown_kw: Keyword to process.
1288 :type teardown_kw: Keyword
1292 def visit_message(self, msg):
1293 """Implements visiting the message.
1295 :param msg: Message to process.
1299 if self.start_message(msg) is not False:
1300 self.end_message(msg)
1302 def start_message(self, msg):
1303 """Called when message starts. Get required information from messages:
1306 :param msg: Message to process.
1311 self.parse_msg[self._msg_type](msg)
1313 def end_message(self, msg):
1314 """Called when message ends. Default implementation does nothing.
1316 :param msg: Message to process.
1325 The data is extracted from output.xml files generated by Jenkins jobs and
1326 stored in pandas' DataFrames.
1332 (as described in ExecutionChecker documentation)
1334 (as described in ExecutionChecker documentation)
1336 (as described in ExecutionChecker documentation)
1339 def __init__(self, spec):
1342 :param spec: Specification.
1343 :type spec: Specification
1350 self._input_data = pd.Series()
1354 """Getter - Input data.
1356 :returns: Input data
1357 :rtype: pandas.Series
1359 return self._input_data
1361 def metadata(self, job, build):
1362 """Getter - metadata
1364 :param job: Job which metadata we want.
1365 :param build: Build which metadata we want.
1369 :rtype: pandas.Series
1371 return self.data[job][build][u"metadata"]
1373 def suites(self, job, build):
1376 :param job: Job which suites we want.
1377 :param build: Build which suites we want.
1381 :rtype: pandas.Series
1383 return self.data[job][str(build)][u"suites"]
1385 def tests(self, job, build):
1388 :param job: Job which tests we want.
1389 :param build: Build which tests we want.
1393 :rtype: pandas.Series
1395 return self.data[job][build][u"tests"]
1397 def _parse_tests(self, job, build):
1398 """Process data from robot output.xml file and return JSON structured
1401 :param job: The name of job which build output data will be processed.
1402 :param build: The build which output data will be processed.
1405 :returns: JSON data structure.
1414 with open(build[u"file-name"], u'r') as data_file:
1416 result = ExecutionResult(data_file)
1417 except errors.DataError as err:
1419 f"Error occurred while parsing output.xml: {repr(err)}"
1422 checker = ExecutionChecker(metadata, self._cfg.mapping,
1424 result.visit(checker)
1428 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1429 """Download and parse the input data file.
1431 :param pid: PID of the process executing this method.
1432 :param job: Name of the Jenkins job which generated the processed input
1434 :param build: Information about the Jenkins build which generated the
1435 processed input file.
1436 :param repeat: Repeat the download specified number of times if not
1444 logging.info(f" Processing the job/build: {job}: {build[u'build']}")
1451 success = download_and_unzip_data_file(self._cfg, job, build, pid)
1457 f"It is not possible to download the input data file from the "
1458 f"job {job}, build {build[u'build']}, or it is damaged. "
1462 logging.info(f" Processing data from build {build[u'build']}")
1463 data = self._parse_tests(job, build)
1466 f"Input data file from the job {job}, build "
1467 f"{build[u'build']} is damaged. Skipped."
1470 state = u"processed"
1473 remove(build[u"file-name"])
1474 except OSError as err:
1476 f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1479 # If the time-period is defined in the specification file, remove all
1480 # files which are outside the time period.
1482 timeperiod = self._cfg.input.get(u"time-period", None)
1483 if timeperiod and data:
1485 timeperiod = timedelta(int(timeperiod))
1486 metadata = data.get(u"metadata", None)
1488 generated = metadata.get(u"generated", None)
1490 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1491 if (now - generated) > timeperiod:
1492 # Remove the data and the file:
1497 f" The build {job}/{build[u'build']} is "
1498 f"outdated, will be removed."
1500 logging.info(u" Done.")
1510 def download_and_parse_data(self, repeat=1):
1511 """Download the input data files, parse input data from input files and
1512 store in pandas' Series.
1514 :param repeat: Repeat the download specified number of times if not
1519 logging.info(u"Downloading and parsing input files ...")
1521 for job, builds in self._cfg.builds.items():
1522 for build in builds:
1524 result = self._download_and_parse_build(job, build, repeat)
1527 build_nr = result[u"build"][u"build"]
1530 data = result[u"data"]
1531 build_data = pd.Series({
1532 u"metadata": pd.Series(
1533 list(data[u"metadata"].values()),
1534 index=list(data[u"metadata"].keys())
1536 u"suites": pd.Series(
1537 list(data[u"suites"].values()),
1538 index=list(data[u"suites"].keys())
1540 u"tests": pd.Series(
1541 list(data[u"tests"].values()),
1542 index=list(data[u"tests"].keys())
1546 if self._input_data.get(job, None) is None:
1547 self._input_data[job] = pd.Series()
1548 self._input_data[job][str(build_nr)] = build_data
1550 self._cfg.set_input_file_name(
1551 job, build_nr, result[u"build"][u"file-name"])
1553 self._cfg.set_input_state(job, build_nr, result[u"state"])
1556 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1557 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1559 logging.info(u"Done.")
1561 def process_local_file(self, local_file, job=u"local", build_nr=1,
1563 """Process local XML file given as a command-line parameter.
1565 :param local_file: The file to process.
1566 :param job: Job name.
1567 :param build_nr: Build number.
1568 :param replace: If True, the information about jobs and builds is
1569 replaced by the new one, otherwise the new jobs and builds are
1571 :type local_file: str
1575 :raises: PresentationError if an error occurs.
1577 if not isfile(local_file):
1578 raise PresentationError(f"The file {local_file} does not exist.")
1581 build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1582 except (IndexError, ValueError):
1587 u"status": u"failed",
1588 u"file-name": local_file
1591 self._cfg.builds = dict()
1592 self._cfg.add_build(job, build)
1594 logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1595 data = self._parse_tests(job, build)
1597 raise PresentationError(
1598 f"Error occurred while parsing the file {local_file}"
1601 build_data = pd.Series({
1602 u"metadata": pd.Series(
1603 list(data[u"metadata"].values()),
1604 index=list(data[u"metadata"].keys())
1606 u"suites": pd.Series(
1607 list(data[u"suites"].values()),
1608 index=list(data[u"suites"].keys())
1610 u"tests": pd.Series(
1611 list(data[u"tests"].values()),
1612 index=list(data[u"tests"].keys())
1616 if self._input_data.get(job, None) is None:
1617 self._input_data[job] = pd.Series()
1618 self._input_data[job][str(build_nr)] = build_data
1620 self._cfg.set_input_state(job, build_nr, u"processed")
1622 def process_local_directory(self, local_dir, replace=True):
1623 """Process local directory with XML file(s). The directory is processed
1624 as a 'job' and the XML files in it as builds.
1625 If the given directory contains only sub-directories, these
1626 sub-directories processed as jobs and corresponding XML files as builds
1629 :param local_dir: Local directory to process.
1630 :param replace: If True, the information about jobs and builds is
1631 replaced by the new one, otherwise the new jobs and builds are
1633 :type local_dir: str
1636 if not isdir(local_dir):
1637 raise PresentationError(
1638 f"The directory {local_dir} does not exist."
1641 # Check if the given directory includes only files, or only directories
1642 _, dirnames, filenames = next(walk(local_dir))
1644 if filenames and not dirnames:
1647 # key: dir (job) name, value: list of file names (builds)
1649 local_dir: [join(local_dir, name) for name in filenames]
1652 elif dirnames and not filenames:
1655 # key: dir (job) name, value: list of file names (builds)
1656 local_builds = dict()
1657 for dirname in dirnames:
1659 join(local_dir, dirname, name)
1660 for name in listdir(join(local_dir, dirname))
1661 if isfile(join(local_dir, dirname, name))
1664 local_builds[dirname] = sorted(builds)
1666 elif not filenames and not dirnames:
1667 raise PresentationError(f"The directory {local_dir} is empty.")
1669 raise PresentationError(
1670 f"The directory {local_dir} can include only files or only "
1671 f"directories, not both.\nThe directory {local_dir} includes "
1672 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1676 self._cfg.builds = dict()
1678 for job, files in local_builds.items():
1679 for idx, local_file in enumerate(files):
1680 self.process_local_file(local_file, job, idx + 1, replace=False)
1683 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1684 """Return the index of character in the string which is the end of tag.
1686 :param tag_filter: The string where the end of tag is being searched.
1687 :param start: The index where the searching is stated.
1688 :param closer: The character which is the tag closer.
1689 :type tag_filter: str
1692 :returns: The index of the tag closer.
1696 idx_opener = tag_filter.index(closer, start)
1697 return tag_filter.index(closer, idx_opener + 1)
1702 def _condition(tag_filter):
1703 """Create a conditional statement from the given tag filter.
1705 :param tag_filter: Filter based on tags from the element specification.
1706 :type tag_filter: str
1707 :returns: Conditional statement which can be evaluated.
1712 index = InputData._end_of_tag(tag_filter, index)
1716 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1718 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1719 continue_on_error=False):
1720 """Filter required data from the given jobs and builds.
1722 The output data structure is:
1725 - test (or suite) 1 ID:
1731 - test (or suite) n ID:
1738 :param element: Element which will use the filtered data.
1739 :param params: Parameters which will be included in the output. If None,
1740 all parameters are included.
1741 :param data: If not None, this data is used instead of data specified
1743 :param data_set: The set of data to be filtered: tests, suites,
1745 :param continue_on_error: Continue if there is error while reading the
1746 data. The Item will be empty then
1747 :type element: pandas.Series
1751 :type continue_on_error: bool
1752 :returns: Filtered data.
1753 :rtype pandas.Series
1757 if data_set == "suites":
1759 elif element[u"filter"] in (u"all", u"template"):
1762 cond = InputData._condition(element[u"filter"])
1763 logging.debug(f" Filter: {cond}")
1765 logging.error(u" No filter defined.")
1769 params = element.get(u"parameters", None)
1771 params.append(u"type")
1773 data_to_filter = data if data else element[u"data"]
1776 for job, builds in data_to_filter.items():
1777 data[job] = pd.Series()
1778 for build in builds:
1779 data[job][str(build)] = pd.Series()
1782 self.data[job][str(build)][data_set].items())
1784 if continue_on_error:
1788 for test_id, test_data in data_dict.items():
1789 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1790 data[job][str(build)][test_id] = pd.Series()
1792 for param, val in test_data.items():
1793 data[job][str(build)][test_id][param] = val
1795 for param in params:
1797 data[job][str(build)][test_id][param] =\
1800 data[job][str(build)][test_id][param] =\
1804 except (KeyError, IndexError, ValueError) as err:
1806 f"Missing mandatory parameter in the element specification: "
1810 except AttributeError as err:
1811 logging.error(repr(err))
1813 except SyntaxError as err:
1815 f"The filter {cond} is not correct. Check if all tags are "
1816 f"enclosed by apostrophes.\n{repr(err)}"
1820 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1821 continue_on_error=False):
1822 """Filter required data from the given jobs and builds.
1824 The output data structure is:
1827 - test (or suite) 1 ID:
1833 - test (or suite) n ID:
1840 :param element: Element which will use the filtered data.
1841 :param params: Parameters which will be included in the output. If None,
1842 all parameters are included.
1843 :param data_set: The set of data to be filtered: tests, suites,
1845 :param continue_on_error: Continue if there is error while reading the
1846 data. The Item will be empty then
1847 :type element: pandas.Series
1850 :type continue_on_error: bool
1851 :returns: Filtered data.
1852 :rtype pandas.Series
1855 include = element.get(u"include", None)
1857 logging.warning(u"No tests to include, skipping the element.")
1861 params = element.get(u"parameters", None)
1863 params.append(u"type")
1867 for job, builds in element[u"data"].items():
1868 data[job] = pd.Series()
1869 for build in builds:
1870 data[job][str(build)] = pd.Series()
1871 for test in include:
1873 reg_ex = re.compile(str(test).lower())
1874 for test_id in self.data[job][
1875 str(build)][data_set].keys():
1876 if re.match(reg_ex, str(test_id).lower()):
1877 test_data = self.data[job][
1878 str(build)][data_set][test_id]
1879 data[job][str(build)][test_id] = pd.Series()
1881 for param, val in test_data.items():
1882 data[job][str(build)][test_id]\
1885 for param in params:
1887 data[job][str(build)][
1891 data[job][str(build)][
1892 test_id][param] = u"No Data"
1893 except KeyError as err:
1894 if continue_on_error:
1895 logging.debug(repr(err))
1897 logging.error(repr(err))
1901 except (KeyError, IndexError, ValueError) as err:
1903 f"Missing mandatory parameter in the element "
1904 f"specification: {repr(err)}"
1907 except AttributeError as err:
1908 logging.error(repr(err))
1912 def merge_data(data):
1913 """Merge data from more jobs and builds to a simple data structure.
1915 The output data structure is:
1917 - test (suite) 1 ID:
1923 - test (suite) n ID:
1926 :param data: Data to merge.
1927 :type data: pandas.Series
1928 :returns: Merged data.
1929 :rtype: pandas.Series
1932 logging.info(u" Merging data ...")
1934 merged_data = pd.Series()
1935 for builds in data.values:
1936 for item in builds.values:
1937 for item_id, item_data in item.items():
1938 merged_data[item_id] = item_data
1941 def print_all_oper_data(self):
1942 """Print all operational data to console.
1950 u"Cycles per Packet",
1951 u"Average Vector Size"
1954 for job in self._input_data.values:
1955 for build in job.values:
1956 for test_id, test_data in build[u"tests"].items():
1958 if test_data.get(u"show-run", None) is None:
1960 for dut_name, data in test_data[u"show-run"].items():
1961 if data.get(u"threads", None) is None:
1963 print(f"Host IP: {data.get(u'host', '')}, "
1964 f"Socket: {data.get(u'socket', '')}")
1965 for thread_nr, thread in data[u"threads"].items():
1966 txt_table = prettytable.PrettyTable(tbl_hdr)
1969 txt_table.add_row(row)
1971 if len(thread) == 0:
1974 avg = f", Average Vector Size per Node: " \
1975 f"{(avg / len(thread)):.2f}"
1976 th_name = u"main" if thread_nr == 0 \
1977 else f"worker_{thread_nr}"
1978 print(f"{dut_name}, {th_name}{avg}")
1979 txt_table.float_format = u".2"
1980 txt_table.align = u"r"
1981 txt_table.align[u"Name"] = u"l"
1982 print(f"{txt_table.get_string()}\n")