1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Data pre-processing
16 - extract data from output.xml files generated by Jenkins jobs and store in
18 - provide access to the data.
19 - filter the data using tags,
27 from collections import OrderedDict
28 from os import remove, walk, listdir
29 from os.path import isfile, isdir, join
30 from datetime import datetime as dt
31 from datetime import timedelta
32 from json import loads
33 from json.decoder import JSONDecodeError
40 from robot.api import ExecutionResult, ResultVisitor
41 from robot import errors
43 from resources.libraries.python import jumpavg
44 from input_data_files import download_and_unzip_data_file
45 from pal_errors import PresentationError
48 # Separator used in file names
52 class ExecutionChecker(ResultVisitor):
53 """Class to traverse through the test suite structure.
55 The functionality implemented in this class generates a json structure:
61 "generated": "Timestamp",
62 "version": "SUT version",
63 "job": "Jenkins job name",
64 "build": "Information about the build"
67 "Suite long name 1": {
69 "doc": "Suite 1 documentation",
70 "parent": "Suite 1 parent",
71 "level": "Level of the suite in the suite hierarchy"
73 "Suite long name N": {
75 "doc": "Suite N documentation",
76 "parent": "Suite 2 parent",
77 "level": "Level of the suite in the suite hierarchy"
84 "parent": "Name of the parent of the test",
85 "doc": "Test documentation",
86 "msg": "Test message",
87 "conf-history": "DUT1 and DUT2 VAT History",
88 "show-run": "Show Run",
89 "tags": ["tag 1", "tag 2", "tag n"],
91 "status": "PASS" | "FAIL",
137 "parent": "Name of the parent of the test",
138 "doc": "Test documentation",
139 "msg": "Test message",
140 "tags": ["tag 1", "tag 2", "tag n"],
142 "status": "PASS" | "FAIL",
149 "parent": "Name of the parent of the test",
150 "doc": "Test documentation",
151 "msg": "Test message",
152 "tags": ["tag 1", "tag 2", "tag n"],
153 "type": "MRR" | "BMRR",
154 "status": "PASS" | "FAIL",
156 "receive-rate": float,
157 # Average of a list, computed using AvgStdevStats.
158 # In CSIT-1180, replace with List[float].
172 "metadata": { # Optional
173 "version": "VPP version",
174 "job": "Jenkins job name",
175 "build": "Information about the build"
179 "doc": "Suite 1 documentation",
180 "parent": "Suite 1 parent",
181 "level": "Level of the suite in the suite hierarchy"
184 "doc": "Suite N documentation",
185 "parent": "Suite 2 parent",
186 "level": "Level of the suite in the suite hierarchy"
192 "parent": "Name of the parent of the test",
193 "doc": "Test documentation"
194 "msg": "Test message"
195 "tags": ["tag 1", "tag 2", "tag n"],
196 "conf-history": "DUT1 and DUT2 VAT History"
197 "show-run": "Show Run"
198 "status": "PASS" | "FAIL"
206 .. note:: ID is the lowercase full path to the test.
209 REGEX_PLR_RATE = re.compile(
210 r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
211 r'PLRsearch upper bound::?\s(\d+.\d+)'
213 REGEX_NDRPDR_RATE = re.compile(
214 r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
215 r'NDR_UPPER:\s(\d+.\d+).*\n'
216 r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
217 r'PDR_UPPER:\s(\d+.\d+)'
219 REGEX_PERF_MSG_INFO = re.compile(
220 r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
221 r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
222 r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
223 r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
224 r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
226 REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
228 # TODO: Remove when not needed
229 REGEX_NDRPDR_LAT_BASE = re.compile(
230 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
231 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
233 REGEX_NDRPDR_LAT = re.compile(
234 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
235 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
236 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
237 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
238 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
239 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
241 # TODO: Remove when not needed
242 REGEX_NDRPDR_LAT_LONG = re.compile(
243 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
244 r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
245 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
246 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
247 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
248 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
249 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
250 r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
251 r'Latency.*\[\'(.*)\', \'(.*)\'\]'
253 REGEX_VERSION_VPP = re.compile(
254 r"(return STDOUT Version:\s*|"
255 r"VPP Version:\s*|VPP version:\s*)(.*)"
257 REGEX_VERSION_DPDK = re.compile(
258 r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
260 REGEX_TCP = re.compile(
261 r'Total\s(rps|cps|throughput):\s(\d*).*$'
263 REGEX_MRR = re.compile(
264 r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
265 r'tx\s(\d*),\srx\s(\d*)'
267 REGEX_BMRR = re.compile(
268 r'Maximum Receive Rate trial results'
269 r' in packets per second: \[(.*)\]'
271 REGEX_RECONF_LOSS = re.compile(
272 r'Packets lost due to reconfig: (\d*)'
274 REGEX_RECONF_TIME = re.compile(
275 r'Implied time lost: (\d*.[\de-]*)'
277 REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
279 REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
281 REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
283 REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
285 REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
287 def __init__(self, metadata, mapping, ignore):
290 :param metadata: Key-value pairs to be included in "metadata" part of
292 :param mapping: Mapping of the old names of test cases to the new
294 :param ignore: List of TCs to be ignored.
300 # Type of message to parse out from the test messages
301 self._msg_type = None
307 self._timestamp = None
309 # Testbed. The testbed is identified by TG node IP address.
312 # Mapping of TCs long names
313 self._mapping = mapping
316 self._ignore = ignore
318 # Number of PAPI History messages found:
320 # 1 - PAPI History of DUT1
321 # 2 - PAPI History of DUT2
322 self._conf_history_lookup_nr = 0
324 self._sh_run_counter = 0
326 # Test ID of currently processed test- the lowercase full path to the
330 # The main data structure
332 u"metadata": OrderedDict(),
333 u"suites": OrderedDict(),
334 u"tests": OrderedDict()
337 # Save the provided metadata
338 for key, val in metadata.items():
339 self._data[u"metadata"][key] = val
341 # Dictionary defining the methods used to parse different types of
344 u"timestamp": self._get_timestamp,
345 u"vpp-version": self._get_vpp_version,
346 u"dpdk-version": self._get_dpdk_version,
347 # TODO: Remove when not needed:
348 u"teardown-vat-history": self._get_vat_history,
349 u"teardown-papi-history": self._get_papi_history,
350 u"test-show-runtime": self._get_show_run,
351 u"testbed": self._get_testbed
356 """Getter - Data parsed from the XML file.
358 :returns: Data parsed from the XML file.
363 def _get_data_from_mrr_test_msg(self, msg):
364 """Get info from message of MRR performance tests.
366 :param msg: Message to be processed.
368 :returns: Processed message or original message if a problem occurs.
372 groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
373 if not groups or groups.lastindex != 1:
374 return u"Test Failed."
377 data = groups.group(1).split(u", ")
378 except (AttributeError, IndexError, ValueError, KeyError):
379 return u"Test Failed."
384 out_str += f"{(float(item) / 1e6):.2f}, "
385 return out_str[:-2] + u"]"
386 except (AttributeError, IndexError, ValueError, KeyError):
387 return u"Test Failed."
389 def _get_data_from_perf_test_msg(self, msg):
390 """Get info from message of NDRPDR performance tests.
392 :param msg: Message to be processed.
394 :returns: Processed message or original message if a problem occurs.
398 groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
399 if not groups or groups.lastindex != 10:
400 return u"Test Failed."
404 u"ndr_low": float(groups.group(1)),
405 u"ndr_low_b": float(groups.group(2)),
406 u"pdr_low": float(groups.group(3)),
407 u"pdr_low_b": float(groups.group(4)),
408 u"pdr_lat_90_1": groups.group(5),
409 u"pdr_lat_90_2": groups.group(6),
410 u"pdr_lat_50_1": groups.group(7),
411 u"pdr_lat_50_2": groups.group(8),
412 u"pdr_lat_10_1": groups.group(9),
413 u"pdr_lat_10_2": groups.group(10),
415 except (AttributeError, IndexError, ValueError, KeyError):
416 return u"Test Failed."
418 def _process_lat(in_str_1, in_str_2):
419 """Extract min, avg, max values from latency string.
421 :param in_str_1: Latency string for one direction produced by robot
423 :param in_str_2: Latency string for second direction produced by
427 :returns: Processed latency string or None if a problem occurs.
430 in_list_1 = in_str_1.split('/', 3)
431 in_list_2 = in_str_2.split('/', 3)
433 if len(in_list_1) != 4 and len(in_list_2) != 4:
436 in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
438 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
439 except hdrh.codec.HdrLengthException:
442 in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
444 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
445 except hdrh.codec.HdrLengthException:
448 if hdr_lat_1 and hdr_lat_2:
450 hdr_lat_1.get_value_at_percentile(50.0),
451 hdr_lat_1.get_value_at_percentile(90.0),
452 hdr_lat_1.get_value_at_percentile(99.0),
453 hdr_lat_2.get_value_at_percentile(50.0),
454 hdr_lat_2.get_value_at_percentile(90.0),
455 hdr_lat_2.get_value_at_percentile(99.0)
465 f"1. {(data[u'ndr_low'] / 1e6):5.2f} "
466 f"{data[u'ndr_low_b']:5.2f}"
467 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f} "
468 f"{data[u'pdr_low_b']:5.2f}"
471 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
472 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
473 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
476 max_len = len(str(max((max(item) for item in latency))))
477 max_len = 4 if max_len < 4 else max_len
479 for idx, lat in enumerate(latency):
484 f"{lat[0]:{max_len}d} "
485 f"{lat[1]:{max_len}d} "
486 f"{lat[2]:{max_len}d} "
487 f"{lat[3]:{max_len}d} "
488 f"{lat[4]:{max_len}d} "
489 f"{lat[5]:{max_len}d} "
494 except (AttributeError, IndexError, ValueError, KeyError):
495 return u"Test Failed."
497 def _get_testbed(self, msg):
498 """Called when extraction of testbed IP is required.
499 The testbed is identified by TG node IP address.
501 :param msg: Message to process.
506 if msg.message.count(u"Setup of TG node") or \
507 msg.message.count(u"Setup of node TG host"):
508 reg_tg_ip = re.compile(
509 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
511 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
512 except (KeyError, ValueError, IndexError, AttributeError):
515 self._data[u"metadata"][u"testbed"] = self._testbed
516 self._msg_type = None
518 def _get_vpp_version(self, msg):
519 """Called when extraction of VPP version is required.
521 :param msg: Message to process.
526 if msg.message.count(u"return STDOUT Version:") or \
527 msg.message.count(u"VPP Version:") or \
528 msg.message.count(u"VPP version:"):
529 self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
531 self._data[u"metadata"][u"version"] = self._version
532 self._msg_type = None
534 def _get_dpdk_version(self, msg):
535 """Called when extraction of DPDK version is required.
537 :param msg: Message to process.
542 if msg.message.count(u"DPDK Version:"):
544 self._version = str(re.search(
545 self.REGEX_VERSION_DPDK, msg.message).group(2))
546 self._data[u"metadata"][u"version"] = self._version
550 self._msg_type = None
552 def _get_timestamp(self, msg):
553 """Called when extraction of timestamp is required.
555 :param msg: Message to process.
560 self._timestamp = msg.timestamp[:14]
561 self._data[u"metadata"][u"generated"] = self._timestamp
562 self._msg_type = None
564 def _get_vat_history(self, msg):
565 """Called when extraction of VAT command history is required.
567 TODO: Remove when not needed.
569 :param msg: Message to process.
573 if msg.message.count(u"VAT command history:"):
574 self._conf_history_lookup_nr += 1
575 if self._conf_history_lookup_nr == 1:
576 self._data[u"tests"][self._test_id][u"conf-history"] = str()
578 self._msg_type = None
579 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
580 r"VAT command history:", u"",
581 msg.message, count=1).replace(u'\n', u' |br| ').\
584 self._data[u"tests"][self._test_id][u"conf-history"] += (
585 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
588 def _get_papi_history(self, msg):
589 """Called when extraction of PAPI command history is required.
591 :param msg: Message to process.
595 if msg.message.count(u"PAPI command history:"):
596 self._conf_history_lookup_nr += 1
597 if self._conf_history_lookup_nr == 1:
598 self._data[u"tests"][self._test_id][u"conf-history"] = str()
600 self._msg_type = None
601 text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
602 r"PAPI command history:", u"",
603 msg.message, count=1).replace(u'\n', u' |br| ').\
605 self._data[u"tests"][self._test_id][u"conf-history"] += (
606 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
609 def _get_show_run(self, msg):
610 """Called when extraction of VPP operational data (output of CLI command
611 Show Runtime) is required.
613 :param msg: Message to process.
618 if not msg.message.count(u"stats runtime"):
622 if self._sh_run_counter > 1:
625 if u"show-run" not in self._data[u"tests"][self._test_id].keys():
626 self._data[u"tests"][self._test_id][u"show-run"] = dict()
628 groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
632 host = groups.group(1)
633 except (AttributeError, IndexError):
636 sock = groups.group(2)
637 except (AttributeError, IndexError):
640 runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
641 replace(u"'", u'"').replace(u'b"', u'"').
642 replace(u'u"', u'"').split(u":", 1)[1])
645 threads_nr = len(runtime[0][u"clocks"])
646 except (IndexError, KeyError):
649 dut = u"DUT{nr}".format(
650 nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
655 u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
659 for idx in range(threads_nr):
660 if item[u"vectors"][idx] > 0:
661 clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
662 elif item[u"calls"][idx] > 0:
663 clocks = item[u"clocks"][idx] / item[u"calls"][idx]
664 elif item[u"suspends"][idx] > 0:
665 clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
669 if item[u"calls"][idx] > 0:
670 vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
674 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
675 int(item[u"suspends"][idx]):
676 oper[u"threads"][idx].append([
679 item[u"vectors"][idx],
680 item[u"suspends"][idx],
685 self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
687 def _get_ndrpdr_throughput(self, msg):
688 """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
691 :param msg: The test message to be parsed.
693 :returns: Parsed data as a dict and the status (PASS/FAIL).
694 :rtype: tuple(dict, str)
698 u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
699 u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
702 groups = re.search(self.REGEX_NDRPDR_RATE, msg)
704 if groups is not None:
706 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
707 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
708 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
709 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
711 except (IndexError, ValueError):
714 return throughput, status
716 def _get_plr_throughput(self, msg):
717 """Get PLRsearch lower bound and PLRsearch upper bound from the test
720 :param msg: The test message to be parsed.
722 :returns: Parsed data as a dict and the status (PASS/FAIL).
723 :rtype: tuple(dict, str)
731 groups = re.search(self.REGEX_PLR_RATE, msg)
733 if groups is not None:
735 throughput[u"LOWER"] = float(groups.group(1))
736 throughput[u"UPPER"] = float(groups.group(2))
738 except (IndexError, ValueError):
741 return throughput, status
743 def _get_ndrpdr_latency(self, msg):
744 """Get LATENCY from the test message.
746 :param msg: The test message to be parsed.
748 :returns: Parsed data as a dict and the status (PASS/FAIL).
749 :rtype: tuple(dict, str)
759 u"direction1": copy.copy(latency_default),
760 u"direction2": copy.copy(latency_default)
763 u"direction1": copy.copy(latency_default),
764 u"direction2": copy.copy(latency_default)
767 u"direction1": copy.copy(latency_default),
768 u"direction2": copy.copy(latency_default)
771 u"direction1": copy.copy(latency_default),
772 u"direction2": copy.copy(latency_default)
775 u"direction1": copy.copy(latency_default),
776 u"direction2": copy.copy(latency_default)
779 u"direction1": copy.copy(latency_default),
780 u"direction2": copy.copy(latency_default)
784 # TODO: Rewrite when long and base are not needed
785 groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
787 groups = re.search(self.REGEX_NDRPDR_LAT, msg)
789 groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
791 return latency, u"FAIL"
793 def process_latency(in_str):
794 """Return object with parsed latency values.
796 TODO: Define class for the return type.
798 :param in_str: Input string, min/avg/max/hdrh format.
800 :returns: Dict with corresponding keys, except hdrh float values.
802 :throws IndexError: If in_str does not have enough substrings.
803 :throws ValueError: If a substring does not convert to float.
805 in_list = in_str.split('/', 3)
808 u"min": float(in_list[0]),
809 u"avg": float(in_list[1]),
810 u"max": float(in_list[2]),
814 if len(in_list) == 4:
815 rval[u"hdrh"] = str(in_list[3])
820 latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
821 latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
822 latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
823 latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
824 if groups.lastindex == 4:
825 return latency, u"PASS"
826 except (IndexError, ValueError):
830 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
831 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
832 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
833 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
834 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
835 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
836 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
837 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
838 if groups.lastindex == 12:
839 return latency, u"PASS"
840 except (IndexError, ValueError):
843 # TODO: Remove when not needed
844 latency[u"NDR10"] = {
845 u"direction1": copy.copy(latency_default),
846 u"direction2": copy.copy(latency_default)
848 latency[u"NDR50"] = {
849 u"direction1": copy.copy(latency_default),
850 u"direction2": copy.copy(latency_default)
852 latency[u"NDR90"] = {
853 u"direction1": copy.copy(latency_default),
854 u"direction2": copy.copy(latency_default)
857 latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
858 latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
859 latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
860 latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
861 latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
862 latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
863 latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
864 latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
865 latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
866 latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
867 latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
868 latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
869 latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
870 latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
871 return latency, u"PASS"
872 except (IndexError, ValueError):
875 return latency, u"FAIL"
878 def _get_hoststack_data(msg, tags):
879 """Get data from the hoststack test message.
881 :param msg: The test message to be parsed.
882 :param tags: Test tags.
885 :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
886 :rtype: tuple(dict, str)
891 msg = msg.replace(u"'", u'"').replace(u" ", u"")
892 if u"LDPRELOAD" in tags:
896 except JSONDecodeError:
898 elif u"VPPECHO" in tags:
900 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
902 client=loads(msg_lst[0]),
903 server=loads(msg_lst[1])
906 except (JSONDecodeError, IndexError):
909 return result, status
911 def visit_suite(self, suite):
912 """Implements traversing through the suite and its direct children.
914 :param suite: Suite to process.
918 if self.start_suite(suite) is not False:
919 suite.suites.visit(self)
920 suite.tests.visit(self)
921 self.end_suite(suite)
923 def start_suite(self, suite):
924 """Called when suite starts.
926 :param suite: Suite to process.
932 parent_name = suite.parent.name
933 except AttributeError:
936 doc_str = suite.doc.\
937 replace(u'"', u"'").\
938 replace(u'\n', u' ').\
939 replace(u'\r', u'').\
940 replace(u'*[', u' |br| *[').\
941 replace(u"*", u"**").\
942 replace(u' |br| *[', u'*[', 1)
944 self._data[u"suites"][suite.longname.lower().
946 replace(u" ", u"_")] = {
947 u"name": suite.name.lower(),
949 u"parent": parent_name,
950 u"level": len(suite.longname.split(u"."))
953 suite.keywords.visit(self)
955 def end_suite(self, suite):
956 """Called when suite ends.
958 :param suite: Suite to process.
963 def visit_test(self, test):
964 """Implements traversing through the test.
966 :param test: Test to process.
970 if self.start_test(test) is not False:
971 test.keywords.visit(self)
974 def start_test(self, test):
975 """Called when test starts.
977 :param test: Test to process.
982 self._sh_run_counter = 0
984 longname_orig = test.longname.lower()
986 # Check the ignore list
987 if longname_orig in self._ignore:
990 tags = [str(tag) for tag in test.tags]
993 # Change the TC long name and name if defined in the mapping table
994 longname = self._mapping.get(longname_orig, None)
995 if longname is not None:
996 name = longname.split(u'.')[-1]
998 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1002 longname = longname_orig
1003 name = test.name.lower()
1005 # Remove TC number from the TC long name (backward compatibility):
1006 self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1007 # Remove TC number from the TC name (not needed):
1008 test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1010 test_result[u"parent"] = test.parent.name.lower()
1011 test_result[u"tags"] = tags
1012 test_result["doc"] = test.doc.\
1013 replace(u'"', u"'").\
1014 replace(u'\n', u' ').\
1015 replace(u'\r', u'').\
1016 replace(u'[', u' |br| [').\
1017 replace(u' |br| [', u'[', 1)
1018 test_result[u"type"] = u"FUNC"
1019 test_result[u"status"] = test.status
1021 if test.status == u"PASS":
1022 if u"NDRPDR" in tags:
1023 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1024 test.message).replace(u'\n', u' |br| ').\
1025 replace(u'\r', u'').replace(u'"', u"'")
1026 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1027 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1028 test.message).replace(u'\n', u' |br| ').\
1029 replace(u'\r', u'').replace(u'"', u"'")
1031 test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\
1032 replace(u'\r', u'').replace(u'"', u"'")
1034 test_result[u"msg"] = u"Test Failed."
1036 if u"PERFTEST" in tags:
1037 # Replace info about cores (e.g. -1c-) with the info about threads
1038 # and cores (e.g. -1t1c-) in the long test case names and in the
1039 # test case names if necessary.
1040 groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1044 for tag in test_result[u"tags"]:
1045 groups = re.search(self.REGEX_TC_TAG, tag)
1051 self._test_id = re.sub(
1052 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1053 self._test_id, count=1
1055 test_result[u"name"] = re.sub(
1056 self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1057 test_result["name"], count=1
1060 test_result[u"status"] = u"FAIL"
1061 self._data[u"tests"][self._test_id] = test_result
1063 f"The test {self._test_id} has no or more than one "
1064 f"multi-threading tags.\n"
1065 f"Tags: {test_result[u'tags']}"
1069 if test.status == u"PASS":
1070 if u"NDRPDR" in tags:
1071 test_result[u"type"] = u"NDRPDR"
1072 test_result[u"throughput"], test_result[u"status"] = \
1073 self._get_ndrpdr_throughput(test.message)
1074 test_result[u"latency"], test_result[u"status"] = \
1075 self._get_ndrpdr_latency(test.message)
1076 elif u"SOAK" in tags:
1077 test_result[u"type"] = u"SOAK"
1078 test_result[u"throughput"], test_result[u"status"] = \
1079 self._get_plr_throughput(test.message)
1080 elif u"HOSTSTACK" in tags:
1081 test_result[u"type"] = u"HOSTSTACK"
1082 test_result[u"result"], test_result[u"status"] = \
1083 self._get_hoststack_data(test.message, tags)
1084 elif u"TCP" in tags:
1085 test_result[u"type"] = u"TCP"
1086 groups = re.search(self.REGEX_TCP, test.message)
1087 test_result[u"result"] = int(groups.group(2))
1088 elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1090 test_result[u"type"] = u"MRR"
1092 test_result[u"type"] = u"BMRR"
1094 test_result[u"result"] = dict()
1095 groups = re.search(self.REGEX_BMRR, test.message)
1096 if groups is not None:
1097 items_str = groups.group(1)
1098 items_float = [float(item.strip()) for item
1099 in items_str.split(",")]
1100 # Use whole list in CSIT-1180.
1101 stats = jumpavg.AvgStdevStats.for_runs(items_float)
1102 test_result[u"result"][u"receive-rate"] = stats.avg
1104 groups = re.search(self.REGEX_MRR, test.message)
1105 test_result[u"result"][u"receive-rate"] = \
1106 float(groups.group(3)) / float(groups.group(1))
1107 elif u"RECONF" in tags:
1108 test_result[u"type"] = u"RECONF"
1109 test_result[u"result"] = None
1111 grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1112 grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1113 test_result[u"result"] = {
1114 u"loss": int(grps_loss.group(1)),
1115 u"time": float(grps_time.group(1))
1117 except (AttributeError, IndexError, ValueError, TypeError):
1118 test_result[u"status"] = u"FAIL"
1119 elif u"DEVICETEST" in tags:
1120 test_result[u"type"] = u"DEVICETEST"
1122 test_result[u"status"] = u"FAIL"
1123 self._data[u"tests"][self._test_id] = test_result
1126 self._data[u"tests"][self._test_id] = test_result
1128 def end_test(self, test):
1129 """Called when test ends.
1131 :param test: Test to process.
1136 def visit_keyword(self, keyword):
1137 """Implements traversing through the keyword and its child keywords.
1139 :param keyword: Keyword to process.
1140 :type keyword: Keyword
1143 if self.start_keyword(keyword) is not False:
1144 self.end_keyword(keyword)
1146 def start_keyword(self, keyword):
1147 """Called when keyword starts. Default implementation does nothing.
1149 :param keyword: Keyword to process.
1150 :type keyword: Keyword
1154 if keyword.type == u"setup":
1155 self.visit_setup_kw(keyword)
1156 elif keyword.type == u"teardown":
1157 self.visit_teardown_kw(keyword)
1159 self.visit_test_kw(keyword)
1160 except AttributeError:
1163 def end_keyword(self, keyword):
1164 """Called when keyword ends. Default implementation does nothing.
1166 :param keyword: Keyword to process.
1167 :type keyword: Keyword
1171 def visit_test_kw(self, test_kw):
1172 """Implements traversing through the test keyword and its child
1175 :param test_kw: Keyword to process.
1176 :type test_kw: Keyword
1179 for keyword in test_kw.keywords:
1180 if self.start_test_kw(keyword) is not False:
1181 self.visit_test_kw(keyword)
1182 self.end_test_kw(keyword)
1184 def start_test_kw(self, test_kw):
1185 """Called when test keyword starts. Default implementation does
1188 :param test_kw: Keyword to process.
1189 :type test_kw: Keyword
1192 if test_kw.name.count(u"Show Runtime On All Duts") or \
1193 test_kw.name.count(u"Show Runtime Counters On All Duts"):
1194 self._msg_type = u"test-show-runtime"
1195 self._sh_run_counter += 1
1196 elif test_kw.name.count(u"Install Dpdk Test On All Duts") and \
1198 self._msg_type = u"dpdk-version"
1201 test_kw.messages.visit(self)
1203 def end_test_kw(self, test_kw):
1204 """Called when keyword ends. Default implementation does nothing.
1206 :param test_kw: Keyword to process.
1207 :type test_kw: Keyword
1211 def visit_setup_kw(self, setup_kw):
1212 """Implements traversing through the teardown keyword and its child
1215 :param setup_kw: Keyword to process.
1216 :type setup_kw: Keyword
1219 for keyword in setup_kw.keywords:
1220 if self.start_setup_kw(keyword) is not False:
1221 self.visit_setup_kw(keyword)
1222 self.end_setup_kw(keyword)
1224 def start_setup_kw(self, setup_kw):
1225 """Called when teardown keyword starts. Default implementation does
1228 :param setup_kw: Keyword to process.
1229 :type setup_kw: Keyword
1232 if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1233 and not self._version:
1234 self._msg_type = u"vpp-version"
1235 elif setup_kw.name.count(u"Set Global Variable") \
1236 and not self._timestamp:
1237 self._msg_type = u"timestamp"
1238 elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1239 self._msg_type = u"testbed"
1242 setup_kw.messages.visit(self)
1244 def end_setup_kw(self, setup_kw):
1245 """Called when keyword ends. Default implementation does nothing.
1247 :param setup_kw: Keyword to process.
1248 :type setup_kw: Keyword
1252 def visit_teardown_kw(self, teardown_kw):
1253 """Implements traversing through the teardown keyword and its child
1256 :param teardown_kw: Keyword to process.
1257 :type teardown_kw: Keyword
1260 for keyword in teardown_kw.keywords:
1261 if self.start_teardown_kw(keyword) is not False:
1262 self.visit_teardown_kw(keyword)
1263 self.end_teardown_kw(keyword)
1265 def start_teardown_kw(self, teardown_kw):
1266 """Called when teardown keyword starts
1268 :param teardown_kw: Keyword to process.
1269 :type teardown_kw: Keyword
1273 if teardown_kw.name.count(u"Show Vat History On All Duts"):
1274 # TODO: Remove when not needed:
1275 self._conf_history_lookup_nr = 0
1276 self._msg_type = u"teardown-vat-history"
1277 teardown_kw.messages.visit(self)
1278 elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1279 self._conf_history_lookup_nr = 0
1280 self._msg_type = u"teardown-papi-history"
1281 teardown_kw.messages.visit(self)
1283 def end_teardown_kw(self, teardown_kw):
1284 """Called when keyword ends. Default implementation does nothing.
1286 :param teardown_kw: Keyword to process.
1287 :type teardown_kw: Keyword
1291 def visit_message(self, msg):
1292 """Implements visiting the message.
1294 :param msg: Message to process.
1298 if self.start_message(msg) is not False:
1299 self.end_message(msg)
1301 def start_message(self, msg):
1302 """Called when message starts. Get required information from messages:
1305 :param msg: Message to process.
1310 self.parse_msg[self._msg_type](msg)
1312 def end_message(self, msg):
1313 """Called when message ends. Default implementation does nothing.
1315 :param msg: Message to process.
1324 The data is extracted from output.xml files generated by Jenkins jobs and
1325 stored in pandas' DataFrames.
1331 (as described in ExecutionChecker documentation)
1333 (as described in ExecutionChecker documentation)
1335 (as described in ExecutionChecker documentation)
1338 def __init__(self, spec):
1341 :param spec: Specification.
1342 :type spec: Specification
1349 self._input_data = pd.Series()
1353 """Getter - Input data.
1355 :returns: Input data
1356 :rtype: pandas.Series
1358 return self._input_data
1360 def metadata(self, job, build):
1361 """Getter - metadata
1363 :param job: Job which metadata we want.
1364 :param build: Build which metadata we want.
1368 :rtype: pandas.Series
1370 return self.data[job][build][u"metadata"]
1372 def suites(self, job, build):
1375 :param job: Job which suites we want.
1376 :param build: Build which suites we want.
1380 :rtype: pandas.Series
1382 return self.data[job][str(build)][u"suites"]
1384 def tests(self, job, build):
1387 :param job: Job which tests we want.
1388 :param build: Build which tests we want.
1392 :rtype: pandas.Series
1394 return self.data[job][build][u"tests"]
1396 def _parse_tests(self, job, build, log):
1397 """Process data from robot output.xml file and return JSON structured
1400 :param job: The name of job which build output data will be processed.
1401 :param build: The build which output data will be processed.
1402 :param log: List of log messages.
1405 :type log: list of tuples (severity, msg)
1406 :returns: JSON data structure.
1415 with open(build[u"file-name"], u'r') as data_file:
1417 result = ExecutionResult(data_file)
1418 except errors.DataError as err:
1420 (u"ERROR", f"Error occurred while parsing output.xml: "
1424 checker = ExecutionChecker(metadata, self._cfg.mapping,
1426 result.visit(checker)
1430 def _download_and_parse_build(self, job, build, repeat, pid=10000):
1431 """Download and parse the input data file.
1433 :param pid: PID of the process executing this method.
1434 :param job: Name of the Jenkins job which generated the processed input
1436 :param build: Information about the Jenkins build which generated the
1437 processed input file.
1438 :param repeat: Repeat the download specified number of times if not
1449 (u"INFO", f" Processing the job/build: {job}: {build[u'build']}")
1457 success = download_and_unzip_data_file(self._cfg, job, build, pid,
1465 f"It is not possible to download the input data file from the "
1466 f"job {job}, build {build[u'build']}, or it is damaged. "
1472 f" Processing data from the build {build[u'build']} ...")
1474 data = self._parse_tests(job, build, logs)
1478 f"Input data file from the job {job}, build "
1479 f"{build[u'build']} is damaged. Skipped.")
1482 state = u"processed"
1485 remove(build[u"file-name"])
1486 except OSError as err:
1488 ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
1492 # If the time-period is defined in the specification file, remove all
1493 # files which are outside the time period.
1494 timeperiod = self._cfg.input.get(u"time-period", None)
1495 if timeperiod and data:
1497 timeperiod = timedelta(int(timeperiod))
1498 metadata = data.get(u"metadata", None)
1500 generated = metadata.get(u"generated", None)
1502 generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1503 if (now - generated) > timeperiod:
1504 # Remove the data and the file:
1509 f" The build {job}/{build[u'build']} is "
1510 f"outdated, will be removed.")
1512 logs.append((u"INFO", u" Done."))
1514 for level, line in logs:
1515 if level == u"INFO":
1517 elif level == u"ERROR":
1519 elif level == u"DEBUG":
1521 elif level == u"CRITICAL":
1522 logging.critical(line)
1523 elif level == u"WARNING":
1524 logging.warning(line)
1526 return {u"data": data, u"state": state, u"job": job, u"build": build}
1528 def download_and_parse_data(self, repeat=1):
1529 """Download the input data files, parse input data from input files and
1530 store in pandas' Series.
1532 :param repeat: Repeat the download specified number of times if not
1537 logging.info(u"Downloading and parsing input files ...")
1539 for job, builds in self._cfg.builds.items():
1540 for build in builds:
1542 result = self._download_and_parse_build(job, build, repeat)
1543 build_nr = result[u"build"][u"build"]
1546 data = result[u"data"]
1547 build_data = pd.Series({
1548 u"metadata": pd.Series(
1549 list(data[u"metadata"].values()),
1550 index=list(data[u"metadata"].keys())
1552 u"suites": pd.Series(
1553 list(data[u"suites"].values()),
1554 index=list(data[u"suites"].keys())
1556 u"tests": pd.Series(
1557 list(data[u"tests"].values()),
1558 index=list(data[u"tests"].keys())
1562 if self._input_data.get(job, None) is None:
1563 self._input_data[job] = pd.Series()
1564 self._input_data[job][str(build_nr)] = build_data
1566 self._cfg.set_input_file_name(
1567 job, build_nr, result[u"build"][u"file-name"])
1569 self._cfg.set_input_state(job, build_nr, result[u"state"])
1572 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1573 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1575 logging.info(u"Done.")
1577 def process_local_file(self, local_file, job=u"local", build_nr=1,
1579 """Process local XML file given as a command-line parameter.
1581 :param local_file: The file to process.
1582 :param job: Job name.
1583 :param build_nr: Build number.
1584 :param replace: If True, the information about jobs and builds is
1585 replaced by the new one, otherwise the new jobs and builds are
1587 :type local_file: str
1591 :raises: PresentationError in an error occurs.
1593 if not isfile(local_file):
1594 raise PresentationError(f"The file {local_file} does not exist.")
1598 u"status": u"failed",
1599 u"file-name": local_file
1602 self._cfg.builds = dict()
1603 self._cfg.add_build(job, build)
1605 logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1606 data = self._parse_tests(job, build, list())
1608 raise PresentationError(
1609 f"Error occurred while parsing the file {local_file}"
1612 build_data = pd.Series({
1613 u"metadata": pd.Series(
1614 list(data[u"metadata"].values()),
1615 index=list(data[u"metadata"].keys())
1617 u"suites": pd.Series(
1618 list(data[u"suites"].values()),
1619 index=list(data[u"suites"].keys())
1621 u"tests": pd.Series(
1622 list(data[u"tests"].values()),
1623 index=list(data[u"tests"].keys())
1627 if self._input_data.get(job, None) is None:
1628 self._input_data[job] = pd.Series()
1629 self._input_data[job][str(build_nr)] = build_data
1631 self._cfg.set_input_state(job, build_nr, u"processed")
1633 def process_local_directory(self, local_dir, replace=True):
1634 """Process local directory with XML file(s). The directory is processed
1635 as a 'job' and the XML files in in as builds.
1636 If the given directory contains only sub-directories, these
1637 sub-directories processed as jobs and corresponding XML files as builds
1640 :param local_dir: Local directory to process.
1641 :param replace: If True, the information about jobs and builds is
1642 replaced by the new one, otherwise the new jobs and builds are
1644 :type local_dir: str
1647 if not isdir(local_dir):
1648 raise PresentationError(
1649 f"The directory {local_dir} does not exist."
1652 # Check if the given directory includes only files, or only directories
1653 _, dirnames, filenames = next(walk(local_dir))
1655 if filenames and not dirnames:
1658 # key: dir (job) name, value: list of file names (builds)
1660 local_dir: [join(local_dir, name) for name in filenames]
1663 elif dirnames and not filenames:
1666 # key: dir (job) name, value: list of file names (builds)
1667 local_builds = dict()
1668 for dirname in dirnames:
1670 join(local_dir, dirname, name)
1671 for name in listdir(join(local_dir, dirname))
1672 if isfile(join(local_dir, dirname, name))
1675 local_builds[dirname] = sorted(builds)
1677 elif not filenames and not dirnames:
1678 raise PresentationError(f"The directory {local_dir} is empty.")
1680 raise PresentationError(
1681 f"The directory {local_dir} can include only files or only "
1682 f"directories, not both.\nThe directory {local_dir} includes "
1683 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1687 self._cfg.builds = dict()
1689 for job, files in local_builds.items():
1690 for idx, local_file in enumerate(files):
1691 self.process_local_file(local_file, job, idx + 1, replace=False)
1694 def _end_of_tag(tag_filter, start=0, closer=u"'"):
1695 """Return the index of character in the string which is the end of tag.
1697 :param tag_filter: The string where the end of tag is being searched.
1698 :param start: The index where the searching is stated.
1699 :param closer: The character which is the tag closer.
1700 :type tag_filter: str
1703 :returns: The index of the tag closer.
1707 idx_opener = tag_filter.index(closer, start)
1708 return tag_filter.index(closer, idx_opener + 1)
1713 def _condition(tag_filter):
1714 """Create a conditional statement from the given tag filter.
1716 :param tag_filter: Filter based on tags from the element specification.
1717 :type tag_filter: str
1718 :returns: Conditional statement which can be evaluated.
1723 index = InputData._end_of_tag(tag_filter, index)
1727 tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1729 def filter_data(self, element, params=None, data=None, data_set=u"tests",
1730 continue_on_error=False):
1731 """Filter required data from the given jobs and builds.
1733 The output data structure is:
1736 - test (or suite) 1 ID:
1742 - test (or suite) n ID:
1749 :param element: Element which will use the filtered data.
1750 :param params: Parameters which will be included in the output. If None,
1751 all parameters are included.
1752 :param data: If not None, this data is used instead of data specified
1754 :param data_set: The set of data to be filtered: tests, suites,
1756 :param continue_on_error: Continue if there is error while reading the
1757 data. The Item will be empty then
1758 :type element: pandas.Series
1762 :type continue_on_error: bool
1763 :returns: Filtered data.
1764 :rtype pandas.Series
1768 if data_set == "suites":
1770 elif element[u"filter"] in (u"all", u"template"):
1773 cond = InputData._condition(element[u"filter"])
1774 logging.debug(f" Filter: {cond}")
1776 logging.error(u" No filter defined.")
1780 params = element.get(u"parameters", None)
1782 params.append(u"type")
1784 data_to_filter = data if data else element[u"data"]
1787 for job, builds in data_to_filter.items():
1788 data[job] = pd.Series()
1789 for build in builds:
1790 data[job][str(build)] = pd.Series()
1793 self.data[job][str(build)][data_set].items())
1795 if continue_on_error:
1799 for test_id, test_data in data_dict.items():
1800 if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1801 data[job][str(build)][test_id] = pd.Series()
1803 for param, val in test_data.items():
1804 data[job][str(build)][test_id][param] = val
1806 for param in params:
1808 data[job][str(build)][test_id][param] =\
1811 data[job][str(build)][test_id][param] =\
1815 except (KeyError, IndexError, ValueError) as err:
1817 f"Missing mandatory parameter in the element specification: "
1821 except AttributeError as err:
1822 logging.error(repr(err))
1824 except SyntaxError as err:
1826 f"The filter {cond} is not correct. Check if all tags are "
1827 f"enclosed by apostrophes.\n{repr(err)}"
1831 def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1832 continue_on_error=False):
1833 """Filter required data from the given jobs and builds.
1835 The output data structure is:
1838 - test (or suite) 1 ID:
1844 - test (or suite) n ID:
1851 :param element: Element which will use the filtered data.
1852 :param params: Parameters which will be included in the output. If None,
1853 all parameters are included.
1854 :param data_set: The set of data to be filtered: tests, suites,
1856 :param continue_on_error: Continue if there is error while reading the
1857 data. The Item will be empty then
1858 :type element: pandas.Series
1861 :type continue_on_error: bool
1862 :returns: Filtered data.
1863 :rtype pandas.Series
1866 include = element.get(u"include", None)
1868 logging.warning(u"No tests to include, skipping the element.")
1872 params = element.get(u"parameters", None)
1874 params.append(u"type")
1878 for job, builds in element[u"data"].items():
1879 data[job] = pd.Series()
1880 for build in builds:
1881 data[job][str(build)] = pd.Series()
1882 for test in include:
1884 reg_ex = re.compile(str(test).lower())
1885 for test_id in self.data[job][
1886 str(build)][data_set].keys():
1887 if re.match(reg_ex, str(test_id).lower()):
1888 test_data = self.data[job][
1889 str(build)][data_set][test_id]
1890 data[job][str(build)][test_id] = pd.Series()
1892 for param, val in test_data.items():
1893 data[job][str(build)][test_id]\
1896 for param in params:
1898 data[job][str(build)][
1902 data[job][str(build)][
1903 test_id][param] = u"No Data"
1904 except KeyError as err:
1905 logging.error(repr(err))
1906 if continue_on_error:
1911 except (KeyError, IndexError, ValueError) as err:
1913 f"Missing mandatory parameter in the element "
1914 f"specification: {repr(err)}"
1917 except AttributeError as err:
1918 logging.error(repr(err))
1922 def merge_data(data):
1923 """Merge data from more jobs and builds to a simple data structure.
1925 The output data structure is:
1927 - test (suite) 1 ID:
1933 - test (suite) n ID:
1936 :param data: Data to merge.
1937 :type data: pandas.Series
1938 :returns: Merged data.
1939 :rtype: pandas.Series
1942 logging.info(u" Merging data ...")
1944 merged_data = pd.Series()
1945 for builds in data.values:
1946 for item in builds.values:
1947 for item_id, item_data in item.items():
1948 merged_data[item_id] = item_data
1951 def print_all_oper_data(self):
1952 """Print all operational data to console.
1960 u"Cycles per Packet",
1961 u"Average Vector Size"
1964 for job in self._input_data.values:
1965 for build in job.values:
1966 for test_id, test_data in build[u"tests"].items():
1968 if test_data.get(u"show-run", None) is None:
1970 for dut_name, data in test_data[u"show-run"].items():
1971 if data.get(u"threads", None) is None:
1973 print(f"Host IP: {data.get(u'host', '')}, "
1974 f"Socket: {data.get(u'socket', '')}")
1975 for thread_nr, thread in data[u"threads"].items():
1976 txt_table = prettytable.PrettyTable(tbl_hdr)
1979 txt_table.add_row(row)
1981 if len(thread) == 0:
1984 avg = f", Average Vector Size per Node: " \
1985 f"{(avg / len(thread)):.2f}"
1986 th_name = u"main" if thread_nr == 0 \
1987 else f"worker_{thread_nr}"
1988 print(f"{dut_name}, {th_name}{avg}")
1989 txt_table.float_format = u".2"
1990 txt_table.align = u"r"
1991 txt_table.align[u"Name"] = u"l"
1992 print(f"{txt_table.get_string()}\n")