resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove, walk, listdir
  29 from os.path import isfile, isdir, join
  30 from datetime import datetime as dt
  31 from datetime import timedelta
  32 from json import loads
  33 from json.decoder import JSONDecodeError
  34
  35 import hdrh.histogram
  36 import hdrh.codec
  37 import prettytable
  38 import pandas as pd
  39
  40 from robot.api import ExecutionResult, ResultVisitor
  41 from robot import errors
  42
  43 from resources.libraries.python import jumpavg
  44 from input_data_files import download_and_unzip_data_file
  45 from pal_errors import PresentationError
  46
  47
  48 # Separator used in file names
  49 SEPARATOR = u"__"
  50
  51
  52 class ExecutionChecker(ResultVisitor):
  53     """Class to traverse through the test suite structure.
  54
  55     The functionality implemented in this class generates a json structure:
  56
  57     Performance tests:
  58
  59     {
  60         "metadata": {
  61             "generated": "Timestamp",
  62             "version": "SUT version",
  63             "job": "Jenkins job name",
  64             "build": "Information about the build"
  65         },
  66         "suites": {
  67             "Suite long name 1": {
  68                 "name": Suite name,
  69                 "doc": "Suite 1 documentation",
  70                 "parent": "Suite 1 parent",
  71                 "level": "Level of the suite in the suite hierarchy"
  72             }
  73             "Suite long name N": {
  74                 "name": Suite name,
  75                 "doc": "Suite N documentation",
  76                 "parent": "Suite 2 parent",
  77                 "level": "Level of the suite in the suite hierarchy"
  78             }
  79         }
  80         "tests": {
  81             # NDRPDR tests:
  82             "ID": {
  83                 "name": "Test name",
  84                 "parent": "Name of the parent of the test",
  85                 "doc": "Test documentation",
  86                 "msg": "Test message",
  87                 "conf-history": "DUT1 and DUT2 VAT History",
  88                 "show-run": "Show Run",
  89                 "tags": ["tag 1", "tag 2", "tag n"],
  90                 "type": "NDRPDR",
  91                 "status": "PASS" | "FAIL",
  92                 "throughput": {
  93                     "NDR": {
  94                         "LOWER": float,
  95                         "UPPER": float
  96                     },
  97                     "PDR": {
  98                         "LOWER": float,
  99                         "UPPER": float
 100                     }
 101                 },
 102                 "latency": {
 103                     "NDR": {
 104                         "direction1": {
 105                             "min": float,
 106                             "avg": float,
 107                             "max": float,
 108                             "hdrh": str
 109                         },
 110                         "direction2": {
 111                             "min": float,
 112                             "avg": float,
 113                             "max": float,
 114                             "hdrh": str
 115                         }
 116                     },
 117                     "PDR": {
 118                         "direction1": {
 119                             "min": float,
 120                             "avg": float,
 121                             "max": float,
 122                             "hdrh": str
 123                         },
 124                         "direction2": {
 125                             "min": float,
 126                             "avg": float,
 127                             "max": float,
 128                             "hdrh": str
 129                         }
 130                     }
 131                 }
 132             }
 133
 134             # TCP tests:
 135             "ID": {
 136                 "name": "Test name",
 137                 "parent": "Name of the parent of the test",
 138                 "doc": "Test documentation",
 139                 "msg": "Test message",
 140                 "tags": ["tag 1", "tag 2", "tag n"],
 141                 "type": "TCP",
 142                 "status": "PASS" | "FAIL",
 143                 "result": int
 144             }
 145
 146             # MRR, BMRR tests:
 147             "ID": {
 148                 "name": "Test name",
 149                 "parent": "Name of the parent of the test",
 150                 "doc": "Test documentation",
 151                 "msg": "Test message",
 152                 "tags": ["tag 1", "tag 2", "tag n"],
 153                 "type": "MRR" | "BMRR",
 154                 "status": "PASS" | "FAIL",
 155                 "result": {
 156                     "receive-rate": float,
 157                     # Average of a list, computed using AvgStdevStats.
 158                     # In CSIT-1180, replace with List[float].
 159                 }
 160             }
 161
 162             "ID" {
 163                 # next test
 164             }
 165         }
 166     }
 167
 168
 169     Functional tests:
 170
 171     {
 172         "metadata": {  # Optional
 173             "version": "VPP version",
 174             "job": "Jenkins job name",
 175             "build": "Information about the build"
 176         },
 177         "suites": {
 178             "Suite name 1": {
 179                 "doc": "Suite 1 documentation",
 180                 "parent": "Suite 1 parent",
 181                 "level": "Level of the suite in the suite hierarchy"
 182             }
 183             "Suite name N": {
 184                 "doc": "Suite N documentation",
 185                 "parent": "Suite 2 parent",
 186                 "level": "Level of the suite in the suite hierarchy"
 187             }
 188         }
 189         "tests": {
 190             "ID": {
 191                 "name": "Test name",
 192                 "parent": "Name of the parent of the test",
 193                 "doc": "Test documentation"
 194                 "msg": "Test message"
 195                 "tags": ["tag 1", "tag 2", "tag n"],
 196                 "conf-history": "DUT1 and DUT2 VAT History"
 197                 "show-run": "Show Run"
 198                 "status": "PASS" | "FAIL"
 199             },
 200             "ID" {
 201                 # next test
 202             }
 203         }
 204     }
 205
 206     .. note:: ID is the lowercase full path to the test.
 207     """
 208
 209     REGEX_PLR_RATE = re.compile(
 210         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 211         r'PLRsearch upper bound::?\s(\d+.\d+)'
 212     )
 213     REGEX_NDRPDR_RATE = re.compile(
 214         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 215         r'NDR_UPPER:\s(\d+.\d+).*\n'
 216         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 217         r'PDR_UPPER:\s(\d+.\d+)'
 218     )
 219     REGEX_NDRPDR_GBPS = re.compile(
 220         r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
 221         r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
 222         r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
 223         r'PDR_UPPER:.*,\s(\d+.\d+)'
 224     )
 225     REGEX_PERF_MSG_INFO = re.compile(
 226         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 227         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 228         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 229         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 230         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 231     )
 232     REGEX_CPS_MSG_INFO = re.compile(
 233         r'NDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*\n'
 234         r'PDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*'
 235     )
 236     REGEX_PPS_MSG_INFO = re.compile(
 237         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 238         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*'
 239     )
 240     REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
 241
 242     # Needed for CPS and PPS tests
 243     REGEX_NDRPDR_LAT_BASE = re.compile(
 244         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 245         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 246     )
 247     REGEX_NDRPDR_LAT = re.compile(
 248         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 249         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 250         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 251         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 252         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 253         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 254     )
 255
 256     REGEX_VERSION_VPP = re.compile(
 257         r"(return STDOUT Version:\s*|"
 258         r"VPP Version:\s*|VPP version:\s*)(.*)"
 259     )
 260     REGEX_VERSION_DPDK = re.compile(
 261         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 262     )
 263     REGEX_TCP = re.compile(
 264         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 265     )
 266     REGEX_MRR = re.compile(
 267         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 268         r'tx\s(\d*),\srx\s(\d*)'
 269     )
 270     REGEX_BMRR = re.compile(
 271         r'.*trial results.*: \[(.*)\]'
 272     )
 273     REGEX_RECONF_LOSS = re.compile(
 274         r'Packets lost due to reconfig: (\d*)'
 275     )
 276     REGEX_RECONF_TIME = re.compile(
 277         r'Implied time lost: (\d*.[\de-]*)'
 278     )
 279     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 280
 281     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 282
 283     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 284
 285     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 286
 287     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 288
 289     def __init__(self, metadata, mapping, ignore):
 290         """Initialisation.
 291
 292         :param metadata: Key-value pairs to be included in "metadata" part of
 293             JSON structure.
 294         :param mapping: Mapping of the old names of test cases to the new
 295             (actual) one.
 296         :param ignore: List of TCs to be ignored.
 297         :type metadata: dict
 298         :type mapping: dict
 299         :type ignore: list
 300         """
 301
 302         # Type of message to parse out from the test messages
 303         self._msg_type = None
 304
 305         # VPP version
 306         self._version = None
 307
 308         # Timestamp
 309         self._timestamp = None
 310
 311         # Testbed. The testbed is identified by TG node IP address.
 312         self._testbed = None
 313
 314         # Mapping of TCs long names
 315         self._mapping = mapping
 316
 317         # Ignore list
 318         self._ignore = ignore
 319
 320         # Number of PAPI History messages found:
 321         # 0 - no message
 322         # 1 - PAPI History of DUT1
 323         # 2 - PAPI History of DUT2
 324         self._conf_history_lookup_nr = 0
 325
 326         self._sh_run_counter = 0
 327
 328         # Test ID of currently processed test- the lowercase full path to the
 329         # test
 330         self._test_id = None
 331
 332         # The main data structure
 333         self._data = {
 334             u"metadata": OrderedDict(),
 335             u"suites": OrderedDict(),
 336             u"tests": OrderedDict()
 337         }
 338
 339         # Save the provided metadata
 340         for key, val in metadata.items():
 341             self._data[u"metadata"][key] = val
 342
 343         # Dictionary defining the methods used to parse different types of
 344         # messages
 345         self.parse_msg = {
 346             u"timestamp": self._get_timestamp,
 347             u"vpp-version": self._get_vpp_version,
 348             u"dpdk-version": self._get_dpdk_version,
 349             u"teardown-papi-history": self._get_papi_history,
 350             u"test-show-runtime": self._get_show_run,
 351             u"testbed": self._get_testbed
 352         }
 353
 354     @property
 355     def data(self):
 356         """Getter - Data parsed from the XML file.
 357
 358         :returns: Data parsed from the XML file.
 359         :rtype: dict
 360         """
 361         return self._data
 362
 363     def _get_data_from_mrr_test_msg(self, msg):
 364         """Get info from message of MRR performance tests.
 365
 366         :param msg: Message to be processed.
 367         :type msg: str
 368         :returns: Processed message or original message if a problem occurs.
 369         :rtype: str
 370         """
 371
 372         groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
 373         if not groups or groups.lastindex != 1:
 374             return u"Test Failed."
 375
 376         try:
 377             data = groups.group(1).split(u", ")
 378         except (AttributeError, IndexError, ValueError, KeyError):
 379             return u"Test Failed."
 380
 381         out_str = u"["
 382         try:
 383             for item in data:
 384                 out_str += f"{(float(item) / 1e6):.2f}, "
 385             return out_str[:-2] + u"]"
 386         except (AttributeError, IndexError, ValueError, KeyError):
 387             return u"Test Failed."
 388
 389     def _get_data_from_cps_test_msg(self, msg):
 390         """Get info from message of NDRPDR CPS tests.
 391
 392         :param msg: Message to be processed.
 393         :type msg: str
 394         :returns: Processed message or "Test Failed." if a problem occurs.
 395         :rtype: str
 396         """
 397
 398         groups = re.search(self.REGEX_CPS_MSG_INFO, msg)
 399         if not groups or groups.lastindex != 2:
 400             return u"Test Failed."
 401
 402         try:
 403             return (
 404                 f"1. {(float(groups.group(1)) / 1e6):5.2f}\n"
 405                 f"2. {(float(groups.group(2)) / 1e6):5.2f}"
 406             )
 407         except (AttributeError, IndexError, ValueError, KeyError):
 408             return u"Test Failed."
 409
 410     def _get_data_from_pps_test_msg(self, msg):
 411         """Get info from message of NDRPDR PPS tests.
 412
 413         :param msg: Message to be processed.
 414         :type msg: str
 415         :returns: Processed message or "Test Failed." if a problem occurs.
 416         :rtype: str
 417         """
 418
 419         groups = re.search(self.REGEX_PPS_MSG_INFO, msg)
 420         if not groups or groups.lastindex != 4:
 421             return u"Test Failed."
 422
 423         try:
 424             return (
 425                 f"1. {(float(groups.group(1)) / 1e6):5.2f}      "
 426                 f"{float(groups.group(2)):5.2f}\n"
 427                 f"2. {(float(groups.group(3)) / 1e6):5.2f}      "
 428                 f"{float(groups.group(4)):5.2f}"
 429             )
 430         except (AttributeError, IndexError, ValueError, KeyError):
 431             return u"Test Failed."
 432
 433     def _get_data_from_perf_test_msg(self, msg):
 434         """Get info from message of NDRPDR performance tests.
 435
 436         :param msg: Message to be processed.
 437         :type msg: str
 438         :returns: Processed message or "Test Failed." if a problem occurs.
 439         :rtype: str
 440         """
 441
 442         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 443         if not groups or groups.lastindex != 10:
 444             return u"Test Failed."
 445
 446         try:
 447             data = {
 448                 u"ndr_low": float(groups.group(1)),
 449                 u"ndr_low_b": float(groups.group(2)),
 450                 u"pdr_low": float(groups.group(3)),
 451                 u"pdr_low_b": float(groups.group(4)),
 452                 u"pdr_lat_90_1": groups.group(5),
 453                 u"pdr_lat_90_2": groups.group(6),
 454                 u"pdr_lat_50_1": groups.group(7),
 455                 u"pdr_lat_50_2": groups.group(8),
 456                 u"pdr_lat_10_1": groups.group(9),
 457                 u"pdr_lat_10_2": groups.group(10),
 458             }
 459         except (AttributeError, IndexError, ValueError, KeyError):
 460             return u"Test Failed."
 461
 462         def _process_lat(in_str_1, in_str_2):
 463             """Extract min, avg, max values from latency string.
 464
 465             :param in_str_1: Latency string for one direction produced by robot
 466                 framework.
 467             :param in_str_2: Latency string for second direction produced by
 468                 robot framework.
 469             :type in_str_1: str
 470             :type in_str_2: str
 471             :returns: Processed latency string or None if a problem occurs.
 472             :rtype: tuple
 473             """
 474             in_list_1 = in_str_1.split('/', 3)
 475             in_list_2 = in_str_2.split('/', 3)
 476
 477             if len(in_list_1) != 4 and len(in_list_2) != 4:
 478                 return None
 479
 480             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 481             try:
 482                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 483             except hdrh.codec.HdrLengthException:
 484                 return None
 485
 486             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 487             try:
 488                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 489             except hdrh.codec.HdrLengthException:
 490                 return None
 491
 492             if hdr_lat_1 and hdr_lat_2:
 493                 hdr_lat = (
 494                     hdr_lat_1.get_value_at_percentile(50.0),
 495                     hdr_lat_1.get_value_at_percentile(90.0),
 496                     hdr_lat_1.get_value_at_percentile(99.0),
 497                     hdr_lat_2.get_value_at_percentile(50.0),
 498                     hdr_lat_2.get_value_at_percentile(90.0),
 499                     hdr_lat_2.get_value_at_percentile(99.0)
 500                 )
 501
 502                 if all(hdr_lat):
 503                     return hdr_lat
 504
 505             return None
 506
 507         try:
 508             out_msg = (
 509                 f"1. {(data[u'ndr_low'] / 1e6):5.2f}      "
 510                 f"{data[u'ndr_low_b']:5.2f}"
 511                 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f}      "
 512                 f"{data[u'pdr_low_b']:5.2f}"
 513             )
 514             latency = (
 515                 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
 516                 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
 517                 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
 518             )
 519             if all(latency):
 520                 max_len = len(str(max((max(item) for item in latency))))
 521                 max_len = 4 if max_len < 4 else max_len
 522
 523                 for idx, lat in enumerate(latency):
 524                     if not idx:
 525                         out_msg += u"\n"
 526                     out_msg += (
 527                         f"\n{idx + 3}. "
 528                         f"{lat[0]:{max_len}d} "
 529                         f"{lat[1]:{max_len}d} "
 530                         f"{lat[2]:{max_len}d}      "
 531                         f"{lat[3]:{max_len}d} "
 532                         f"{lat[4]:{max_len}d} "
 533                         f"{lat[5]:{max_len}d} "
 534                     )
 535
 536             return out_msg
 537
 538         except (AttributeError, IndexError, ValueError, KeyError):
 539             return u"Test Failed."
 540
 541     def _get_testbed(self, msg):
 542         """Called when extraction of testbed IP is required.
 543         The testbed is identified by TG node IP address.
 544
 545         :param msg: Message to process.
 546         :type msg: Message
 547         :returns: Nothing.
 548         """
 549
 550         if msg.message.count(u"Setup of TG node") or \
 551                 msg.message.count(u"Setup of node TG host"):
 552             reg_tg_ip = re.compile(
 553                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 554             try:
 555                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 556             except (KeyError, ValueError, IndexError, AttributeError):
 557                 pass
 558             finally:
 559                 self._data[u"metadata"][u"testbed"] = self._testbed
 560                 self._msg_type = None
 561
 562     def _get_vpp_version(self, msg):
 563         """Called when extraction of VPP version is required.
 564
 565         :param msg: Message to process.
 566         :type msg: Message
 567         :returns: Nothing.
 568         """
 569
 570         if msg.message.count(u"return STDOUT Version:") or \
 571                 msg.message.count(u"VPP Version:") or \
 572                 msg.message.count(u"VPP version:"):
 573             self._version = str(
 574                 re.search(self.REGEX_VERSION_VPP, msg.message).group(2)
 575             )
 576             self._data[u"metadata"][u"version"] = self._version
 577             self._msg_type = None
 578
 579     def _get_dpdk_version(self, msg):
 580         """Called when extraction of DPDK version is required.
 581
 582         :param msg: Message to process.
 583         :type msg: Message
 584         :returns: Nothing.
 585         """
 586
 587         if msg.message.count(u"DPDK Version:"):
 588             try:
 589                 self._version = str(re.search(
 590                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 591                 self._data[u"metadata"][u"version"] = self._version
 592             except IndexError:
 593                 pass
 594             finally:
 595                 self._msg_type = None
 596
 597     def _get_timestamp(self, msg):
 598         """Called when extraction of timestamp is required.
 599
 600         :param msg: Message to process.
 601         :type msg: Message
 602         :returns: Nothing.
 603         """
 604
 605         self._timestamp = msg.timestamp[:14]
 606         self._data[u"metadata"][u"generated"] = self._timestamp
 607         self._msg_type = None
 608
 609     def _get_papi_history(self, msg):
 610         """Called when extraction of PAPI command history is required.
 611
 612         :param msg: Message to process.
 613         :type msg: Message
 614         :returns: Nothing.
 615         """
 616         if msg.message.count(u"PAPI command history:"):
 617             self._conf_history_lookup_nr += 1
 618             if self._conf_history_lookup_nr == 1:
 619                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 620             else:
 621                 self._msg_type = None
 622             text = re.sub(
 623                 r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} PAPI command history:",
 624                 u"",
 625                 msg.message,
 626                 count=1
 627             ).replace(u'"', u"'")
 628             self._data[u"tests"][self._test_id][u"conf-history"] += (
 629                 f"**DUT{str(self._conf_history_lookup_nr)}:** {text}"
 630             )
 631
 632     def _get_show_run(self, msg):
 633         """Called when extraction of VPP operational data (output of CLI command
 634         Show Runtime) is required.
 635
 636         :param msg: Message to process.
 637         :type msg: Message
 638         :returns: Nothing.
 639         """
 640
 641         if not msg.message.count(u"stats runtime"):
 642             return
 643
 644         # Temporary solution
 645         if self._sh_run_counter > 1:
 646             return
 647
 648         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 649             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 650
 651         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 652         if not groups:
 653             return
 654         try:
 655             host = groups.group(1)
 656         except (AttributeError, IndexError):
 657             host = u""
 658         try:
 659             sock = groups.group(2)
 660         except (AttributeError, IndexError):
 661             sock = u""
 662
 663         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 664                         replace(u"'", u'"').replace(u'b"', u'"').
 665                         replace(u'u"', u'"').split(u":", 1)[1])
 666
 667         try:
 668             threads_nr = len(runtime[0][u"clocks"])
 669         except (IndexError, KeyError):
 670             return
 671
 672         dut = u"dut{nr}".format(
 673             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 674
 675         oper = {
 676             u"host": host,
 677             u"socket": sock,
 678             # Needed for json converter, enable when 'threads' is gone.
 679             # u"runtime": runtime,
 680             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 681         }
 682
 683         for item in runtime:
 684             for idx in range(threads_nr):
 685                 if item[u"vectors"][idx] > 0:
 686                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 687                 elif item[u"calls"][idx] > 0:
 688                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 689                 elif item[u"suspends"][idx] > 0:
 690                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 691                 else:
 692                     clocks = 0.0
 693
 694                 if item[u"calls"][idx] > 0:
 695                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 696                 else:
 697                     vectors_call = 0.0
 698
 699                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 700                         int(item[u"suspends"][idx]):
 701                     oper[u"threads"][idx].append([
 702                         item[u"name"],
 703                         item[u"calls"][idx],
 704                         item[u"vectors"][idx],
 705                         item[u"suspends"][idx],
 706                         clocks,
 707                         vectors_call
 708                     ])
 709
 710         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 711
 712     def _get_ndrpdr_throughput(self, msg):
 713         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 714         message.
 715
 716         :param msg: The test message to be parsed.
 717         :type msg: str
 718         :returns: Parsed data as a dict and the status (PASS/FAIL).
 719         :rtype: tuple(dict, str)
 720         """
 721
 722         throughput = {
 723             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 724             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 725         }
 726         status = u"FAIL"
 727         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 728
 729         if groups is not None:
 730             try:
 731                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 732                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 733                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 734                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 735                 status = u"PASS"
 736             except (IndexError, ValueError):
 737                 pass
 738
 739         return throughput, status
 740
 741     def _get_ndrpdr_throughput_gbps(self, msg):
 742         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
 743         test message.
 744
 745         :param msg: The test message to be parsed.
 746         :type msg: str
 747         :returns: Parsed data as a dict and the status (PASS/FAIL).
 748         :rtype: tuple(dict, str)
 749         """
 750
 751         gbps = {
 752             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 753             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 754         }
 755         status = u"FAIL"
 756         groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
 757
 758         if groups is not None:
 759             try:
 760                 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
 761                 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
 762                 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
 763                 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
 764                 status = u"PASS"
 765             except (IndexError, ValueError):
 766                 pass
 767
 768         return gbps, status
 769
 770     def _get_plr_throughput(self, msg):
 771         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 772         message.
 773
 774         :param msg: The test message to be parsed.
 775         :type msg: str
 776         :returns: Parsed data as a dict and the status (PASS/FAIL).
 777         :rtype: tuple(dict, str)
 778         """
 779
 780         throughput = {
 781             u"LOWER": -1.0,
 782             u"UPPER": -1.0
 783         }
 784         status = u"FAIL"
 785         groups = re.search(self.REGEX_PLR_RATE, msg)
 786
 787         if groups is not None:
 788             try:
 789                 throughput[u"LOWER"] = float(groups.group(1))
 790                 throughput[u"UPPER"] = float(groups.group(2))
 791                 status = u"PASS"
 792             except (IndexError, ValueError):
 793                 pass
 794
 795         return throughput, status
 796
 797     def _get_ndrpdr_latency(self, msg):
 798         """Get LATENCY from the test message.
 799
 800         :param msg: The test message to be parsed.
 801         :type msg: str
 802         :returns: Parsed data as a dict and the status (PASS/FAIL).
 803         :rtype: tuple(dict, str)
 804         """
 805         latency_default = {
 806             u"min": -1.0,
 807             u"avg": -1.0,
 808             u"max": -1.0,
 809             u"hdrh": u""
 810         }
 811         latency = {
 812             u"NDR": {
 813                 u"direction1": copy.copy(latency_default),
 814                 u"direction2": copy.copy(latency_default)
 815             },
 816             u"PDR": {
 817                 u"direction1": copy.copy(latency_default),
 818                 u"direction2": copy.copy(latency_default)
 819             },
 820             u"LAT0": {
 821                 u"direction1": copy.copy(latency_default),
 822                 u"direction2": copy.copy(latency_default)
 823             },
 824             u"PDR10": {
 825                 u"direction1": copy.copy(latency_default),
 826                 u"direction2": copy.copy(latency_default)
 827             },
 828             u"PDR50": {
 829                 u"direction1": copy.copy(latency_default),
 830                 u"direction2": copy.copy(latency_default)
 831             },
 832             u"PDR90": {
 833                 u"direction1": copy.copy(latency_default),
 834                 u"direction2": copy.copy(latency_default)
 835             },
 836         }
 837
 838         groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 839         if groups is None:
 840             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 841         if groups is None:
 842             return latency, u"FAIL"
 843
 844         def process_latency(in_str):
 845             """Return object with parsed latency values.
 846
 847             TODO: Define class for the return type.
 848
 849             :param in_str: Input string, min/avg/max/hdrh format.
 850             :type in_str: str
 851             :returns: Dict with corresponding keys, except hdrh float values.
 852             :rtype dict:
 853             :throws IndexError: If in_str does not have enough substrings.
 854             :throws ValueError: If a substring does not convert to float.
 855             """
 856             in_list = in_str.split('/', 3)
 857
 858             rval = {
 859                 u"min": float(in_list[0]),
 860                 u"avg": float(in_list[1]),
 861                 u"max": float(in_list[2]),
 862                 u"hdrh": u""
 863             }
 864
 865             if len(in_list) == 4:
 866                 rval[u"hdrh"] = str(in_list[3])
 867
 868             return rval
 869
 870         try:
 871             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 872             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 873             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 874             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 875             if groups.lastindex == 4:
 876                 return latency, u"PASS"
 877         except (IndexError, ValueError):
 878             pass
 879
 880         try:
 881             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 882             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 883             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 884             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 885             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 886             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 887             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 888             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 889             if groups.lastindex == 12:
 890                 return latency, u"PASS"
 891         except (IndexError, ValueError):
 892             pass
 893
 894         return latency, u"FAIL"
 895
 896     @staticmethod
 897     def _get_hoststack_data(msg, tags):
 898         """Get data from the hoststack test message.
 899
 900         :param msg: The test message to be parsed.
 901         :param tags: Test tags.
 902         :type msg: str
 903         :type tags: list
 904         :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
 905         :rtype: tuple(dict, str)
 906         """
 907         result = dict()
 908         status = u"FAIL"
 909
 910         msg = msg.replace(u"'", u'"').replace(u" ", u"")
 911         if u"LDPRELOAD" in tags:
 912             try:
 913                 result = loads(msg)
 914                 status = u"PASS"
 915             except JSONDecodeError:
 916                 pass
 917         elif u"VPPECHO" in tags:
 918             try:
 919                 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
 920                 result = dict(
 921                     client=loads(msg_lst[0]),
 922                     server=loads(msg_lst[1])
 923                 )
 924                 status = u"PASS"
 925             except (JSONDecodeError, IndexError):
 926                 pass
 927
 928         return result, status
 929
 930     def visit_suite(self, suite):
 931         """Implements traversing through the suite and its direct children.
 932
 933         :param suite: Suite to process.
 934         :type suite: Suite
 935         :returns: Nothing.
 936         """
 937         if self.start_suite(suite) is not False:
 938             suite.suites.visit(self)
 939             suite.tests.visit(self)
 940             self.end_suite(suite)
 941
 942     def start_suite(self, suite):
 943         """Called when suite starts.
 944
 945         :param suite: Suite to process.
 946         :type suite: Suite
 947         :returns: Nothing.
 948         """
 949
 950         try:
 951             parent_name = suite.parent.name
 952         except AttributeError:
 953             return
 954
 955         self._data[u"suites"][suite.longname.lower().
 956                               replace(u'"', u"'").
 957                               replace(u" ", u"_")] = {
 958                                   u"name": suite.name.lower(),
 959                                   u"doc": suite.doc,
 960                                   u"parent": parent_name,
 961                                   u"level": len(suite.longname.split(u"."))
 962                               }
 963
 964         suite.keywords.visit(self)
 965
 966     def end_suite(self, suite):
 967         """Called when suite ends.
 968
 969         :param suite: Suite to process.
 970         :type suite: Suite
 971         :returns: Nothing.
 972         """
 973
 974     def visit_test(self, test):
 975         """Implements traversing through the test.
 976
 977         :param test: Test to process.
 978         :type test: Test
 979         :returns: Nothing.
 980         """
 981         if self.start_test(test) is not False:
 982             test.keywords.visit(self)
 983             self.end_test(test)
 984
 985     def start_test(self, test):
 986         """Called when test starts.
 987
 988         :param test: Test to process.
 989         :type test: Test
 990         :returns: Nothing.
 991         """
 992
 993         self._sh_run_counter = 0
 994
 995         longname_orig = test.longname.lower()
 996
 997         # Check the ignore list
 998         if longname_orig in self._ignore:
 999             return
1000
1001         tags = [str(tag) for tag in test.tags]
1002         test_result = dict()
1003
1004         # Change the TC long name and name if defined in the mapping table
1005         longname = self._mapping.get(longname_orig, None)
1006         if longname is not None:
1007             name = longname.split(u'.')[-1]
1008             logging.debug(
1009                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1010                 f"{name}"
1011             )
1012         else:
1013             longname = longname_orig
1014             name = test.name.lower()
1015
1016         # Remove TC number from the TC long name (backward compatibility):
1017         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1018         # Remove TC number from the TC name (not needed):
1019         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1020
1021         test_result[u"parent"] = test.parent.name.lower()
1022         test_result[u"tags"] = tags
1023         test_result["doc"] = test.doc
1024         test_result[u"type"] = u""
1025         test_result[u"status"] = test.status
1026         test_result[u"starttime"] = test.starttime
1027         test_result[u"endtime"] = test.endtime
1028
1029         if test.status == u"PASS":
1030             if u"NDRPDR" in tags:
1031                 if u"TCP_PPS" in tags or u"UDP_PPS" in tags:
1032                     test_result[u"msg"] = self._get_data_from_pps_test_msg(
1033                         test.message)
1034                 elif u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1035                     test_result[u"msg"] = self._get_data_from_cps_test_msg(
1036                         test.message)
1037                 else:
1038                     test_result[u"msg"] = self._get_data_from_perf_test_msg(
1039                         test.message)
1040             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1041                 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1042                     test.message)
1043             else:
1044                 test_result[u"msg"] = test.message
1045         else:
1046             test_result[u"msg"] = test.message
1047
1048         if u"PERFTEST" in tags:
1049             # Replace info about cores (e.g. -1c-) with the info about threads
1050             # and cores (e.g. -1t1c-) in the long test case names and in the
1051             # test case names if necessary.
1052             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1053             if not groups:
1054                 tag_count = 0
1055                 tag_tc = str()
1056                 for tag in test_result[u"tags"]:
1057                     groups = re.search(self.REGEX_TC_TAG, tag)
1058                     if groups:
1059                         tag_count += 1
1060                         tag_tc = tag
1061
1062                 if tag_count == 1:
1063                     self._test_id = re.sub(
1064                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1065                         self._test_id, count=1
1066                     )
1067                     test_result[u"name"] = re.sub(
1068                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1069                         test_result["name"], count=1
1070                     )
1071                 else:
1072                     test_result[u"status"] = u"FAIL"
1073                     self._data[u"tests"][self._test_id] = test_result
1074                     logging.debug(
1075                         f"The test {self._test_id} has no or more than one "
1076                         f"multi-threading tags.\n"
1077                         f"Tags: {test_result[u'tags']}"
1078                     )
1079                     return
1080
1081         if u"DEVICETEST" in tags:
1082             test_result[u"type"] = u"DEVICETEST"
1083         elif u"NDRPDR" in tags:
1084             if u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1085                 test_result[u"type"] = u"CPS"
1086             else:
1087                 test_result[u"type"] = u"NDRPDR"
1088             if test.status == u"PASS":
1089                 test_result[u"throughput"], test_result[u"status"] = \
1090                     self._get_ndrpdr_throughput(test.message)
1091                 test_result[u"gbps"], test_result[u"status"] = \
1092                     self._get_ndrpdr_throughput_gbps(test.message)
1093                 test_result[u"latency"], test_result[u"status"] = \
1094                     self._get_ndrpdr_latency(test.message)
1095         elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1096             if u"MRR" in tags:
1097                 test_result[u"type"] = u"MRR"
1098             else:
1099                 test_result[u"type"] = u"BMRR"
1100             if test.status == u"PASS":
1101                 test_result[u"result"] = dict()
1102                 groups = re.search(self.REGEX_BMRR, test.message)
1103                 if groups is not None:
1104                     items_str = groups.group(1)
1105                     items_float = [
1106                         float(item.strip().replace(u"'", u""))
1107                         for item in items_str.split(",")
1108                     ]
1109                     # Use whole list in CSIT-1180.
1110                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1111                     test_result[u"result"][u"samples"] = items_float
1112                     test_result[u"result"][u"receive-rate"] = stats.avg
1113                     test_result[u"result"][u"receive-stdev"] = stats.stdev
1114                 else:
1115                     groups = re.search(self.REGEX_MRR, test.message)
1116                     test_result[u"result"][u"receive-rate"] = \
1117                         float(groups.group(3)) / float(groups.group(1))
1118         elif u"SOAK" in tags:
1119             test_result[u"type"] = u"SOAK"
1120             if test.status == u"PASS":
1121                 test_result[u"throughput"], test_result[u"status"] = \
1122                     self._get_plr_throughput(test.message)
1123         elif u"HOSTSTACK" in tags:
1124             test_result[u"type"] = u"HOSTSTACK"
1125             if test.status == u"PASS":
1126                 test_result[u"result"], test_result[u"status"] = \
1127                     self._get_hoststack_data(test.message, tags)
1128         # elif u"TCP" in tags:  # This might be not used
1129         #     test_result[u"type"] = u"TCP"
1130         #     if test.status == u"PASS":
1131         #         groups = re.search(self.REGEX_TCP, test.message)
1132         #         test_result[u"result"] = int(groups.group(2))
1133         elif u"RECONF" in tags:
1134             test_result[u"type"] = u"RECONF"
1135             if test.status == u"PASS":
1136                 test_result[u"result"] = None
1137                 try:
1138                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1139                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1140                     test_result[u"result"] = {
1141                         u"loss": int(grps_loss.group(1)),
1142                         u"time": float(grps_time.group(1))
1143                     }
1144                 except (AttributeError, IndexError, ValueError, TypeError):
1145                     test_result[u"status"] = u"FAIL"
1146         else:
1147             test_result[u"status"] = u"FAIL"
1148
1149         self._data[u"tests"][self._test_id] = test_result
1150
1151     def end_test(self, test):
1152         """Called when test ends.
1153
1154         :param test: Test to process.
1155         :type test: Test
1156         :returns: Nothing.
1157         """
1158
1159     def visit_keyword(self, keyword):
1160         """Implements traversing through the keyword and its child keywords.
1161
1162         :param keyword: Keyword to process.
1163         :type keyword: Keyword
1164         :returns: Nothing.
1165         """
1166         if self.start_keyword(keyword) is not False:
1167             self.end_keyword(keyword)
1168
1169     def start_keyword(self, keyword):
1170         """Called when keyword starts. Default implementation does nothing.
1171
1172         :param keyword: Keyword to process.
1173         :type keyword: Keyword
1174         :returns: Nothing.
1175         """
1176         try:
1177             if keyword.type == u"setup":
1178                 self.visit_setup_kw(keyword)
1179             elif keyword.type == u"teardown":
1180                 self.visit_teardown_kw(keyword)
1181             else:
1182                 self.visit_test_kw(keyword)
1183         except AttributeError:
1184             pass
1185
1186     def end_keyword(self, keyword):
1187         """Called when keyword ends. Default implementation does nothing.
1188
1189         :param keyword: Keyword to process.
1190         :type keyword: Keyword
1191         :returns: Nothing.
1192         """
1193
1194     def visit_test_kw(self, test_kw):
1195         """Implements traversing through the test keyword and its child
1196         keywords.
1197
1198         :param test_kw: Keyword to process.
1199         :type test_kw: Keyword
1200         :returns: Nothing.
1201         """
1202         for keyword in test_kw.keywords:
1203             if self.start_test_kw(keyword) is not False:
1204                 self.visit_test_kw(keyword)
1205                 self.end_test_kw(keyword)
1206
1207     def start_test_kw(self, test_kw):
1208         """Called when test keyword starts. Default implementation does
1209         nothing.
1210
1211         :param test_kw: Keyword to process.
1212         :type test_kw: Keyword
1213         :returns: Nothing.
1214         """
1215         if test_kw.name.count(u"Show Runtime On All Duts") or \
1216                 test_kw.name.count(u"Show Runtime Counters On All Duts") or \
1217                 test_kw.name.count(u"Vpp Show Runtime On All Duts"):
1218             self._msg_type = u"test-show-runtime"
1219             self._sh_run_counter += 1
1220         else:
1221             return
1222         test_kw.messages.visit(self)
1223
1224     def end_test_kw(self, test_kw):
1225         """Called when keyword ends. Default implementation does nothing.
1226
1227         :param test_kw: Keyword to process.
1228         :type test_kw: Keyword
1229         :returns: Nothing.
1230         """
1231
1232     def visit_setup_kw(self, setup_kw):
1233         """Implements traversing through the teardown keyword and its child
1234         keywords.
1235
1236         :param setup_kw: Keyword to process.
1237         :type setup_kw: Keyword
1238         :returns: Nothing.
1239         """
1240         for keyword in setup_kw.keywords:
1241             if self.start_setup_kw(keyword) is not False:
1242                 self.visit_setup_kw(keyword)
1243                 self.end_setup_kw(keyword)
1244
1245     def start_setup_kw(self, setup_kw):
1246         """Called when teardown keyword starts. Default implementation does
1247         nothing.
1248
1249         :param setup_kw: Keyword to process.
1250         :type setup_kw: Keyword
1251         :returns: Nothing.
1252         """
1253         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1254                 and not self._version:
1255             self._msg_type = u"vpp-version"
1256         elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1257                 not self._version:
1258             self._msg_type = u"dpdk-version"
1259         elif setup_kw.name.count(u"Set Global Variable") \
1260                 and not self._timestamp:
1261             self._msg_type = u"timestamp"
1262         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1263             self._msg_type = u"testbed"
1264         else:
1265             return
1266         setup_kw.messages.visit(self)
1267
1268     def end_setup_kw(self, setup_kw):
1269         """Called when keyword ends. Default implementation does nothing.
1270
1271         :param setup_kw: Keyword to process.
1272         :type setup_kw: Keyword
1273         :returns: Nothing.
1274         """
1275
1276     def visit_teardown_kw(self, teardown_kw):
1277         """Implements traversing through the teardown keyword and its child
1278         keywords.
1279
1280         :param teardown_kw: Keyword to process.
1281         :type teardown_kw: Keyword
1282         :returns: Nothing.
1283         """
1284         for keyword in teardown_kw.keywords:
1285             if self.start_teardown_kw(keyword) is not False:
1286                 self.visit_teardown_kw(keyword)
1287                 self.end_teardown_kw(keyword)
1288
1289     def start_teardown_kw(self, teardown_kw):
1290         """Called when teardown keyword starts
1291
1292         :param teardown_kw: Keyword to process.
1293         :type teardown_kw: Keyword
1294         :returns: Nothing.
1295         """
1296         if teardown_kw.name.count(u"Show Papi History On All Duts"):
1297             self._conf_history_lookup_nr = 0
1298             self._msg_type = u"teardown-papi-history"
1299             teardown_kw.messages.visit(self)
1300
1301     def end_teardown_kw(self, teardown_kw):
1302         """Called when keyword ends. Default implementation does nothing.
1303
1304         :param teardown_kw: Keyword to process.
1305         :type teardown_kw: Keyword
1306         :returns: Nothing.
1307         """
1308
1309     def visit_message(self, msg):
1310         """Implements visiting the message.
1311
1312         :param msg: Message to process.
1313         :type msg: Message
1314         :returns: Nothing.
1315         """
1316         if self.start_message(msg) is not False:
1317             self.end_message(msg)
1318
1319     def start_message(self, msg):
1320         """Called when message starts. Get required information from messages:
1321         - VPP version.
1322
1323         :param msg: Message to process.
1324         :type msg: Message
1325         :returns: Nothing.
1326         """
1327         if self._msg_type:
1328             self.parse_msg[self._msg_type](msg)
1329
1330     def end_message(self, msg):
1331         """Called when message ends. Default implementation does nothing.
1332
1333         :param msg: Message to process.
1334         :type msg: Message
1335         :returns: Nothing.
1336         """
1337
1338
1339 class InputData:
1340     """Input data
1341
1342     The data is extracted from output.xml files generated by Jenkins jobs and
1343     stored in pandas' DataFrames.
1344
1345     The data structure:
1346     - job name
1347       - build number
1348         - metadata
1349           (as described in ExecutionChecker documentation)
1350         - suites
1351           (as described in ExecutionChecker documentation)
1352         - tests
1353           (as described in ExecutionChecker documentation)
1354     """
1355
1356     def __init__(self, spec):
1357         """Initialization.
1358
1359         :param spec: Specification.
1360         :type spec: Specification
1361         """
1362
1363         # Specification:
1364         self._cfg = spec
1365
1366         # Data store:
1367         self._input_data = pd.Series()
1368
1369     @property
1370     def data(self):
1371         """Getter - Input data.
1372
1373         :returns: Input data
1374         :rtype: pandas.Series
1375         """
1376         return self._input_data
1377
1378     def metadata(self, job, build):
1379         """Getter - metadata
1380
1381         :param job: Job which metadata we want.
1382         :param build: Build which metadata we want.
1383         :type job: str
1384         :type build: str
1385         :returns: Metadata
1386         :rtype: pandas.Series
1387         """
1388         return self.data[job][build][u"metadata"]
1389
1390     def suites(self, job, build):
1391         """Getter - suites
1392
1393         :param job: Job which suites we want.
1394         :param build: Build which suites we want.
1395         :type job: str
1396         :type build: str
1397         :returns: Suites.
1398         :rtype: pandas.Series
1399         """
1400         return self.data[job][str(build)][u"suites"]
1401
1402     def tests(self, job, build):
1403         """Getter - tests
1404
1405         :param job: Job which tests we want.
1406         :param build: Build which tests we want.
1407         :type job: str
1408         :type build: str
1409         :returns: Tests.
1410         :rtype: pandas.Series
1411         """
1412         return self.data[job][build][u"tests"]
1413
1414     def _parse_tests(self, job, build):
1415         """Process data from robot output.xml file and return JSON structured
1416         data.
1417
1418         :param job: The name of job which build output data will be processed.
1419         :param build: The build which output data will be processed.
1420         :type job: str
1421         :type build: dict
1422         :returns: JSON data structure.
1423         :rtype: dict
1424         """
1425
1426         metadata = {
1427             u"job": job,
1428             u"build": build
1429         }
1430
1431         with open(build[u"file-name"], u'r') as data_file:
1432             try:
1433                 result = ExecutionResult(data_file)
1434             except errors.DataError as err:
1435                 logging.error(
1436                     f"Error occurred while parsing output.xml: {repr(err)}"
1437                 )
1438                 return None
1439         checker = ExecutionChecker(
1440             metadata, self._cfg.mapping, self._cfg.ignore
1441         )
1442         result.visit(checker)
1443
1444         return checker.data
1445
1446     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1447         """Download and parse the input data file.
1448
1449         :param pid: PID of the process executing this method.
1450         :param job: Name of the Jenkins job which generated the processed input
1451             file.
1452         :param build: Information about the Jenkins build which generated the
1453             processed input file.
1454         :param repeat: Repeat the download specified number of times if not
1455             successful.
1456         :type pid: int
1457         :type job: str
1458         :type build: dict
1459         :type repeat: int
1460         """
1461
1462         logging.info(f"Processing the job/build: {job}: {build[u'build']}")
1463
1464         state = u"failed"
1465         success = False
1466         data = None
1467         do_repeat = repeat
1468         while do_repeat:
1469             success = download_and_unzip_data_file(self._cfg, job, build, pid)
1470             if success:
1471                 break
1472             do_repeat -= 1
1473         if not success:
1474             logging.error(
1475                 f"It is not possible to download the input data file from the "
1476                 f"job {job}, build {build[u'build']}, or it is damaged. "
1477                 f"Skipped."
1478             )
1479         if success:
1480             logging.info(f"  Processing data from build {build[u'build']}")
1481             data = self._parse_tests(job, build)
1482             if data is None:
1483                 logging.error(
1484                     f"Input data file from the job {job}, build "
1485                     f"{build[u'build']} is damaged. Skipped."
1486                 )
1487             else:
1488                 state = u"processed"
1489
1490             try:
1491                 remove(build[u"file-name"])
1492             except OSError as err:
1493                 logging.error(
1494                     f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1495                 )
1496
1497         # If the time-period is defined in the specification file, remove all
1498         # files which are outside the time period.
1499         is_last = False
1500         timeperiod = self._cfg.environment.get(u"time-period", None)
1501         if timeperiod and data:
1502             now = dt.utcnow()
1503             timeperiod = timedelta(int(timeperiod))
1504             metadata = data.get(u"metadata", None)
1505             if metadata:
1506                 generated = metadata.get(u"generated", None)
1507                 if generated:
1508                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1509                     if (now - generated) > timeperiod:
1510                         # Remove the data and the file:
1511                         state = u"removed"
1512                         data = None
1513                         is_last = True
1514                         logging.info(
1515                             f"  The build {job}/{build[u'build']} is "
1516                             f"outdated, will be removed."
1517                         )
1518         return {
1519             u"data": data,
1520             u"state": state,
1521             u"job": job,
1522             u"build": build,
1523             u"last": is_last
1524         }
1525
1526     def download_and_parse_data(self, repeat=1):
1527         """Download the input data files, parse input data from input files and
1528         store in pandas' Series.
1529
1530         :param repeat: Repeat the download specified number of times if not
1531             successful.
1532         :type repeat: int
1533         """
1534
1535         logging.info(u"Downloading and parsing input files ...")
1536
1537         for job, builds in self._cfg.input.items():
1538             for build in builds:
1539
1540                 result = self._download_and_parse_build(job, build, repeat)
1541                 if result[u"last"]:
1542                     break
1543                 build_nr = result[u"build"][u"build"]
1544
1545                 if result[u"data"]:
1546                     data = result[u"data"]
1547                     build_data = pd.Series({
1548                         u"metadata": pd.Series(
1549                             list(data[u"metadata"].values()),
1550                             index=list(data[u"metadata"].keys())
1551                         ),
1552                         u"suites": pd.Series(
1553                             list(data[u"suites"].values()),
1554                             index=list(data[u"suites"].keys())
1555                         ),
1556                         u"tests": pd.Series(
1557                             list(data[u"tests"].values()),
1558                             index=list(data[u"tests"].keys())
1559                         )
1560                     })
1561
1562                     if self._input_data.get(job, None) is None:
1563                         self._input_data[job] = pd.Series()
1564                     self._input_data[job][str(build_nr)] = build_data
1565                     self._cfg.set_input_file_name(
1566                         job, build_nr, result[u"build"][u"file-name"]
1567                     )
1568                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1569
1570                 mem_alloc = \
1571                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1572                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1573
1574         logging.info(u"Done.")
1575
1576         msg = f"Successful downloads from the sources:\n"
1577         for source in self._cfg.environment[u"data-sources"]:
1578             if source[u"successful-downloads"]:
1579                 msg += (
1580                     f"{source[u'url']}/{source[u'path']}/"
1581                     f"{source[u'file-name']}: "
1582                     f"{source[u'successful-downloads']}\n"
1583                 )
1584         logging.info(msg)
1585
1586     def process_local_file(self, local_file, job=u"local", build_nr=1,
1587                            replace=True):
1588         """Process local XML file given as a command-line parameter.
1589
1590         :param local_file: The file to process.
1591         :param job: Job name.
1592         :param build_nr: Build number.
1593         :param replace: If True, the information about jobs and builds is
1594             replaced by the new one, otherwise the new jobs and builds are
1595             added.
1596         :type local_file: str
1597         :type job: str
1598         :type build_nr: int
1599         :type replace: bool
1600         :raises: PresentationError if an error occurs.
1601         """
1602         if not isfile(local_file):
1603             raise PresentationError(f"The file {local_file} does not exist.")
1604
1605         try:
1606             build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1607         except (IndexError, ValueError):
1608             pass
1609
1610         build = {
1611             u"build": build_nr,
1612             u"status": u"failed",
1613             u"file-name": local_file
1614         }
1615         if replace:
1616             self._cfg.input = dict()
1617         self._cfg.add_build(job, build)
1618
1619         logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1620         data = self._parse_tests(job, build)
1621         if data is None:
1622             raise PresentationError(
1623                 f"Error occurred while parsing the file {local_file}"
1624             )
1625
1626         build_data = pd.Series({
1627             u"metadata": pd.Series(
1628                 list(data[u"metadata"].values()),
1629                 index=list(data[u"metadata"].keys())
1630             ),
1631             u"suites": pd.Series(
1632                 list(data[u"suites"].values()),
1633                 index=list(data[u"suites"].keys())
1634             ),
1635             u"tests": pd.Series(
1636                 list(data[u"tests"].values()),
1637                 index=list(data[u"tests"].keys())
1638             )
1639         })
1640
1641         if self._input_data.get(job, None) is None:
1642             self._input_data[job] = pd.Series()
1643         self._input_data[job][str(build_nr)] = build_data
1644
1645         self._cfg.set_input_state(job, build_nr, u"processed")
1646
1647     def process_local_directory(self, local_dir, replace=True):
1648         """Process local directory with XML file(s). The directory is processed
1649         as a 'job' and the XML files in it as builds.
1650         If the given directory contains only sub-directories, these
1651         sub-directories processed as jobs and corresponding XML files as builds
1652         of their job.
1653
1654         :param local_dir: Local directory to process.
1655         :param replace: If True, the information about jobs and builds is
1656             replaced by the new one, otherwise the new jobs and builds are
1657             added.
1658         :type local_dir: str
1659         :type replace: bool
1660         """
1661         if not isdir(local_dir):
1662             raise PresentationError(
1663                 f"The directory {local_dir} does not exist."
1664             )
1665
1666         # Check if the given directory includes only files, or only directories
1667         _, dirnames, filenames = next(walk(local_dir))
1668
1669         if filenames and not dirnames:
1670             filenames.sort()
1671             # local_builds:
1672             # key: dir (job) name, value: list of file names (builds)
1673             local_builds = {
1674                 local_dir: [join(local_dir, name) for name in filenames]
1675             }
1676
1677         elif dirnames and not filenames:
1678             dirnames.sort()
1679             # local_builds:
1680             # key: dir (job) name, value: list of file names (builds)
1681             local_builds = dict()
1682             for dirname in dirnames:
1683                 builds = [
1684                     join(local_dir, dirname, name)
1685                     for name in listdir(join(local_dir, dirname))
1686                     if isfile(join(local_dir, dirname, name))
1687                 ]
1688                 if builds:
1689                     local_builds[dirname] = sorted(builds)
1690
1691         elif not filenames and not dirnames:
1692             raise PresentationError(f"The directory {local_dir} is empty.")
1693         else:
1694             raise PresentationError(
1695                 f"The directory {local_dir} can include only files or only "
1696                 f"directories, not both.\nThe directory {local_dir} includes "
1697                 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1698             )
1699
1700         if replace:
1701             self._cfg.input = dict()
1702
1703         for job, files in local_builds.items():
1704             for idx, local_file in enumerate(files):
1705                 self.process_local_file(local_file, job, idx + 1, replace=False)
1706
1707     @staticmethod
1708     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1709         """Return the index of character in the string which is the end of tag.
1710
1711         :param tag_filter: The string where the end of tag is being searched.
1712         :param start: The index where the searching is stated.
1713         :param closer: The character which is the tag closer.
1714         :type tag_filter: str
1715         :type start: int
1716         :type closer: str
1717         :returns: The index of the tag closer.
1718         :rtype: int
1719         """
1720         try:
1721             idx_opener = tag_filter.index(closer, start)
1722             return tag_filter.index(closer, idx_opener + 1)
1723         except ValueError:
1724             return None
1725
1726     @staticmethod
1727     def _condition(tag_filter):
1728         """Create a conditional statement from the given tag filter.
1729
1730         :param tag_filter: Filter based on tags from the element specification.
1731         :type tag_filter: str
1732         :returns: Conditional statement which can be evaluated.
1733         :rtype: str
1734         """
1735         index = 0
1736         while True:
1737             index = InputData._end_of_tag(tag_filter, index)
1738             if index is None:
1739                 return tag_filter
1740             index += 1
1741             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1742
1743     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1744                     continue_on_error=False):
1745         """Filter required data from the given jobs and builds.
1746
1747         The output data structure is:
1748         - job 1
1749           - build 1
1750             - test (or suite) 1 ID:
1751               - param 1
1752               - param 2
1753               ...
1754               - param n
1755             ...
1756             - test (or suite) n ID:
1757             ...
1758           ...
1759           - build n
1760         ...
1761         - job n
1762
1763         :param element: Element which will use the filtered data.
1764         :param params: Parameters which will be included in the output. If None,
1765             all parameters are included.
1766         :param data: If not None, this data is used instead of data specified
1767             in the element.
1768         :param data_set: The set of data to be filtered: tests, suites,
1769             metadata.
1770         :param continue_on_error: Continue if there is error while reading the
1771             data. The Item will be empty then
1772         :type element: pandas.Series
1773         :type params: list
1774         :type data: dict
1775         :type data_set: str
1776         :type continue_on_error: bool
1777         :returns: Filtered data.
1778         :rtype pandas.Series
1779         """
1780
1781         try:
1782             if data_set == "suites":
1783                 cond = u"True"
1784             elif element[u"filter"] in (u"all", u"template"):
1785                 cond = u"True"
1786             else:
1787                 cond = InputData._condition(element[u"filter"])
1788             logging.debug(f"   Filter: {cond}")
1789         except KeyError:
1790             logging.error(u"  No filter defined.")
1791             return None
1792
1793         if params is None:
1794             params = element.get(u"parameters", None)
1795             if params:
1796                 params.extend((u"type", u"status"))
1797
1798         data_to_filter = data if data else element[u"data"]
1799         data = pd.Series()
1800         try:
1801             for job, builds in data_to_filter.items():
1802                 data[job] = pd.Series()
1803                 for build in builds:
1804                     data[job][str(build)] = pd.Series()
1805                     try:
1806                         data_dict = dict(
1807                             self.data[job][str(build)][data_set].items())
1808                     except KeyError:
1809                         if continue_on_error:
1810                             continue
1811                         return None
1812
1813                     for test_id, test_data in data_dict.items():
1814                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1815                             data[job][str(build)][test_id] = pd.Series()
1816                             if params is None:
1817                                 for param, val in test_data.items():
1818                                     data[job][str(build)][test_id][param] = val
1819                             else:
1820                                 for param in params:
1821                                     try:
1822                                         data[job][str(build)][test_id][param] =\
1823                                             test_data[param]
1824                                     except KeyError:
1825                                         data[job][str(build)][test_id][param] =\
1826                                             u"No Data"
1827             return data
1828
1829         except (KeyError, IndexError, ValueError) as err:
1830             logging.error(
1831                 f"Missing mandatory parameter in the element specification: "
1832                 f"{repr(err)}"
1833             )
1834             return None
1835         except AttributeError as err:
1836             logging.error(repr(err))
1837             return None
1838         except SyntaxError as err:
1839             logging.error(
1840                 f"The filter {cond} is not correct. Check if all tags are "
1841                 f"enclosed by apostrophes.\n{repr(err)}"
1842             )
1843             return None
1844
1845     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1846                              continue_on_error=False):
1847         """Filter required data from the given jobs and builds.
1848
1849         The output data structure is:
1850         - job 1
1851           - build 1
1852             - test (or suite) 1 ID:
1853               - param 1
1854               - param 2
1855               ...
1856               - param n
1857             ...
1858             - test (or suite) n ID:
1859             ...
1860           ...
1861           - build n
1862         ...
1863         - job n
1864
1865         :param element: Element which will use the filtered data.
1866         :param params: Parameters which will be included in the output. If None,
1867         all parameters are included.
1868         :param data_set: The set of data to be filtered: tests, suites,
1869         metadata.
1870         :param continue_on_error: Continue if there is error while reading the
1871         data. The Item will be empty then
1872         :type element: pandas.Series
1873         :type params: list
1874         :type data_set: str
1875         :type continue_on_error: bool
1876         :returns: Filtered data.
1877         :rtype pandas.Series
1878         """
1879
1880         include = element.get(u"include", None)
1881         if not include:
1882             logging.warning(u"No tests to include, skipping the element.")
1883             return None
1884
1885         if params is None:
1886             params = element.get(u"parameters", None)
1887             if params and u"type" not in params:
1888                 params.append(u"type")
1889
1890         cores = element.get(u"core", None)
1891         if cores:
1892             tests = list()
1893             for core in cores:
1894                 for test in include:
1895                     tests.append(test.format(core=core))
1896         else:
1897             tests = include
1898
1899         data = pd.Series()
1900         try:
1901             for job, builds in element[u"data"].items():
1902                 data[job] = pd.Series()
1903                 for build in builds:
1904                     data[job][str(build)] = pd.Series()
1905                     for test in tests:
1906                         try:
1907                             reg_ex = re.compile(str(test).lower())
1908                             for test_id in self.data[job][
1909                                     str(build)][data_set].keys():
1910                                 if re.match(reg_ex, str(test_id).lower()):
1911                                     test_data = self.data[job][
1912                                         str(build)][data_set][test_id]
1913                                     data[job][str(build)][test_id] = pd.Series()
1914                                     if params is None:
1915                                         for param, val in test_data.items():
1916                                             data[job][str(build)][test_id]\
1917                                                 [param] = val
1918                                     else:
1919                                         for param in params:
1920                                             try:
1921                                                 data[job][str(build)][
1922                                                     test_id][param] = \
1923                                                     test_data[param]
1924                                             except KeyError:
1925                                                 data[job][str(build)][
1926                                                     test_id][param] = u"No Data"
1927                         except KeyError as err:
1928                             if continue_on_error:
1929                                 logging.debug(repr(err))
1930                                 continue
1931                             logging.error(repr(err))
1932                             return None
1933             return data
1934
1935         except (KeyError, IndexError, ValueError) as err:
1936             logging.error(
1937                 f"Missing mandatory parameter in the element "
1938                 f"specification: {repr(err)}"
1939             )
1940             return None
1941         except AttributeError as err:
1942             logging.error(repr(err))
1943             return None
1944
1945     @staticmethod
1946     def merge_data(data):
1947         """Merge data from more jobs and builds to a simple data structure.
1948
1949         The output data structure is:
1950
1951         - test (suite) 1 ID:
1952           - param 1
1953           - param 2
1954           ...
1955           - param n
1956         ...
1957         - test (suite) n ID:
1958         ...
1959
1960         :param data: Data to merge.
1961         :type data: pandas.Series
1962         :returns: Merged data.
1963         :rtype: pandas.Series
1964         """
1965
1966         logging.info(u"    Merging data ...")
1967
1968         merged_data = pd.Series()
1969         for builds in data.values:
1970             for item in builds.values:
1971                 for item_id, item_data in item.items():
1972                     merged_data[item_id] = item_data
1973         return merged_data
1974
1975     def print_all_oper_data(self):
1976         """Print all operational data to console.
1977         """
1978
1979         tbl_hdr = (
1980             u"Name",
1981             u"Nr of Vectors",
1982             u"Nr of Packets",
1983             u"Suspends",
1984             u"Cycles per Packet",
1985             u"Average Vector Size"
1986         )
1987
1988         for job in self._input_data.values:
1989             for build in job.values:
1990                 for test_id, test_data in build[u"tests"].items():
1991                     print(f"{test_id}")
1992                     if test_data.get(u"show-run", None) is None:
1993                         continue
1994                     for dut_name, data in test_data[u"show-run"].items():
1995                         if data.get(u"threads", None) is None:
1996                             continue
1997                         print(f"Host IP: {data.get(u'host', '')}, "
1998                               f"Socket: {data.get(u'socket', '')}")
1999                         for thread_nr, thread in data[u"threads"].items():
2000                             txt_table = prettytable.PrettyTable(tbl_hdr)
2001                             avg = 0.0
2002                             for row in thread:
2003                                 txt_table.add_row(row)
2004                                 avg += row[-1]
2005                             if len(thread) == 0:
2006                                 avg = u""
2007                             else:
2008                                 avg = f", Average Vector Size per Node: " \
2009                                       f"{(avg / len(thread)):.2f}"
2010                             th_name = u"main" if thread_nr == 0 \
2011                                 else f"worker_{thread_nr}"
2012                             print(f"{dut_name}, {th_name}{avg}")
2013                             txt_table.float_format = u".2"
2014                             txt_table.align = u"r"
2015                             txt_table.align[u"Name"] = u"l"
2016                             print(f"{txt_table.get_string()}\n")