resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove
  29 from datetime import datetime as dt
  30 from datetime import timedelta
  31 from json import loads
  32 from json.decoder import JSONDecodeError
  33
  34 import hdrh.histogram
  35 import hdrh.codec
  36 import prettytable
  37 import pandas as pd
  38
  39 from robot.api import ExecutionResult, ResultVisitor
  40 from robot import errors
  41
  42 from resources.libraries.python import jumpavg
  43 from input_data_files import download_and_unzip_data_file
  44
  45
  46 # Separator used in file names
  47 SEPARATOR = u"__"
  48
  49
  50 class ExecutionChecker(ResultVisitor):
  51     """Class to traverse through the test suite structure.
  52
  53     The functionality implemented in this class generates a json structure:
  54
  55     Performance tests:
  56
  57     {
  58         "metadata": {
  59             "generated": "Timestamp",
  60             "version": "SUT version",
  61             "job": "Jenkins job name",
  62             "build": "Information about the build"
  63         },
  64         "suites": {
  65             "Suite long name 1": {
  66                 "name": Suite name,
  67                 "doc": "Suite 1 documentation",
  68                 "parent": "Suite 1 parent",
  69                 "level": "Level of the suite in the suite hierarchy"
  70             }
  71             "Suite long name N": {
  72                 "name": Suite name,
  73                 "doc": "Suite N documentation",
  74                 "parent": "Suite 2 parent",
  75                 "level": "Level of the suite in the suite hierarchy"
  76             }
  77         }
  78         "tests": {
  79             # NDRPDR tests:
  80             "ID": {
  81                 "name": "Test name",
  82                 "parent": "Name of the parent of the test",
  83                 "doc": "Test documentation",
  84                 "msg": "Test message",
  85                 "conf-history": "DUT1 and DUT2 VAT History",
  86                 "show-run": "Show Run",
  87                 "tags": ["tag 1", "tag 2", "tag n"],
  88                 "type": "NDRPDR",
  89                 "status": "PASS" | "FAIL",
  90                 "throughput": {
  91                     "NDR": {
  92                         "LOWER": float,
  93                         "UPPER": float
  94                     },
  95                     "PDR": {
  96                         "LOWER": float,
  97                         "UPPER": float
  98                     }
  99                 },
 100                 "latency": {
 101                     "NDR": {
 102                         "direction1": {
 103                             "min": float,
 104                             "avg": float,
 105                             "max": float,
 106                             "hdrh": str
 107                         },
 108                         "direction2": {
 109                             "min": float,
 110                             "avg": float,
 111                             "max": float,
 112                             "hdrh": str
 113                         }
 114                     },
 115                     "PDR": {
 116                         "direction1": {
 117                             "min": float,
 118                             "avg": float,
 119                             "max": float,
 120                             "hdrh": str
 121                         },
 122                         "direction2": {
 123                             "min": float,
 124                             "avg": float,
 125                             "max": float,
 126                             "hdrh": str
 127                         }
 128                     }
 129                 }
 130             }
 131
 132             # TCP tests:
 133             "ID": {
 134                 "name": "Test name",
 135                 "parent": "Name of the parent of the test",
 136                 "doc": "Test documentation",
 137                 "msg": "Test message",
 138                 "tags": ["tag 1", "tag 2", "tag n"],
 139                 "type": "TCP",
 140                 "status": "PASS" | "FAIL",
 141                 "result": int
 142             }
 143
 144             # MRR, BMRR tests:
 145             "ID": {
 146                 "name": "Test name",
 147                 "parent": "Name of the parent of the test",
 148                 "doc": "Test documentation",
 149                 "msg": "Test message",
 150                 "tags": ["tag 1", "tag 2", "tag n"],
 151                 "type": "MRR" | "BMRR",
 152                 "status": "PASS" | "FAIL",
 153                 "result": {
 154                     "receive-rate": float,
 155                     # Average of a list, computed using AvgStdevStats.
 156                     # In CSIT-1180, replace with List[float].
 157                 }
 158             }
 159
 160             "ID" {
 161                 # next test
 162             }
 163         }
 164     }
 165
 166
 167     Functional tests:
 168
 169     {
 170         "metadata": {  # Optional
 171             "version": "VPP version",
 172             "job": "Jenkins job name",
 173             "build": "Information about the build"
 174         },
 175         "suites": {
 176             "Suite name 1": {
 177                 "doc": "Suite 1 documentation",
 178                 "parent": "Suite 1 parent",
 179                 "level": "Level of the suite in the suite hierarchy"
 180             }
 181             "Suite name N": {
 182                 "doc": "Suite N documentation",
 183                 "parent": "Suite 2 parent",
 184                 "level": "Level of the suite in the suite hierarchy"
 185             }
 186         }
 187         "tests": {
 188             "ID": {
 189                 "name": "Test name",
 190                 "parent": "Name of the parent of the test",
 191                 "doc": "Test documentation"
 192                 "msg": "Test message"
 193                 "tags": ["tag 1", "tag 2", "tag n"],
 194                 "conf-history": "DUT1 and DUT2 VAT History"
 195                 "show-run": "Show Run"
 196                 "status": "PASS" | "FAIL"
 197             },
 198             "ID" {
 199                 # next test
 200             }
 201         }
 202     }
 203
 204     .. note:: ID is the lowercase full path to the test.
 205     """
 206
 207     REGEX_PLR_RATE = re.compile(
 208         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 209         r'PLRsearch upper bound::?\s(\d+.\d+)'
 210     )
 211     REGEX_NDRPDR_RATE = re.compile(
 212         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 213         r'NDR_UPPER:\s(\d+.\d+).*\n'
 214         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 215         r'PDR_UPPER:\s(\d+.\d+)'
 216     )
 217     REGEX_PERF_MSG_INFO = re.compile(
 218         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 219         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 220         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 221         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 222         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 223     )
 224     REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
 225
 226     # TODO: Remove when not needed
 227     REGEX_NDRPDR_LAT_BASE = re.compile(
 228         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 229         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 230     )
 231     REGEX_NDRPDR_LAT = re.compile(
 232         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 233         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 234         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 235         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 236         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 237         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 238     )
 239     # TODO: Remove when not needed
 240     REGEX_NDRPDR_LAT_LONG = re.compile(
 241         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 242         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 243         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 244         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 245         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 246         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 247         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 248         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 249         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 250     )
 251     REGEX_VERSION_VPP = re.compile(
 252         r"(return STDOUT Version:\s*|"
 253         r"VPP Version:\s*|VPP version:\s*)(.*)"
 254     )
 255     REGEX_VERSION_DPDK = re.compile(
 256         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 257     )
 258     REGEX_TCP = re.compile(
 259         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 260     )
 261     REGEX_MRR = re.compile(
 262         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 263         r'tx\s(\d*),\srx\s(\d*)'
 264     )
 265     REGEX_BMRR = re.compile(
 266         r'Maximum Receive Rate trial results'
 267         r' in packets per second: \[(.*)\]'
 268     )
 269     REGEX_RECONF_LOSS = re.compile(
 270         r'Packets lost due to reconfig: (\d*)'
 271     )
 272     REGEX_RECONF_TIME = re.compile(
 273         r'Implied time lost: (\d*.[\de-]*)'
 274     )
 275     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 276
 277     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 278
 279     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 280
 281     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 282
 283     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 284
 285     def __init__(self, metadata, mapping, ignore):
 286         """Initialisation.
 287
 288         :param metadata: Key-value pairs to be included in "metadata" part of
 289             JSON structure.
 290         :param mapping: Mapping of the old names of test cases to the new
 291             (actual) one.
 292         :param ignore: List of TCs to be ignored.
 293         :type metadata: dict
 294         :type mapping: dict
 295         :type ignore: list
 296         """
 297
 298         # Type of message to parse out from the test messages
 299         self._msg_type = None
 300
 301         # VPP version
 302         self._version = None
 303
 304         # Timestamp
 305         self._timestamp = None
 306
 307         # Testbed. The testbed is identified by TG node IP address.
 308         self._testbed = None
 309
 310         # Mapping of TCs long names
 311         self._mapping = mapping
 312
 313         # Ignore list
 314         self._ignore = ignore
 315
 316         # Number of PAPI History messages found:
 317         # 0 - no message
 318         # 1 - PAPI History of DUT1
 319         # 2 - PAPI History of DUT2
 320         self._conf_history_lookup_nr = 0
 321
 322         self._sh_run_counter = 0
 323
 324         # Test ID of currently processed test- the lowercase full path to the
 325         # test
 326         self._test_id = None
 327
 328         # The main data structure
 329         self._data = {
 330             u"metadata": OrderedDict(),
 331             u"suites": OrderedDict(),
 332             u"tests": OrderedDict()
 333         }
 334
 335         # Save the provided metadata
 336         for key, val in metadata.items():
 337             self._data[u"metadata"][key] = val
 338
 339         # Dictionary defining the methods used to parse different types of
 340         # messages
 341         self.parse_msg = {
 342             u"timestamp": self._get_timestamp,
 343             u"vpp-version": self._get_vpp_version,
 344             u"dpdk-version": self._get_dpdk_version,
 345             # TODO: Remove when not needed:
 346             u"teardown-vat-history": self._get_vat_history,
 347             u"teardown-papi-history": self._get_papi_history,
 348             u"test-show-runtime": self._get_show_run,
 349             u"testbed": self._get_testbed
 350         }
 351
 352     @property
 353     def data(self):
 354         """Getter - Data parsed from the XML file.
 355
 356         :returns: Data parsed from the XML file.
 357         :rtype: dict
 358         """
 359         return self._data
 360
 361     def _get_data_from_mrr_test_msg(self, msg):
 362         """Get info from message of MRR performance tests.
 363
 364         :param msg: Message to be processed.
 365         :type msg: str
 366         :returns: Processed message or original message if a problem occurs.
 367         :rtype: str
 368         """
 369
 370         groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
 371         if not groups or groups.lastindex != 1:
 372             return u"Test Failed."
 373
 374         try:
 375             data = groups.group(1).split(u", ")
 376         except (AttributeError, IndexError, ValueError, KeyError):
 377             return u"Test Failed."
 378
 379         out_str = u"["
 380         try:
 381             for item in data:
 382                 out_str += f"{(float(item) / 1e6):.2f}, "
 383             return out_str[:-2] + u"]"
 384         except (AttributeError, IndexError, ValueError, KeyError):
 385             return u"Test Failed."
 386
 387     def _get_data_from_perf_test_msg(self, msg):
 388         """Get info from message of NDRPDR performance tests.
 389
 390         :param msg: Message to be processed.
 391         :type msg: str
 392         :returns: Processed message or original message if a problem occurs.
 393         :rtype: str
 394         """
 395
 396         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 397         if not groups or groups.lastindex != 10:
 398             return u"Test Failed."
 399
 400         try:
 401             data = {
 402                 u"ndr_low": float(groups.group(1)),
 403                 u"ndr_low_b": float(groups.group(2)),
 404                 u"pdr_low": float(groups.group(3)),
 405                 u"pdr_low_b": float(groups.group(4)),
 406                 u"pdr_lat_90_1": groups.group(5),
 407                 u"pdr_lat_90_2": groups.group(6),
 408                 u"pdr_lat_50_1": groups.group(7),
 409                 u"pdr_lat_50_2": groups.group(8),
 410                 u"pdr_lat_10_1": groups.group(9),
 411                 u"pdr_lat_10_2": groups.group(10),
 412             }
 413         except (AttributeError, IndexError, ValueError, KeyError):
 414             return u"Test Failed."
 415
 416         def _process_lat(in_str_1, in_str_2):
 417             """Extract min, avg, max values from latency string.
 418
 419             :param in_str_1: Latency string for one direction produced by robot
 420                 framework.
 421             :param in_str_2: Latency string for second direction produced by
 422                 robot framework.
 423             :type in_str_1: str
 424             :type in_str_2: str
 425             :returns: Processed latency string or None if a problem occurs.
 426             :rtype: tuple
 427             """
 428             in_list_1 = in_str_1.split('/', 3)
 429             in_list_2 = in_str_2.split('/', 3)
 430
 431             if len(in_list_1) != 4 and len(in_list_2) != 4:
 432                 return None
 433
 434             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 435             try:
 436                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 437             except hdrh.codec.HdrLengthException:
 438                 return None
 439
 440             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 441             try:
 442                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 443             except hdrh.codec.HdrLengthException:
 444                 return None
 445
 446             if hdr_lat_1 and hdr_lat_2:
 447                 hdr_lat = (
 448                     hdr_lat_1.get_value_at_percentile(50.0),
 449                     hdr_lat_1.get_value_at_percentile(90.0),
 450                     hdr_lat_1.get_value_at_percentile(99.0),
 451                     hdr_lat_2.get_value_at_percentile(50.0),
 452                     hdr_lat_2.get_value_at_percentile(90.0),
 453                     hdr_lat_2.get_value_at_percentile(99.0)
 454                 )
 455
 456                 if all(hdr_lat):
 457                     return hdr_lat
 458
 459             return None
 460
 461         try:
 462             out_msg = (
 463                 f"1. {(data[u'ndr_low'] / 1e6):5.2f}      "
 464                 f"{data[u'ndr_low_b']:5.2f}"
 465                 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f}      "
 466                 f"{data[u'pdr_low_b']:5.2f}"
 467             )
 468             latency = (
 469                 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
 470                 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
 471                 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
 472             )
 473             if all(latency):
 474                 max_len = len(str(max((max(item) for item in latency))))
 475                 max_len = 4 if max_len < 4 else max_len
 476
 477                 for idx, lat in enumerate(latency):
 478                     if not idx:
 479                         out_msg += u"\n"
 480                     out_msg += (
 481                         f"\n{idx + 3}. "
 482                         f"{lat[0]:{max_len}d} "
 483                         f"{lat[1]:{max_len}d} "
 484                         f"{lat[2]:{max_len}d}      "
 485                         f"{lat[3]:{max_len}d} "
 486                         f"{lat[4]:{max_len}d} "
 487                         f"{lat[5]:{max_len}d} "
 488                     )
 489
 490             return out_msg
 491
 492         except (AttributeError, IndexError, ValueError, KeyError):
 493             return u"Test Failed."
 494
 495     def _get_testbed(self, msg):
 496         """Called when extraction of testbed IP is required.
 497         The testbed is identified by TG node IP address.
 498
 499         :param msg: Message to process.
 500         :type msg: Message
 501         :returns: Nothing.
 502         """
 503
 504         if msg.message.count(u"Setup of TG node") or \
 505                 msg.message.count(u"Setup of node TG host"):
 506             reg_tg_ip = re.compile(
 507                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 508             try:
 509                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 510             except (KeyError, ValueError, IndexError, AttributeError):
 511                 pass
 512             finally:
 513                 self._data[u"metadata"][u"testbed"] = self._testbed
 514                 self._msg_type = None
 515
 516     def _get_vpp_version(self, msg):
 517         """Called when extraction of VPP version is required.
 518
 519         :param msg: Message to process.
 520         :type msg: Message
 521         :returns: Nothing.
 522         """
 523
 524         if msg.message.count(u"return STDOUT Version:") or \
 525             msg.message.count(u"VPP Version:") or \
 526             msg.message.count(u"VPP version:"):
 527             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 528                                 group(2))
 529             self._data[u"metadata"][u"version"] = self._version
 530             self._msg_type = None
 531
 532     def _get_dpdk_version(self, msg):
 533         """Called when extraction of DPDK version is required.
 534
 535         :param msg: Message to process.
 536         :type msg: Message
 537         :returns: Nothing.
 538         """
 539
 540         if msg.message.count(u"DPDK Version:"):
 541             try:
 542                 self._version = str(re.search(
 543                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 544                 self._data[u"metadata"][u"version"] = self._version
 545             except IndexError:
 546                 pass
 547             finally:
 548                 self._msg_type = None
 549
 550     def _get_timestamp(self, msg):
 551         """Called when extraction of timestamp is required.
 552
 553         :param msg: Message to process.
 554         :type msg: Message
 555         :returns: Nothing.
 556         """
 557
 558         self._timestamp = msg.timestamp[:14]
 559         self._data[u"metadata"][u"generated"] = self._timestamp
 560         self._msg_type = None
 561
 562     def _get_vat_history(self, msg):
 563         """Called when extraction of VAT command history is required.
 564
 565         TODO: Remove when not needed.
 566
 567         :param msg: Message to process.
 568         :type msg: Message
 569         :returns: Nothing.
 570         """
 571         if msg.message.count(u"VAT command history:"):
 572             self._conf_history_lookup_nr += 1
 573             if self._conf_history_lookup_nr == 1:
 574                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 575             else:
 576                 self._msg_type = None
 577             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 578                           r"VAT command history:", u"",
 579                           msg.message, count=1).replace(u'\n', u' |br| ').\
 580                 replace(u'"', u"'")
 581
 582             self._data[u"tests"][self._test_id][u"conf-history"] += (
 583                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 584             )
 585
 586     def _get_papi_history(self, msg):
 587         """Called when extraction of PAPI command history is required.
 588
 589         :param msg: Message to process.
 590         :type msg: Message
 591         :returns: Nothing.
 592         """
 593         if msg.message.count(u"PAPI command history:"):
 594             self._conf_history_lookup_nr += 1
 595             if self._conf_history_lookup_nr == 1:
 596                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 597             else:
 598                 self._msg_type = None
 599             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 600                           r"PAPI command history:", u"",
 601                           msg.message, count=1).replace(u'\n', u' |br| ').\
 602                 replace(u'"', u"'")
 603             self._data[u"tests"][self._test_id][u"conf-history"] += (
 604                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 605             )
 606
 607     def _get_show_run(self, msg):
 608         """Called when extraction of VPP operational data (output of CLI command
 609         Show Runtime) is required.
 610
 611         :param msg: Message to process.
 612         :type msg: Message
 613         :returns: Nothing.
 614         """
 615
 616         if not msg.message.count(u"stats runtime"):
 617             return
 618
 619         # Temporary solution
 620         if self._sh_run_counter > 1:
 621             return
 622
 623         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 624             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 625
 626         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 627         if not groups:
 628             return
 629         try:
 630             host = groups.group(1)
 631         except (AttributeError, IndexError):
 632             host = u""
 633         try:
 634             sock = groups.group(2)
 635         except (AttributeError, IndexError):
 636             sock = u""
 637
 638         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 639                         replace(u"'", u'"').replace(u'b"', u'"').
 640                         replace(u'u"', u'"').split(u":", 1)[1])
 641
 642         try:
 643             threads_nr = len(runtime[0][u"clocks"])
 644         except (IndexError, KeyError):
 645             return
 646
 647         dut = u"DUT{nr}".format(
 648             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 649
 650         oper = {
 651             u"host": host,
 652             u"socket": sock,
 653             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 654         }
 655
 656         for item in runtime:
 657             for idx in range(threads_nr):
 658                 if item[u"vectors"][idx] > 0:
 659                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 660                 elif item[u"calls"][idx] > 0:
 661                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 662                 elif item[u"suspends"][idx] > 0:
 663                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 664                 else:
 665                     clocks = 0.0
 666
 667                 if item[u"calls"][idx] > 0:
 668                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 669                 else:
 670                     vectors_call = 0.0
 671
 672                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 673                         int(item[u"suspends"][idx]):
 674                     oper[u"threads"][idx].append([
 675                         item[u"name"],
 676                         item[u"calls"][idx],
 677                         item[u"vectors"][idx],
 678                         item[u"suspends"][idx],
 679                         clocks,
 680                         vectors_call
 681                     ])
 682
 683         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 684
 685     def _get_ndrpdr_throughput(self, msg):
 686         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 687         message.
 688
 689         :param msg: The test message to be parsed.
 690         :type msg: str
 691         :returns: Parsed data as a dict and the status (PASS/FAIL).
 692         :rtype: tuple(dict, str)
 693         """
 694
 695         throughput = {
 696             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 697             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 698         }
 699         status = u"FAIL"
 700         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 701
 702         if groups is not None:
 703             try:
 704                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 705                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 706                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 707                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 708                 status = u"PASS"
 709             except (IndexError, ValueError):
 710                 pass
 711
 712         return throughput, status
 713
 714     def _get_plr_throughput(self, msg):
 715         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 716         message.
 717
 718         :param msg: The test message to be parsed.
 719         :type msg: str
 720         :returns: Parsed data as a dict and the status (PASS/FAIL).
 721         :rtype: tuple(dict, str)
 722         """
 723
 724         throughput = {
 725             u"LOWER": -1.0,
 726             u"UPPER": -1.0
 727         }
 728         status = u"FAIL"
 729         groups = re.search(self.REGEX_PLR_RATE, msg)
 730
 731         if groups is not None:
 732             try:
 733                 throughput[u"LOWER"] = float(groups.group(1))
 734                 throughput[u"UPPER"] = float(groups.group(2))
 735                 status = u"PASS"
 736             except (IndexError, ValueError):
 737                 pass
 738
 739         return throughput, status
 740
 741     def _get_ndrpdr_latency(self, msg):
 742         """Get LATENCY from the test message.
 743
 744         :param msg: The test message to be parsed.
 745         :type msg: str
 746         :returns: Parsed data as a dict and the status (PASS/FAIL).
 747         :rtype: tuple(dict, str)
 748         """
 749         latency_default = {
 750             u"min": -1.0,
 751             u"avg": -1.0,
 752             u"max": -1.0,
 753             u"hdrh": u""
 754         }
 755         latency = {
 756             u"NDR": {
 757                 u"direction1": copy.copy(latency_default),
 758                 u"direction2": copy.copy(latency_default)
 759             },
 760             u"PDR": {
 761                 u"direction1": copy.copy(latency_default),
 762                 u"direction2": copy.copy(latency_default)
 763             },
 764             u"LAT0": {
 765                 u"direction1": copy.copy(latency_default),
 766                 u"direction2": copy.copy(latency_default)
 767             },
 768             u"PDR10": {
 769                 u"direction1": copy.copy(latency_default),
 770                 u"direction2": copy.copy(latency_default)
 771             },
 772             u"PDR50": {
 773                 u"direction1": copy.copy(latency_default),
 774                 u"direction2": copy.copy(latency_default)
 775             },
 776             u"PDR90": {
 777                 u"direction1": copy.copy(latency_default),
 778                 u"direction2": copy.copy(latency_default)
 779             },
 780         }
 781
 782         # TODO: Rewrite when long and base are not needed
 783         groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
 784         if groups is None:
 785             groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 786         if groups is None:
 787             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 788         if groups is None:
 789             return latency, u"FAIL"
 790
 791         def process_latency(in_str):
 792             """Return object with parsed latency values.
 793
 794             TODO: Define class for the return type.
 795
 796             :param in_str: Input string, min/avg/max/hdrh format.
 797             :type in_str: str
 798             :returns: Dict with corresponding keys, except hdrh float values.
 799             :rtype dict:
 800             :throws IndexError: If in_str does not have enough substrings.
 801             :throws ValueError: If a substring does not convert to float.
 802             """
 803             in_list = in_str.split('/', 3)
 804
 805             rval = {
 806                 u"min": float(in_list[0]),
 807                 u"avg": float(in_list[1]),
 808                 u"max": float(in_list[2]),
 809                 u"hdrh": u""
 810             }
 811
 812             if len(in_list) == 4:
 813                 rval[u"hdrh"] = str(in_list[3])
 814
 815             return rval
 816
 817         try:
 818             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 819             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 820             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 821             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 822             if groups.lastindex == 4:
 823                 return latency, u"PASS"
 824         except (IndexError, ValueError):
 825             pass
 826
 827         try:
 828             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 829             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 830             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 831             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 832             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 833             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 834             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 835             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 836             if groups.lastindex == 12:
 837                 return latency, u"PASS"
 838         except (IndexError, ValueError):
 839             pass
 840
 841         # TODO: Remove when not needed
 842         latency[u"NDR10"] = {
 843             u"direction1": copy.copy(latency_default),
 844             u"direction2": copy.copy(latency_default)
 845         }
 846         latency[u"NDR50"] = {
 847             u"direction1": copy.copy(latency_default),
 848             u"direction2": copy.copy(latency_default)
 849         }
 850         latency[u"NDR90"] = {
 851             u"direction1": copy.copy(latency_default),
 852             u"direction2": copy.copy(latency_default)
 853         }
 854         try:
 855             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
 856             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
 857             latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
 858             latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
 859             latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
 860             latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
 861             latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
 862             latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
 863             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
 864             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
 865             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
 866             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
 867             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
 868             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
 869             return latency, u"PASS"
 870         except (IndexError, ValueError):
 871             pass
 872
 873         return latency, u"FAIL"
 874
 875     @staticmethod
 876     def _get_hoststack_data(msg, tags):
 877         """Get data from the hoststack test message.
 878
 879         :param msg: The test message to be parsed.
 880         :param tags: Test tags.
 881         :type msg: str
 882         :type tags: list
 883         :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
 884         :rtype: tuple(dict, str)
 885         """
 886         result = dict()
 887         status = u"FAIL"
 888
 889         msg = msg.replace(u"'", u'"').replace(u" ", u"")
 890         if u"LDPRELOAD" in tags:
 891             try:
 892                 result = loads(msg)
 893                 status = u"PASS"
 894             except JSONDecodeError:
 895                 pass
 896         elif u"VPPECHO" in tags:
 897             try:
 898                 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
 899                 result = dict(
 900                     client=loads(msg_lst[0]),
 901                     server=loads(msg_lst[1])
 902                 )
 903                 status = u"PASS"
 904             except (JSONDecodeError, IndexError):
 905                 pass
 906
 907         return result, status
 908
 909     def visit_suite(self, suite):
 910         """Implements traversing through the suite and its direct children.
 911
 912         :param suite: Suite to process.
 913         :type suite: Suite
 914         :returns: Nothing.
 915         """
 916         if self.start_suite(suite) is not False:
 917             suite.suites.visit(self)
 918             suite.tests.visit(self)
 919             self.end_suite(suite)
 920
 921     def start_suite(self, suite):
 922         """Called when suite starts.
 923
 924         :param suite: Suite to process.
 925         :type suite: Suite
 926         :returns: Nothing.
 927         """
 928
 929         try:
 930             parent_name = suite.parent.name
 931         except AttributeError:
 932             return
 933
 934         doc_str = suite.doc.\
 935             replace(u'"', u"'").\
 936             replace(u'\n', u' ').\
 937             replace(u'\r', u'').\
 938             replace(u'*[', u' |br| *[').\
 939             replace(u"*", u"**").\
 940             replace(u' |br| *[', u'*[', 1)
 941
 942         self._data[u"suites"][suite.longname.lower().
 943                               replace(u'"', u"'").
 944                               replace(u" ", u"_")] = {
 945                                   u"name": suite.name.lower(),
 946                                   u"doc": doc_str,
 947                                   u"parent": parent_name,
 948                                   u"level": len(suite.longname.split(u"."))
 949                               }
 950
 951         suite.keywords.visit(self)
 952
 953     def end_suite(self, suite):
 954         """Called when suite ends.
 955
 956         :param suite: Suite to process.
 957         :type suite: Suite
 958         :returns: Nothing.
 959         """
 960
 961     def visit_test(self, test):
 962         """Implements traversing through the test.
 963
 964         :param test: Test to process.
 965         :type test: Test
 966         :returns: Nothing.
 967         """
 968         if self.start_test(test) is not False:
 969             test.keywords.visit(self)
 970             self.end_test(test)
 971
 972     def start_test(self, test):
 973         """Called when test starts.
 974
 975         :param test: Test to process.
 976         :type test: Test
 977         :returns: Nothing.
 978         """
 979
 980         self._sh_run_counter = 0
 981
 982         longname_orig = test.longname.lower()
 983
 984         # Check the ignore list
 985         if longname_orig in self._ignore:
 986             return
 987
 988         tags = [str(tag) for tag in test.tags]
 989         test_result = dict()
 990
 991         # Change the TC long name and name if defined in the mapping table
 992         longname = self._mapping.get(longname_orig, None)
 993         if longname is not None:
 994             name = longname.split(u'.')[-1]
 995             logging.debug(
 996                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
 997                 f"{name}"
 998             )
 999         else:
1000             longname = longname_orig
1001             name = test.name.lower()
1002
1003         # Remove TC number from the TC long name (backward compatibility):
1004         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1005         # Remove TC number from the TC name (not needed):
1006         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1007
1008         test_result[u"parent"] = test.parent.name.lower()
1009         test_result[u"tags"] = tags
1010         test_result["doc"] = test.doc.\
1011             replace(u'"', u"'").\
1012             replace(u'\n', u' ').\
1013             replace(u'\r', u'').\
1014             replace(u'[', u' |br| [').\
1015             replace(u' |br| [', u'[', 1)
1016         test_result[u"type"] = u"FUNC"
1017         test_result[u"status"] = test.status
1018
1019         if test.status == u"PASS":
1020             if u"NDRPDR" in tags:
1021                 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1022                     test.message).replace(u'\n', u' |br| ').\
1023                     replace(u'\r', u'').replace(u'"', u"'")
1024             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1025                 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1026                     test.message).replace(u'\n', u' |br| ').\
1027                     replace(u'\r', u'').replace(u'"', u"'")
1028             else:
1029                 test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\
1030                     replace(u'\r', u'').replace(u'"', u"'")
1031         else:
1032             test_result[u"msg"] = u"Test Failed."
1033
1034         if u"PERFTEST" in tags:
1035             # Replace info about cores (e.g. -1c-) with the info about threads
1036             # and cores (e.g. -1t1c-) in the long test case names and in the
1037             # test case names if necessary.
1038             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1039             if not groups:
1040                 tag_count = 0
1041                 tag_tc = str()
1042                 for tag in test_result[u"tags"]:
1043                     groups = re.search(self.REGEX_TC_TAG, tag)
1044                     if groups:
1045                         tag_count += 1
1046                         tag_tc = tag
1047
1048                 if tag_count == 1:
1049                     self._test_id = re.sub(
1050                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1051                         self._test_id, count=1
1052                     )
1053                     test_result[u"name"] = re.sub(
1054                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1055                         test_result["name"], count=1
1056                     )
1057                 else:
1058                     test_result[u"status"] = u"FAIL"
1059                     self._data[u"tests"][self._test_id] = test_result
1060                     logging.debug(
1061                         f"The test {self._test_id} has no or more than one "
1062                         f"multi-threading tags.\n"
1063                         f"Tags: {test_result[u'tags']}"
1064                     )
1065                     return
1066
1067         if test.status == u"PASS":
1068             if u"NDRPDR" in tags:
1069                 test_result[u"type"] = u"NDRPDR"
1070                 test_result[u"throughput"], test_result[u"status"] = \
1071                     self._get_ndrpdr_throughput(test.message)
1072                 test_result[u"latency"], test_result[u"status"] = \
1073                     self._get_ndrpdr_latency(test.message)
1074             elif u"SOAK" in tags:
1075                 test_result[u"type"] = u"SOAK"
1076                 test_result[u"throughput"], test_result[u"status"] = \
1077                     self._get_plr_throughput(test.message)
1078             elif u"HOSTSTACK" in tags:
1079                 test_result[u"type"] = u"HOSTSTACK"
1080                 test_result[u"result"], test_result[u"status"] = \
1081                     self._get_hoststack_data(test.message, tags)
1082             elif u"TCP" in tags:
1083                 test_result[u"type"] = u"TCP"
1084                 groups = re.search(self.REGEX_TCP, test.message)
1085                 test_result[u"result"] = int(groups.group(2))
1086             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1087                 if u"MRR" in tags:
1088                     test_result[u"type"] = u"MRR"
1089                 else:
1090                     test_result[u"type"] = u"BMRR"
1091
1092                 test_result[u"result"] = dict()
1093                 groups = re.search(self.REGEX_BMRR, test.message)
1094                 if groups is not None:
1095                     items_str = groups.group(1)
1096                     items_float = [float(item.strip()) for item
1097                                    in items_str.split(",")]
1098                     # Use whole list in CSIT-1180.
1099                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1100                     test_result[u"result"][u"receive-rate"] = stats.avg
1101                 else:
1102                     groups = re.search(self.REGEX_MRR, test.message)
1103                     test_result[u"result"][u"receive-rate"] = \
1104                         float(groups.group(3)) / float(groups.group(1))
1105             elif u"RECONF" in tags:
1106                 test_result[u"type"] = u"RECONF"
1107                 test_result[u"result"] = None
1108                 try:
1109                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1110                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1111                     test_result[u"result"] = {
1112                         u"loss": int(grps_loss.group(1)),
1113                         u"time": float(grps_time.group(1))
1114                     }
1115                 except (AttributeError, IndexError, ValueError, TypeError):
1116                     test_result[u"status"] = u"FAIL"
1117             elif u"DEVICETEST" in tags:
1118                 test_result[u"type"] = u"DEVICETEST"
1119             else:
1120                 test_result[u"status"] = u"FAIL"
1121                 self._data[u"tests"][self._test_id] = test_result
1122                 return
1123
1124         self._data[u"tests"][self._test_id] = test_result
1125
1126     def end_test(self, test):
1127         """Called when test ends.
1128
1129         :param test: Test to process.
1130         :type test: Test
1131         :returns: Nothing.
1132         """
1133
1134     def visit_keyword(self, keyword):
1135         """Implements traversing through the keyword and its child keywords.
1136
1137         :param keyword: Keyword to process.
1138         :type keyword: Keyword
1139         :returns: Nothing.
1140         """
1141         if self.start_keyword(keyword) is not False:
1142             self.end_keyword(keyword)
1143
1144     def start_keyword(self, keyword):
1145         """Called when keyword starts. Default implementation does nothing.
1146
1147         :param keyword: Keyword to process.
1148         :type keyword: Keyword
1149         :returns: Nothing.
1150         """
1151         try:
1152             if keyword.type == u"setup":
1153                 self.visit_setup_kw(keyword)
1154             elif keyword.type == u"teardown":
1155                 self.visit_teardown_kw(keyword)
1156             else:
1157                 self.visit_test_kw(keyword)
1158         except AttributeError:
1159             pass
1160
1161     def end_keyword(self, keyword):
1162         """Called when keyword ends. Default implementation does nothing.
1163
1164         :param keyword: Keyword to process.
1165         :type keyword: Keyword
1166         :returns: Nothing.
1167         """
1168
1169     def visit_test_kw(self, test_kw):
1170         """Implements traversing through the test keyword and its child
1171         keywords.
1172
1173         :param test_kw: Keyword to process.
1174         :type test_kw: Keyword
1175         :returns: Nothing.
1176         """
1177         for keyword in test_kw.keywords:
1178             if self.start_test_kw(keyword) is not False:
1179                 self.visit_test_kw(keyword)
1180                 self.end_test_kw(keyword)
1181
1182     def start_test_kw(self, test_kw):
1183         """Called when test keyword starts. Default implementation does
1184         nothing.
1185
1186         :param test_kw: Keyword to process.
1187         :type test_kw: Keyword
1188         :returns: Nothing.
1189         """
1190         if test_kw.name.count(u"Show Runtime On All Duts") or \
1191                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
1192             self._msg_type = u"test-show-runtime"
1193             self._sh_run_counter += 1
1194         elif test_kw.name.count(u"Install Dpdk Test On All Duts") and \
1195                 not self._version:
1196             self._msg_type = u"dpdk-version"
1197         else:
1198             return
1199         test_kw.messages.visit(self)
1200
1201     def end_test_kw(self, test_kw):
1202         """Called when keyword ends. Default implementation does nothing.
1203
1204         :param test_kw: Keyword to process.
1205         :type test_kw: Keyword
1206         :returns: Nothing.
1207         """
1208
1209     def visit_setup_kw(self, setup_kw):
1210         """Implements traversing through the teardown keyword and its child
1211         keywords.
1212
1213         :param setup_kw: Keyword to process.
1214         :type setup_kw: Keyword
1215         :returns: Nothing.
1216         """
1217         for keyword in setup_kw.keywords:
1218             if self.start_setup_kw(keyword) is not False:
1219                 self.visit_setup_kw(keyword)
1220                 self.end_setup_kw(keyword)
1221
1222     def start_setup_kw(self, setup_kw):
1223         """Called when teardown keyword starts. Default implementation does
1224         nothing.
1225
1226         :param setup_kw: Keyword to process.
1227         :type setup_kw: Keyword
1228         :returns: Nothing.
1229         """
1230         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1231                 and not self._version:
1232             self._msg_type = u"vpp-version"
1233         elif setup_kw.name.count(u"Set Global Variable") \
1234                 and not self._timestamp:
1235             self._msg_type = u"timestamp"
1236         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1237             self._msg_type = u"testbed"
1238         else:
1239             return
1240         setup_kw.messages.visit(self)
1241
1242     def end_setup_kw(self, setup_kw):
1243         """Called when keyword ends. Default implementation does nothing.
1244
1245         :param setup_kw: Keyword to process.
1246         :type setup_kw: Keyword
1247         :returns: Nothing.
1248         """
1249
1250     def visit_teardown_kw(self, teardown_kw):
1251         """Implements traversing through the teardown keyword and its child
1252         keywords.
1253
1254         :param teardown_kw: Keyword to process.
1255         :type teardown_kw: Keyword
1256         :returns: Nothing.
1257         """
1258         for keyword in teardown_kw.keywords:
1259             if self.start_teardown_kw(keyword) is not False:
1260                 self.visit_teardown_kw(keyword)
1261                 self.end_teardown_kw(keyword)
1262
1263     def start_teardown_kw(self, teardown_kw):
1264         """Called when teardown keyword starts
1265
1266         :param teardown_kw: Keyword to process.
1267         :type teardown_kw: Keyword
1268         :returns: Nothing.
1269         """
1270
1271         if teardown_kw.name.count(u"Show Vat History On All Duts"):
1272             # TODO: Remove when not needed:
1273             self._conf_history_lookup_nr = 0
1274             self._msg_type = u"teardown-vat-history"
1275             teardown_kw.messages.visit(self)
1276         elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1277             self._conf_history_lookup_nr = 0
1278             self._msg_type = u"teardown-papi-history"
1279             teardown_kw.messages.visit(self)
1280
1281     def end_teardown_kw(self, teardown_kw):
1282         """Called when keyword ends. Default implementation does nothing.
1283
1284         :param teardown_kw: Keyword to process.
1285         :type teardown_kw: Keyword
1286         :returns: Nothing.
1287         """
1288
1289     def visit_message(self, msg):
1290         """Implements visiting the message.
1291
1292         :param msg: Message to process.
1293         :type msg: Message
1294         :returns: Nothing.
1295         """
1296         if self.start_message(msg) is not False:
1297             self.end_message(msg)
1298
1299     def start_message(self, msg):
1300         """Called when message starts. Get required information from messages:
1301         - VPP version.
1302
1303         :param msg: Message to process.
1304         :type msg: Message
1305         :returns: Nothing.
1306         """
1307
1308         if self._msg_type:
1309             self.parse_msg[self._msg_type](msg)
1310
1311     def end_message(self, msg):
1312         """Called when message ends. Default implementation does nothing.
1313
1314         :param msg: Message to process.
1315         :type msg: Message
1316         :returns: Nothing.
1317         """
1318
1319
1320 class InputData:
1321     """Input data
1322
1323     The data is extracted from output.xml files generated by Jenkins jobs and
1324     stored in pandas' DataFrames.
1325
1326     The data structure:
1327     - job name
1328       - build number
1329         - metadata
1330           (as described in ExecutionChecker documentation)
1331         - suites
1332           (as described in ExecutionChecker documentation)
1333         - tests
1334           (as described in ExecutionChecker documentation)
1335     """
1336
1337     def __init__(self, spec):
1338         """Initialization.
1339
1340         :param spec: Specification.
1341         :type spec: Specification
1342         """
1343
1344         # Specification:
1345         self._cfg = spec
1346
1347         # Data store:
1348         self._input_data = pd.Series()
1349
1350     @property
1351     def data(self):
1352         """Getter - Input data.
1353
1354         :returns: Input data
1355         :rtype: pandas.Series
1356         """
1357         return self._input_data
1358
1359     def metadata(self, job, build):
1360         """Getter - metadata
1361
1362         :param job: Job which metadata we want.
1363         :param build: Build which metadata we want.
1364         :type job: str
1365         :type build: str
1366         :returns: Metadata
1367         :rtype: pandas.Series
1368         """
1369
1370         return self.data[job][build][u"metadata"]
1371
1372     def suites(self, job, build):
1373         """Getter - suites
1374
1375         :param job: Job which suites we want.
1376         :param build: Build which suites we want.
1377         :type job: str
1378         :type build: str
1379         :returns: Suites.
1380         :rtype: pandas.Series
1381         """
1382
1383         return self.data[job][str(build)][u"suites"]
1384
1385     def tests(self, job, build):
1386         """Getter - tests
1387
1388         :param job: Job which tests we want.
1389         :param build: Build which tests we want.
1390         :type job: str
1391         :type build: str
1392         :returns: Tests.
1393         :rtype: pandas.Series
1394         """
1395
1396         return self.data[job][build][u"tests"]
1397
1398     def _parse_tests(self, job, build, log):
1399         """Process data from robot output.xml file and return JSON structured
1400         data.
1401
1402         :param job: The name of job which build output data will be processed.
1403         :param build: The build which output data will be processed.
1404         :param log: List of log messages.
1405         :type job: str
1406         :type build: dict
1407         :type log: list of tuples (severity, msg)
1408         :returns: JSON data structure.
1409         :rtype: dict
1410         """
1411
1412         metadata = {
1413             u"job": job,
1414             u"build": build
1415         }
1416
1417         with open(build[u"file-name"], u'r') as data_file:
1418             try:
1419                 result = ExecutionResult(data_file)
1420             except errors.DataError as err:
1421                 log.append(
1422                     (u"ERROR", f"Error occurred while parsing output.xml: "
1423                                f"{repr(err)}")
1424                 )
1425                 return None
1426         checker = ExecutionChecker(metadata, self._cfg.mapping,
1427                                    self._cfg.ignore)
1428         result.visit(checker)
1429
1430         return checker.data
1431
1432     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1433         """Download and parse the input data file.
1434
1435         :param pid: PID of the process executing this method.
1436         :param job: Name of the Jenkins job which generated the processed input
1437             file.
1438         :param build: Information about the Jenkins build which generated the
1439             processed input file.
1440         :param repeat: Repeat the download specified number of times if not
1441             successful.
1442         :type pid: int
1443         :type job: str
1444         :type build: dict
1445         :type repeat: int
1446         """
1447
1448         logs = list()
1449
1450         logs.append(
1451             (u"INFO", f"  Processing the job/build: {job}: {build[u'build']}")
1452         )
1453
1454         state = u"failed"
1455         success = False
1456         data = None
1457         do_repeat = repeat
1458         while do_repeat:
1459             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1460                                                    logs)
1461             if success:
1462                 break
1463             do_repeat -= 1
1464         if not success:
1465             logs.append(
1466                 (u"ERROR",
1467                  f"It is not possible to download the input data file from the "
1468                  f"job {job}, build {build[u'build']}, or it is damaged. "
1469                  f"Skipped.")
1470             )
1471         if success:
1472             logs.append(
1473                 (u"INFO",
1474                  f"    Processing data from the build {build[u'build']} ...")
1475             )
1476             data = self._parse_tests(job, build, logs)
1477             if data is None:
1478                 logs.append(
1479                     (u"ERROR",
1480                      f"Input data file from the job {job}, build "
1481                      f"{build[u'build']} is damaged. Skipped.")
1482                 )
1483             else:
1484                 state = u"processed"
1485
1486             try:
1487                 remove(build[u"file-name"])
1488             except OSError as err:
1489                 logs.append(
1490                     ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
1491                               f"{repr(err)}")
1492                 )
1493
1494         # If the time-period is defined in the specification file, remove all
1495         # files which are outside the time period.
1496         timeperiod = self._cfg.input.get(u"time-period", None)
1497         if timeperiod and data:
1498             now = dt.utcnow()
1499             timeperiod = timedelta(int(timeperiod))
1500             metadata = data.get(u"metadata", None)
1501             if metadata:
1502                 generated = metadata.get(u"generated", None)
1503                 if generated:
1504                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1505                     if (now - generated) > timeperiod:
1506                         # Remove the data and the file:
1507                         state = u"removed"
1508                         data = None
1509                         logs.append(
1510                             (u"INFO",
1511                              f"    The build {job}/{build[u'build']} is "
1512                              f"outdated, will be removed.")
1513                         )
1514         logs.append((u"INFO", u"  Done."))
1515
1516         for level, line in logs:
1517             if level == u"INFO":
1518                 logging.info(line)
1519             elif level == u"ERROR":
1520                 logging.error(line)
1521             elif level == u"DEBUG":
1522                 logging.debug(line)
1523             elif level == u"CRITICAL":
1524                 logging.critical(line)
1525             elif level == u"WARNING":
1526                 logging.warning(line)
1527
1528         return {u"data": data, u"state": state, u"job": job, u"build": build}
1529
1530     def download_and_parse_data(self, repeat=1):
1531         """Download the input data files, parse input data from input files and
1532         store in pandas' Series.
1533
1534         :param repeat: Repeat the download specified number of times if not
1535             successful.
1536         :type repeat: int
1537         """
1538
1539         logging.info(u"Downloading and parsing input files ...")
1540
1541         for job, builds in self._cfg.builds.items():
1542             for build in builds:
1543
1544                 result = self._download_and_parse_build(job, build, repeat)
1545                 build_nr = result[u"build"][u"build"]
1546
1547                 if result[u"data"]:
1548                     data = result[u"data"]
1549                     build_data = pd.Series({
1550                         u"metadata": pd.Series(
1551                             list(data[u"metadata"].values()),
1552                             index=list(data[u"metadata"].keys())
1553                         ),
1554                         u"suites": pd.Series(
1555                             list(data[u"suites"].values()),
1556                             index=list(data[u"suites"].keys())
1557                         ),
1558                         u"tests": pd.Series(
1559                             list(data[u"tests"].values()),
1560                             index=list(data[u"tests"].keys())
1561                         )
1562                     })
1563
1564                     if self._input_data.get(job, None) is None:
1565                         self._input_data[job] = pd.Series()
1566                     self._input_data[job][str(build_nr)] = build_data
1567
1568                     self._cfg.set_input_file_name(
1569                         job, build_nr, result[u"build"][u"file-name"])
1570
1571                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1572
1573                 mem_alloc = \
1574                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1575                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1576
1577         logging.info(u"Done.")
1578
1579     @staticmethod
1580     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1581         """Return the index of character in the string which is the end of tag.
1582
1583         :param tag_filter: The string where the end of tag is being searched.
1584         :param start: The index where the searching is stated.
1585         :param closer: The character which is the tag closer.
1586         :type tag_filter: str
1587         :type start: int
1588         :type closer: str
1589         :returns: The index of the tag closer.
1590         :rtype: int
1591         """
1592
1593         try:
1594             idx_opener = tag_filter.index(closer, start)
1595             return tag_filter.index(closer, idx_opener + 1)
1596         except ValueError:
1597             return None
1598
1599     @staticmethod
1600     def _condition(tag_filter):
1601         """Create a conditional statement from the given tag filter.
1602
1603         :param tag_filter: Filter based on tags from the element specification.
1604         :type tag_filter: str
1605         :returns: Conditional statement which can be evaluated.
1606         :rtype: str
1607         """
1608
1609         index = 0
1610         while True:
1611             index = InputData._end_of_tag(tag_filter, index)
1612             if index is None:
1613                 return tag_filter
1614             index += 1
1615             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1616
1617     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1618                     continue_on_error=False):
1619         """Filter required data from the given jobs and builds.
1620
1621         The output data structure is:
1622
1623         - job 1
1624           - build 1
1625             - test (or suite) 1 ID:
1626               - param 1
1627               - param 2
1628               ...
1629               - param n
1630             ...
1631             - test (or suite) n ID:
1632             ...
1633           ...
1634           - build n
1635         ...
1636         - job n
1637
1638         :param element: Element which will use the filtered data.
1639         :param params: Parameters which will be included in the output. If None,
1640             all parameters are included.
1641         :param data: If not None, this data is used instead of data specified
1642             in the element.
1643         :param data_set: The set of data to be filtered: tests, suites,
1644             metadata.
1645         :param continue_on_error: Continue if there is error while reading the
1646             data. The Item will be empty then
1647         :type element: pandas.Series
1648         :type params: list
1649         :type data: dict
1650         :type data_set: str
1651         :type continue_on_error: bool
1652         :returns: Filtered data.
1653         :rtype pandas.Series
1654         """
1655
1656         try:
1657             if data_set == "suites":
1658                 cond = u"True"
1659             elif element[u"filter"] in (u"all", u"template"):
1660                 cond = u"True"
1661             else:
1662                 cond = InputData._condition(element[u"filter"])
1663             logging.debug(f"   Filter: {cond}")
1664         except KeyError:
1665             logging.error(u"  No filter defined.")
1666             return None
1667
1668         if params is None:
1669             params = element.get(u"parameters", None)
1670             if params:
1671                 params.append(u"type")
1672
1673         data_to_filter = data if data else element[u"data"]
1674         data = pd.Series()
1675         try:
1676             for job, builds in data_to_filter.items():
1677                 data[job] = pd.Series()
1678                 for build in builds:
1679                     data[job][str(build)] = pd.Series()
1680                     try:
1681                         data_dict = dict(
1682                             self.data[job][str(build)][data_set].items())
1683                     except KeyError:
1684                         if continue_on_error:
1685                             continue
1686                         return None
1687
1688                     for test_id, test_data in data_dict.items():
1689                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1690                             data[job][str(build)][test_id] = pd.Series()
1691                             if params is None:
1692                                 for param, val in test_data.items():
1693                                     data[job][str(build)][test_id][param] = val
1694                             else:
1695                                 for param in params:
1696                                     try:
1697                                         data[job][str(build)][test_id][param] =\
1698                                             test_data[param]
1699                                     except KeyError:
1700                                         data[job][str(build)][test_id][param] =\
1701                                             u"No Data"
1702             return data
1703
1704         except (KeyError, IndexError, ValueError) as err:
1705             logging.error(
1706                 f"Missing mandatory parameter in the element specification: "
1707                 f"{repr(err)}"
1708             )
1709             return None
1710         except AttributeError as err:
1711             logging.error(repr(err))
1712             return None
1713         except SyntaxError as err:
1714             logging.error(
1715                 f"The filter {cond} is not correct. Check if all tags are "
1716                 f"enclosed by apostrophes.\n{repr(err)}"
1717             )
1718             return None
1719
1720     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1721                              continue_on_error=False):
1722         """Filter required data from the given jobs and builds.
1723
1724         The output data structure is:
1725
1726         - job 1
1727           - build 1
1728             - test (or suite) 1 ID:
1729               - param 1
1730               - param 2
1731               ...
1732               - param n
1733             ...
1734             - test (or suite) n ID:
1735             ...
1736           ...
1737           - build n
1738         ...
1739         - job n
1740
1741         :param element: Element which will use the filtered data.
1742         :param params: Parameters which will be included in the output. If None,
1743         all parameters are included.
1744         :param data_set: The set of data to be filtered: tests, suites,
1745         metadata.
1746         :param continue_on_error: Continue if there is error while reading the
1747         data. The Item will be empty then
1748         :type element: pandas.Series
1749         :type params: list
1750         :type data_set: str
1751         :type continue_on_error: bool
1752         :returns: Filtered data.
1753         :rtype pandas.Series
1754         """
1755
1756         include = element.get(u"include", None)
1757         if not include:
1758             logging.warning(u"No tests to include, skipping the element.")
1759             return None
1760
1761         if params is None:
1762             params = element.get(u"parameters", None)
1763             if params:
1764                 params.append(u"type")
1765
1766         data = pd.Series()
1767         try:
1768             for job, builds in element[u"data"].items():
1769                 data[job] = pd.Series()
1770                 for build in builds:
1771                     data[job][str(build)] = pd.Series()
1772                     for test in include:
1773                         try:
1774                             reg_ex = re.compile(str(test).lower())
1775                             for test_id in self.data[job][
1776                                     str(build)][data_set].keys():
1777                                 if re.match(reg_ex, str(test_id).lower()):
1778                                     test_data = self.data[job][
1779                                         str(build)][data_set][test_id]
1780                                     data[job][str(build)][test_id] = pd.Series()
1781                                     if params is None:
1782                                         for param, val in test_data.items():
1783                                             data[job][str(build)][test_id]\
1784                                                 [param] = val
1785                                     else:
1786                                         for param in params:
1787                                             try:
1788                                                 data[job][str(build)][
1789                                                     test_id][param] = \
1790                                                     test_data[param]
1791                                             except KeyError:
1792                                                 data[job][str(build)][
1793                                                     test_id][param] = u"No Data"
1794                         except KeyError as err:
1795                             logging.error(repr(err))
1796                             if continue_on_error:
1797                                 continue
1798                             return None
1799             return data
1800
1801         except (KeyError, IndexError, ValueError) as err:
1802             logging.error(
1803                 f"Missing mandatory parameter in the element "
1804                 f"specification: {repr(err)}"
1805             )
1806             return None
1807         except AttributeError as err:
1808             logging.error(repr(err))
1809             return None
1810
1811     @staticmethod
1812     def merge_data(data):
1813         """Merge data from more jobs and builds to a simple data structure.
1814
1815         The output data structure is:
1816
1817         - test (suite) 1 ID:
1818           - param 1
1819           - param 2
1820           ...
1821           - param n
1822         ...
1823         - test (suite) n ID:
1824         ...
1825
1826         :param data: Data to merge.
1827         :type data: pandas.Series
1828         :returns: Merged data.
1829         :rtype: pandas.Series
1830         """
1831
1832         logging.info(u"    Merging data ...")
1833
1834         merged_data = pd.Series()
1835         for builds in data.values:
1836             for item in builds.values:
1837                 for item_id, item_data in item.items():
1838                     merged_data[item_id] = item_data
1839
1840         return merged_data
1841
1842     def print_all_oper_data(self):
1843         """Print all operational data to console.
1844         """
1845
1846         tbl_hdr = (
1847             u"Name",
1848             u"Nr of Vectors",
1849             u"Nr of Packets",
1850             u"Suspends",
1851             u"Cycles per Packet",
1852             u"Average Vector Size"
1853         )
1854
1855         for job in self._input_data.values:
1856             for build in job.values:
1857                 for test_id, test_data in build[u"tests"].items():
1858                     print(f"{test_id}")
1859                     if test_data.get(u"show-run", None) is None:
1860                         continue
1861                     for dut_name, data in test_data[u"show-run"].items():
1862                         if data.get(u"threads", None) is None:
1863                             continue
1864                         print(f"Host IP: {data.get(u'host', '')}, "
1865                               f"Socket: {data.get(u'socket', '')}")
1866                         for thread_nr, thread in data[u"threads"].items():
1867                             txt_table = prettytable.PrettyTable(tbl_hdr)
1868                             avg = 0.0
1869                             for row in thread:
1870                                 txt_table.add_row(row)
1871                                 avg += row[-1]
1872                             if len(thread) == 0:
1873                                 avg = u""
1874                             else:
1875                                 avg = f", Average Vector Size per Node: " \
1876                                       f"{(avg / len(thread)):.2f}"
1877                             th_name = u"main" if thread_nr == 0 \
1878                                 else f"worker_{thread_nr}"
1879                             print(f"{dut_name}, {th_name}{avg}")
1880                             txt_table.float_format = u".2"
1881                             txt_table.align = u"r"
1882                             txt_table.align[u"Name"] = u"l"
1883                             print(f"{txt_table.get_string()}\n")