resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove, walk, listdir
  29 from os.path import isfile, isdir, join
  30 from datetime import datetime as dt
  31 from datetime import timedelta
  32 from json import loads
  33 from json.decoder import JSONDecodeError
  34
  35 import hdrh.histogram
  36 import hdrh.codec
  37 import prettytable
  38 import pandas as pd
  39
  40 from robot.api import ExecutionResult, ResultVisitor
  41 from robot import errors
  42
  43 from resources.libraries.python import jumpavg
  44 from input_data_files import download_and_unzip_data_file
  45 from pal_errors import PresentationError
  46
  47
  48 # Separator used in file names
  49 SEPARATOR = u"__"
  50
  51
  52 class ExecutionChecker(ResultVisitor):
  53     """Class to traverse through the test suite structure.
  54
  55     The functionality implemented in this class generates a json structure:
  56
  57     Performance tests:
  58
  59     {
  60         "metadata": {
  61             "generated": "Timestamp",
  62             "version": "SUT version",
  63             "job": "Jenkins job name",
  64             "build": "Information about the build"
  65         },
  66         "suites": {
  67             "Suite long name 1": {
  68                 "name": Suite name,
  69                 "doc": "Suite 1 documentation",
  70                 "parent": "Suite 1 parent",
  71                 "level": "Level of the suite in the suite hierarchy"
  72             }
  73             "Suite long name N": {
  74                 "name": Suite name,
  75                 "doc": "Suite N documentation",
  76                 "parent": "Suite 2 parent",
  77                 "level": "Level of the suite in the suite hierarchy"
  78             }
  79         }
  80         "tests": {
  81             # NDRPDR tests:
  82             "ID": {
  83                 "name": "Test name",
  84                 "parent": "Name of the parent of the test",
  85                 "doc": "Test documentation",
  86                 "msg": "Test message",
  87                 "conf-history": "DUT1 and DUT2 VAT History",
  88                 "show-run": "Show Run",
  89                 "tags": ["tag 1", "tag 2", "tag n"],
  90                 "type": "NDRPDR",
  91                 "status": "PASS" | "FAIL",
  92                 "throughput": {
  93                     "NDR": {
  94                         "LOWER": float,
  95                         "UPPER": float
  96                     },
  97                     "PDR": {
  98                         "LOWER": float,
  99                         "UPPER": float
 100                     }
 101                 },
 102                 "latency": {
 103                     "NDR": {
 104                         "direction1": {
 105                             "min": float,
 106                             "avg": float,
 107                             "max": float,
 108                             "hdrh": str
 109                         },
 110                         "direction2": {
 111                             "min": float,
 112                             "avg": float,
 113                             "max": float,
 114                             "hdrh": str
 115                         }
 116                     },
 117                     "PDR": {
 118                         "direction1": {
 119                             "min": float,
 120                             "avg": float,
 121                             "max": float,
 122                             "hdrh": str
 123                         },
 124                         "direction2": {
 125                             "min": float,
 126                             "avg": float,
 127                             "max": float,
 128                             "hdrh": str
 129                         }
 130                     }
 131                 }
 132             }
 133
 134             # TCP tests:
 135             "ID": {
 136                 "name": "Test name",
 137                 "parent": "Name of the parent of the test",
 138                 "doc": "Test documentation",
 139                 "msg": "Test message",
 140                 "tags": ["tag 1", "tag 2", "tag n"],
 141                 "type": "TCP",
 142                 "status": "PASS" | "FAIL",
 143                 "result": int
 144             }
 145
 146             # MRR, BMRR tests:
 147             "ID": {
 148                 "name": "Test name",
 149                 "parent": "Name of the parent of the test",
 150                 "doc": "Test documentation",
 151                 "msg": "Test message",
 152                 "tags": ["tag 1", "tag 2", "tag n"],
 153                 "type": "MRR" | "BMRR",
 154                 "status": "PASS" | "FAIL",
 155                 "result": {
 156                     "receive-rate": float,
 157                     # Average of a list, computed using AvgStdevStats.
 158                     # In CSIT-1180, replace with List[float].
 159                 }
 160             }
 161
 162             "ID" {
 163                 # next test
 164             }
 165         }
 166     }
 167
 168
 169     Functional tests:
 170
 171     {
 172         "metadata": {  # Optional
 173             "version": "VPP version",
 174             "job": "Jenkins job name",
 175             "build": "Information about the build"
 176         },
 177         "suites": {
 178             "Suite name 1": {
 179                 "doc": "Suite 1 documentation",
 180                 "parent": "Suite 1 parent",
 181                 "level": "Level of the suite in the suite hierarchy"
 182             }
 183             "Suite name N": {
 184                 "doc": "Suite N documentation",
 185                 "parent": "Suite 2 parent",
 186                 "level": "Level of the suite in the suite hierarchy"
 187             }
 188         }
 189         "tests": {
 190             "ID": {
 191                 "name": "Test name",
 192                 "parent": "Name of the parent of the test",
 193                 "doc": "Test documentation"
 194                 "msg": "Test message"
 195                 "tags": ["tag 1", "tag 2", "tag n"],
 196                 "conf-history": "DUT1 and DUT2 VAT History"
 197                 "show-run": "Show Run"
 198                 "status": "PASS" | "FAIL"
 199             },
 200             "ID" {
 201                 # next test
 202             }
 203         }
 204     }
 205
 206     .. note:: ID is the lowercase full path to the test.
 207     """
 208
 209     REGEX_PLR_RATE = re.compile(
 210         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 211         r'PLRsearch upper bound::?\s(\d+.\d+)'
 212     )
 213     REGEX_NDRPDR_RATE = re.compile(
 214         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 215         r'NDR_UPPER:\s(\d+.\d+).*\n'
 216         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 217         r'PDR_UPPER:\s(\d+.\d+)'
 218     )
 219     REGEX_PERF_MSG_INFO = re.compile(
 220         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 221         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 222         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 223         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 224         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 225     )
 226     REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
 227
 228     # TODO: Remove when not needed
 229     REGEX_NDRPDR_LAT_BASE = re.compile(
 230         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 231         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 232     )
 233     REGEX_NDRPDR_LAT = re.compile(
 234         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 235         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 236         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 237         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 238         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 239         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 240     )
 241     # TODO: Remove when not needed
 242     REGEX_NDRPDR_LAT_LONG = re.compile(
 243         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 244         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 245         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 246         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 247         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 248         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 249         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 250         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 251         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 252     )
 253     REGEX_VERSION_VPP = re.compile(
 254         r"(return STDOUT Version:\s*|"
 255         r"VPP Version:\s*|VPP version:\s*)(.*)"
 256     )
 257     REGEX_VERSION_DPDK = re.compile(
 258         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 259     )
 260     REGEX_TCP = re.compile(
 261         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 262     )
 263     REGEX_MRR = re.compile(
 264         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 265         r'tx\s(\d*),\srx\s(\d*)'
 266     )
 267     REGEX_BMRR = re.compile(
 268         r'Maximum Receive Rate trial results'
 269         r' in packets per second: \[(.*)\]'
 270     )
 271     REGEX_RECONF_LOSS = re.compile(
 272         r'Packets lost due to reconfig: (\d*)'
 273     )
 274     REGEX_RECONF_TIME = re.compile(
 275         r'Implied time lost: (\d*.[\de-]*)'
 276     )
 277     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 278
 279     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 280
 281     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 282
 283     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 284
 285     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 286
 287     def __init__(self, metadata, mapping, ignore):
 288         """Initialisation.
 289
 290         :param metadata: Key-value pairs to be included in "metadata" part of
 291             JSON structure.
 292         :param mapping: Mapping of the old names of test cases to the new
 293             (actual) one.
 294         :param ignore: List of TCs to be ignored.
 295         :type metadata: dict
 296         :type mapping: dict
 297         :type ignore: list
 298         """
 299
 300         # Type of message to parse out from the test messages
 301         self._msg_type = None
 302
 303         # VPP version
 304         self._version = None
 305
 306         # Timestamp
 307         self._timestamp = None
 308
 309         # Testbed. The testbed is identified by TG node IP address.
 310         self._testbed = None
 311
 312         # Mapping of TCs long names
 313         self._mapping = mapping
 314
 315         # Ignore list
 316         self._ignore = ignore
 317
 318         # Number of PAPI History messages found:
 319         # 0 - no message
 320         # 1 - PAPI History of DUT1
 321         # 2 - PAPI History of DUT2
 322         self._conf_history_lookup_nr = 0
 323
 324         self._sh_run_counter = 0
 325
 326         # Test ID of currently processed test- the lowercase full path to the
 327         # test
 328         self._test_id = None
 329
 330         # The main data structure
 331         self._data = {
 332             u"metadata": OrderedDict(),
 333             u"suites": OrderedDict(),
 334             u"tests": OrderedDict()
 335         }
 336
 337         # Save the provided metadata
 338         for key, val in metadata.items():
 339             self._data[u"metadata"][key] = val
 340
 341         # Dictionary defining the methods used to parse different types of
 342         # messages
 343         self.parse_msg = {
 344             u"timestamp": self._get_timestamp,
 345             u"vpp-version": self._get_vpp_version,
 346             u"dpdk-version": self._get_dpdk_version,
 347             # TODO: Remove when not needed:
 348             u"teardown-vat-history": self._get_vat_history,
 349             u"teardown-papi-history": self._get_papi_history,
 350             u"test-show-runtime": self._get_show_run,
 351             u"testbed": self._get_testbed
 352         }
 353
 354     @property
 355     def data(self):
 356         """Getter - Data parsed from the XML file.
 357
 358         :returns: Data parsed from the XML file.
 359         :rtype: dict
 360         """
 361         return self._data
 362
 363     def _get_data_from_mrr_test_msg(self, msg):
 364         """Get info from message of MRR performance tests.
 365
 366         :param msg: Message to be processed.
 367         :type msg: str
 368         :returns: Processed message or original message if a problem occurs.
 369         :rtype: str
 370         """
 371
 372         groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
 373         if not groups or groups.lastindex != 1:
 374             return u"Test Failed."
 375
 376         try:
 377             data = groups.group(1).split(u", ")
 378         except (AttributeError, IndexError, ValueError, KeyError):
 379             return u"Test Failed."
 380
 381         out_str = u"["
 382         try:
 383             for item in data:
 384                 out_str += f"{(float(item) / 1e6):.2f}, "
 385             return out_str[:-2] + u"]"
 386         except (AttributeError, IndexError, ValueError, KeyError):
 387             return u"Test Failed."
 388
 389     def _get_data_from_perf_test_msg(self, msg):
 390         """Get info from message of NDRPDR performance tests.
 391
 392         :param msg: Message to be processed.
 393         :type msg: str
 394         :returns: Processed message or original message if a problem occurs.
 395         :rtype: str
 396         """
 397
 398         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 399         if not groups or groups.lastindex != 10:
 400             return u"Test Failed."
 401
 402         try:
 403             data = {
 404                 u"ndr_low": float(groups.group(1)),
 405                 u"ndr_low_b": float(groups.group(2)),
 406                 u"pdr_low": float(groups.group(3)),
 407                 u"pdr_low_b": float(groups.group(4)),
 408                 u"pdr_lat_90_1": groups.group(5),
 409                 u"pdr_lat_90_2": groups.group(6),
 410                 u"pdr_lat_50_1": groups.group(7),
 411                 u"pdr_lat_50_2": groups.group(8),
 412                 u"pdr_lat_10_1": groups.group(9),
 413                 u"pdr_lat_10_2": groups.group(10),
 414             }
 415         except (AttributeError, IndexError, ValueError, KeyError):
 416             return u"Test Failed."
 417
 418         def _process_lat(in_str_1, in_str_2):
 419             """Extract min, avg, max values from latency string.
 420
 421             :param in_str_1: Latency string for one direction produced by robot
 422                 framework.
 423             :param in_str_2: Latency string for second direction produced by
 424                 robot framework.
 425             :type in_str_1: str
 426             :type in_str_2: str
 427             :returns: Processed latency string or None if a problem occurs.
 428             :rtype: tuple
 429             """
 430             in_list_1 = in_str_1.split('/', 3)
 431             in_list_2 = in_str_2.split('/', 3)
 432
 433             if len(in_list_1) != 4 and len(in_list_2) != 4:
 434                 return None
 435
 436             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 437             try:
 438                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 439             except hdrh.codec.HdrLengthException:
 440                 return None
 441
 442             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 443             try:
 444                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 445             except hdrh.codec.HdrLengthException:
 446                 return None
 447
 448             if hdr_lat_1 and hdr_lat_2:
 449                 hdr_lat = (
 450                     hdr_lat_1.get_value_at_percentile(50.0),
 451                     hdr_lat_1.get_value_at_percentile(90.0),
 452                     hdr_lat_1.get_value_at_percentile(99.0),
 453                     hdr_lat_2.get_value_at_percentile(50.0),
 454                     hdr_lat_2.get_value_at_percentile(90.0),
 455                     hdr_lat_2.get_value_at_percentile(99.0)
 456                 )
 457
 458                 if all(hdr_lat):
 459                     return hdr_lat
 460
 461             return None
 462
 463         try:
 464             out_msg = (
 465                 f"1. {(data[u'ndr_low'] / 1e6):5.2f}      "
 466                 f"{data[u'ndr_low_b']:5.2f}"
 467                 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f}      "
 468                 f"{data[u'pdr_low_b']:5.2f}"
 469             )
 470             latency = (
 471                 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
 472                 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
 473                 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
 474             )
 475             if all(latency):
 476                 max_len = len(str(max((max(item) for item in latency))))
 477                 max_len = 4 if max_len < 4 else max_len
 478
 479                 for idx, lat in enumerate(latency):
 480                     if not idx:
 481                         out_msg += u"\n"
 482                     out_msg += (
 483                         f"\n{idx + 3}. "
 484                         f"{lat[0]:{max_len}d} "
 485                         f"{lat[1]:{max_len}d} "
 486                         f"{lat[2]:{max_len}d}      "
 487                         f"{lat[3]:{max_len}d} "
 488                         f"{lat[4]:{max_len}d} "
 489                         f"{lat[5]:{max_len}d} "
 490                     )
 491
 492             return out_msg
 493
 494         except (AttributeError, IndexError, ValueError, KeyError):
 495             return u"Test Failed."
 496
 497     def _get_testbed(self, msg):
 498         """Called when extraction of testbed IP is required.
 499         The testbed is identified by TG node IP address.
 500
 501         :param msg: Message to process.
 502         :type msg: Message
 503         :returns: Nothing.
 504         """
 505
 506         if msg.message.count(u"Setup of TG node") or \
 507                 msg.message.count(u"Setup of node TG host"):
 508             reg_tg_ip = re.compile(
 509                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 510             try:
 511                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 512             except (KeyError, ValueError, IndexError, AttributeError):
 513                 pass
 514             finally:
 515                 self._data[u"metadata"][u"testbed"] = self._testbed
 516                 self._msg_type = None
 517
 518     def _get_vpp_version(self, msg):
 519         """Called when extraction of VPP version is required.
 520
 521         :param msg: Message to process.
 522         :type msg: Message
 523         :returns: Nothing.
 524         """
 525
 526         if msg.message.count(u"return STDOUT Version:") or \
 527             msg.message.count(u"VPP Version:") or \
 528             msg.message.count(u"VPP version:"):
 529             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 530                                 group(2))
 531             self._data[u"metadata"][u"version"] = self._version
 532             self._msg_type = None
 533
 534     def _get_dpdk_version(self, msg):
 535         """Called when extraction of DPDK version is required.
 536
 537         :param msg: Message to process.
 538         :type msg: Message
 539         :returns: Nothing.
 540         """
 541
 542         if msg.message.count(u"DPDK Version:"):
 543             try:
 544                 self._version = str(re.search(
 545                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 546                 self._data[u"metadata"][u"version"] = self._version
 547             except IndexError:
 548                 pass
 549             finally:
 550                 self._msg_type = None
 551
 552     def _get_timestamp(self, msg):
 553         """Called when extraction of timestamp is required.
 554
 555         :param msg: Message to process.
 556         :type msg: Message
 557         :returns: Nothing.
 558         """
 559
 560         self._timestamp = msg.timestamp[:14]
 561         self._data[u"metadata"][u"generated"] = self._timestamp
 562         self._msg_type = None
 563
 564     def _get_vat_history(self, msg):
 565         """Called when extraction of VAT command history is required.
 566
 567         TODO: Remove when not needed.
 568
 569         :param msg: Message to process.
 570         :type msg: Message
 571         :returns: Nothing.
 572         """
 573         if msg.message.count(u"VAT command history:"):
 574             self._conf_history_lookup_nr += 1
 575             if self._conf_history_lookup_nr == 1:
 576                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 577             else:
 578                 self._msg_type = None
 579             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 580                           r"VAT command history:", u"",
 581                           msg.message, count=1).replace(u'\n', u' |br| ').\
 582                 replace(u'"', u"'")
 583
 584             self._data[u"tests"][self._test_id][u"conf-history"] += (
 585                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 586             )
 587
 588     def _get_papi_history(self, msg):
 589         """Called when extraction of PAPI command history is required.
 590
 591         :param msg: Message to process.
 592         :type msg: Message
 593         :returns: Nothing.
 594         """
 595         if msg.message.count(u"PAPI command history:"):
 596             self._conf_history_lookup_nr += 1
 597             if self._conf_history_lookup_nr == 1:
 598                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 599             else:
 600                 self._msg_type = None
 601             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 602                           r"PAPI command history:", u"",
 603                           msg.message, count=1).replace(u'\n', u' |br| ').\
 604                 replace(u'"', u"'")
 605             self._data[u"tests"][self._test_id][u"conf-history"] += (
 606                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 607             )
 608
 609     def _get_show_run(self, msg):
 610         """Called when extraction of VPP operational data (output of CLI command
 611         Show Runtime) is required.
 612
 613         :param msg: Message to process.
 614         :type msg: Message
 615         :returns: Nothing.
 616         """
 617
 618         if not msg.message.count(u"stats runtime"):
 619             return
 620
 621         # Temporary solution
 622         if self._sh_run_counter > 1:
 623             return
 624
 625         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 626             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 627
 628         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 629         if not groups:
 630             return
 631         try:
 632             host = groups.group(1)
 633         except (AttributeError, IndexError):
 634             host = u""
 635         try:
 636             sock = groups.group(2)
 637         except (AttributeError, IndexError):
 638             sock = u""
 639
 640         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 641                         replace(u"'", u'"').replace(u'b"', u'"').
 642                         replace(u'u"', u'"').split(u":", 1)[1])
 643
 644         try:
 645             threads_nr = len(runtime[0][u"clocks"])
 646         except (IndexError, KeyError):
 647             return
 648
 649         dut = u"DUT{nr}".format(
 650             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 651
 652         oper = {
 653             u"host": host,
 654             u"socket": sock,
 655             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 656         }
 657
 658         for item in runtime:
 659             for idx in range(threads_nr):
 660                 if item[u"vectors"][idx] > 0:
 661                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 662                 elif item[u"calls"][idx] > 0:
 663                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 664                 elif item[u"suspends"][idx] > 0:
 665                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 666                 else:
 667                     clocks = 0.0
 668
 669                 if item[u"calls"][idx] > 0:
 670                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 671                 else:
 672                     vectors_call = 0.0
 673
 674                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 675                         int(item[u"suspends"][idx]):
 676                     oper[u"threads"][idx].append([
 677                         item[u"name"],
 678                         item[u"calls"][idx],
 679                         item[u"vectors"][idx],
 680                         item[u"suspends"][idx],
 681                         clocks,
 682                         vectors_call
 683                     ])
 684
 685         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 686
 687     def _get_ndrpdr_throughput(self, msg):
 688         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 689         message.
 690
 691         :param msg: The test message to be parsed.
 692         :type msg: str
 693         :returns: Parsed data as a dict and the status (PASS/FAIL).
 694         :rtype: tuple(dict, str)
 695         """
 696
 697         throughput = {
 698             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 699             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 700         }
 701         status = u"FAIL"
 702         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 703
 704         if groups is not None:
 705             try:
 706                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 707                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 708                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 709                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 710                 status = u"PASS"
 711             except (IndexError, ValueError):
 712                 pass
 713
 714         return throughput, status
 715
 716     def _get_plr_throughput(self, msg):
 717         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 718         message.
 719
 720         :param msg: The test message to be parsed.
 721         :type msg: str
 722         :returns: Parsed data as a dict and the status (PASS/FAIL).
 723         :rtype: tuple(dict, str)
 724         """
 725
 726         throughput = {
 727             u"LOWER": -1.0,
 728             u"UPPER": -1.0
 729         }
 730         status = u"FAIL"
 731         groups = re.search(self.REGEX_PLR_RATE, msg)
 732
 733         if groups is not None:
 734             try:
 735                 throughput[u"LOWER"] = float(groups.group(1))
 736                 throughput[u"UPPER"] = float(groups.group(2))
 737                 status = u"PASS"
 738             except (IndexError, ValueError):
 739                 pass
 740
 741         return throughput, status
 742
 743     def _get_ndrpdr_latency(self, msg):
 744         """Get LATENCY from the test message.
 745
 746         :param msg: The test message to be parsed.
 747         :type msg: str
 748         :returns: Parsed data as a dict and the status (PASS/FAIL).
 749         :rtype: tuple(dict, str)
 750         """
 751         latency_default = {
 752             u"min": -1.0,
 753             u"avg": -1.0,
 754             u"max": -1.0,
 755             u"hdrh": u""
 756         }
 757         latency = {
 758             u"NDR": {
 759                 u"direction1": copy.copy(latency_default),
 760                 u"direction2": copy.copy(latency_default)
 761             },
 762             u"PDR": {
 763                 u"direction1": copy.copy(latency_default),
 764                 u"direction2": copy.copy(latency_default)
 765             },
 766             u"LAT0": {
 767                 u"direction1": copy.copy(latency_default),
 768                 u"direction2": copy.copy(latency_default)
 769             },
 770             u"PDR10": {
 771                 u"direction1": copy.copy(latency_default),
 772                 u"direction2": copy.copy(latency_default)
 773             },
 774             u"PDR50": {
 775                 u"direction1": copy.copy(latency_default),
 776                 u"direction2": copy.copy(latency_default)
 777             },
 778             u"PDR90": {
 779                 u"direction1": copy.copy(latency_default),
 780                 u"direction2": copy.copy(latency_default)
 781             },
 782         }
 783
 784         # TODO: Rewrite when long and base are not needed
 785         groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
 786         if groups is None:
 787             groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 788         if groups is None:
 789             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 790         if groups is None:
 791             return latency, u"FAIL"
 792
 793         def process_latency(in_str):
 794             """Return object with parsed latency values.
 795
 796             TODO: Define class for the return type.
 797
 798             :param in_str: Input string, min/avg/max/hdrh format.
 799             :type in_str: str
 800             :returns: Dict with corresponding keys, except hdrh float values.
 801             :rtype dict:
 802             :throws IndexError: If in_str does not have enough substrings.
 803             :throws ValueError: If a substring does not convert to float.
 804             """
 805             in_list = in_str.split('/', 3)
 806
 807             rval = {
 808                 u"min": float(in_list[0]),
 809                 u"avg": float(in_list[1]),
 810                 u"max": float(in_list[2]),
 811                 u"hdrh": u""
 812             }
 813
 814             if len(in_list) == 4:
 815                 rval[u"hdrh"] = str(in_list[3])
 816
 817             return rval
 818
 819         try:
 820             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 821             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 822             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 823             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 824             if groups.lastindex == 4:
 825                 return latency, u"PASS"
 826         except (IndexError, ValueError):
 827             pass
 828
 829         try:
 830             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 831             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 832             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 833             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 834             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 835             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 836             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 837             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 838             if groups.lastindex == 12:
 839                 return latency, u"PASS"
 840         except (IndexError, ValueError):
 841             pass
 842
 843         # TODO: Remove when not needed
 844         latency[u"NDR10"] = {
 845             u"direction1": copy.copy(latency_default),
 846             u"direction2": copy.copy(latency_default)
 847         }
 848         latency[u"NDR50"] = {
 849             u"direction1": copy.copy(latency_default),
 850             u"direction2": copy.copy(latency_default)
 851         }
 852         latency[u"NDR90"] = {
 853             u"direction1": copy.copy(latency_default),
 854             u"direction2": copy.copy(latency_default)
 855         }
 856         try:
 857             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
 858             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
 859             latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
 860             latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
 861             latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
 862             latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
 863             latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
 864             latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
 865             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
 866             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
 867             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
 868             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
 869             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
 870             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
 871             return latency, u"PASS"
 872         except (IndexError, ValueError):
 873             pass
 874
 875         return latency, u"FAIL"
 876
 877     @staticmethod
 878     def _get_hoststack_data(msg, tags):
 879         """Get data from the hoststack test message.
 880
 881         :param msg: The test message to be parsed.
 882         :param tags: Test tags.
 883         :type msg: str
 884         :type tags: list
 885         :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
 886         :rtype: tuple(dict, str)
 887         """
 888         result = dict()
 889         status = u"FAIL"
 890
 891         msg = msg.replace(u"'", u'"').replace(u" ", u"")
 892         if u"LDPRELOAD" in tags:
 893             try:
 894                 result = loads(msg)
 895                 status = u"PASS"
 896             except JSONDecodeError:
 897                 pass
 898         elif u"VPPECHO" in tags:
 899             try:
 900                 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
 901                 result = dict(
 902                     client=loads(msg_lst[0]),
 903                     server=loads(msg_lst[1])
 904                 )
 905                 status = u"PASS"
 906             except (JSONDecodeError, IndexError):
 907                 pass
 908
 909         return result, status
 910
 911     def visit_suite(self, suite):
 912         """Implements traversing through the suite and its direct children.
 913
 914         :param suite: Suite to process.
 915         :type suite: Suite
 916         :returns: Nothing.
 917         """
 918         if self.start_suite(suite) is not False:
 919             suite.suites.visit(self)
 920             suite.tests.visit(self)
 921             self.end_suite(suite)
 922
 923     def start_suite(self, suite):
 924         """Called when suite starts.
 925
 926         :param suite: Suite to process.
 927         :type suite: Suite
 928         :returns: Nothing.
 929         """
 930
 931         try:
 932             parent_name = suite.parent.name
 933         except AttributeError:
 934             return
 935
 936         doc_str = suite.doc.\
 937             replace(u'"', u"'").\
 938             replace(u'\n', u' ').\
 939             replace(u'\r', u'').\
 940             replace(u'*[', u' |br| *[').\
 941             replace(u"*", u"**").\
 942             replace(u' |br| *[', u'*[', 1)
 943
 944         self._data[u"suites"][suite.longname.lower().
 945                               replace(u'"', u"'").
 946                               replace(u" ", u"_")] = {
 947                                   u"name": suite.name.lower(),
 948                                   u"doc": doc_str,
 949                                   u"parent": parent_name,
 950                                   u"level": len(suite.longname.split(u"."))
 951                               }
 952
 953         suite.keywords.visit(self)
 954
 955     def end_suite(self, suite):
 956         """Called when suite ends.
 957
 958         :param suite: Suite to process.
 959         :type suite: Suite
 960         :returns: Nothing.
 961         """
 962
 963     def visit_test(self, test):
 964         """Implements traversing through the test.
 965
 966         :param test: Test to process.
 967         :type test: Test
 968         :returns: Nothing.
 969         """
 970         if self.start_test(test) is not False:
 971             test.keywords.visit(self)
 972             self.end_test(test)
 973
 974     def start_test(self, test):
 975         """Called when test starts.
 976
 977         :param test: Test to process.
 978         :type test: Test
 979         :returns: Nothing.
 980         """
 981
 982         self._sh_run_counter = 0
 983
 984         longname_orig = test.longname.lower()
 985
 986         # Check the ignore list
 987         if longname_orig in self._ignore:
 988             return
 989
 990         tags = [str(tag) for tag in test.tags]
 991         test_result = dict()
 992
 993         # Change the TC long name and name if defined in the mapping table
 994         longname = self._mapping.get(longname_orig, None)
 995         if longname is not None:
 996             name = longname.split(u'.')[-1]
 997             logging.debug(
 998                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
 999                 f"{name}"
1000             )
1001         else:
1002             longname = longname_orig
1003             name = test.name.lower()
1004
1005         # Remove TC number from the TC long name (backward compatibility):
1006         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1007         # Remove TC number from the TC name (not needed):
1008         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1009
1010         test_result[u"parent"] = test.parent.name.lower()
1011         test_result[u"tags"] = tags
1012         test_result["doc"] = test.doc.\
1013             replace(u'"', u"'").\
1014             replace(u'\n', u' ').\
1015             replace(u'\r', u'').\
1016             replace(u'[', u' |br| [').\
1017             replace(u' |br| [', u'[', 1)
1018         test_result[u"type"] = u"FUNC"
1019         test_result[u"status"] = test.status
1020
1021         if test.status == u"PASS":
1022             if u"NDRPDR" in tags:
1023                 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1024                     test.message).replace(u'\n', u' |br| ').\
1025                     replace(u'\r', u'').replace(u'"', u"'")
1026             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1027                 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1028                     test.message).replace(u'\n', u' |br| ').\
1029                     replace(u'\r', u'').replace(u'"', u"'")
1030             else:
1031                 test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\
1032                     replace(u'\r', u'').replace(u'"', u"'")
1033         else:
1034             test_result[u"msg"] = u"Test Failed."
1035
1036         if u"PERFTEST" in tags:
1037             # Replace info about cores (e.g. -1c-) with the info about threads
1038             # and cores (e.g. -1t1c-) in the long test case names and in the
1039             # test case names if necessary.
1040             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1041             if not groups:
1042                 tag_count = 0
1043                 tag_tc = str()
1044                 for tag in test_result[u"tags"]:
1045                     groups = re.search(self.REGEX_TC_TAG, tag)
1046                     if groups:
1047                         tag_count += 1
1048                         tag_tc = tag
1049
1050                 if tag_count == 1:
1051                     self._test_id = re.sub(
1052                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1053                         self._test_id, count=1
1054                     )
1055                     test_result[u"name"] = re.sub(
1056                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1057                         test_result["name"], count=1
1058                     )
1059                 else:
1060                     test_result[u"status"] = u"FAIL"
1061                     self._data[u"tests"][self._test_id] = test_result
1062                     logging.debug(
1063                         f"The test {self._test_id} has no or more than one "
1064                         f"multi-threading tags.\n"
1065                         f"Tags: {test_result[u'tags']}"
1066                     )
1067                     return
1068
1069         if test.status == u"PASS":
1070             if u"NDRPDR" in tags:
1071                 test_result[u"type"] = u"NDRPDR"
1072                 test_result[u"throughput"], test_result[u"status"] = \
1073                     self._get_ndrpdr_throughput(test.message)
1074                 test_result[u"latency"], test_result[u"status"] = \
1075                     self._get_ndrpdr_latency(test.message)
1076             elif u"SOAK" in tags:
1077                 test_result[u"type"] = u"SOAK"
1078                 test_result[u"throughput"], test_result[u"status"] = \
1079                     self._get_plr_throughput(test.message)
1080             elif u"HOSTSTACK" in tags:
1081                 test_result[u"type"] = u"HOSTSTACK"
1082                 test_result[u"result"], test_result[u"status"] = \
1083                     self._get_hoststack_data(test.message, tags)
1084             elif u"TCP" in tags:
1085                 test_result[u"type"] = u"TCP"
1086                 groups = re.search(self.REGEX_TCP, test.message)
1087                 test_result[u"result"] = int(groups.group(2))
1088             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1089                 if u"MRR" in tags:
1090                     test_result[u"type"] = u"MRR"
1091                 else:
1092                     test_result[u"type"] = u"BMRR"
1093
1094                 test_result[u"result"] = dict()
1095                 groups = re.search(self.REGEX_BMRR, test.message)
1096                 if groups is not None:
1097                     items_str = groups.group(1)
1098                     items_float = [float(item.strip()) for item
1099                                    in items_str.split(",")]
1100                     # Use whole list in CSIT-1180.
1101                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1102                     test_result[u"result"][u"receive-rate"] = stats.avg
1103                     test_result[u"result"][u"receive-stdev"] = stats.stdev
1104                 else:
1105                     groups = re.search(self.REGEX_MRR, test.message)
1106                     test_result[u"result"][u"receive-rate"] = \
1107                         float(groups.group(3)) / float(groups.group(1))
1108             elif u"RECONF" in tags:
1109                 test_result[u"type"] = u"RECONF"
1110                 test_result[u"result"] = None
1111                 try:
1112                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1113                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1114                     test_result[u"result"] = {
1115                         u"loss": int(grps_loss.group(1)),
1116                         u"time": float(grps_time.group(1))
1117                     }
1118                 except (AttributeError, IndexError, ValueError, TypeError):
1119                     test_result[u"status"] = u"FAIL"
1120             elif u"DEVICETEST" in tags:
1121                 test_result[u"type"] = u"DEVICETEST"
1122             else:
1123                 test_result[u"status"] = u"FAIL"
1124                 self._data[u"tests"][self._test_id] = test_result
1125                 return
1126
1127         self._data[u"tests"][self._test_id] = test_result
1128
1129     def end_test(self, test):
1130         """Called when test ends.
1131
1132         :param test: Test to process.
1133         :type test: Test
1134         :returns: Nothing.
1135         """
1136
1137     def visit_keyword(self, keyword):
1138         """Implements traversing through the keyword and its child keywords.
1139
1140         :param keyword: Keyword to process.
1141         :type keyword: Keyword
1142         :returns: Nothing.
1143         """
1144         if self.start_keyword(keyword) is not False:
1145             self.end_keyword(keyword)
1146
1147     def start_keyword(self, keyword):
1148         """Called when keyword starts. Default implementation does nothing.
1149
1150         :param keyword: Keyword to process.
1151         :type keyword: Keyword
1152         :returns: Nothing.
1153         """
1154         try:
1155             if keyword.type == u"setup":
1156                 self.visit_setup_kw(keyword)
1157             elif keyword.type == u"teardown":
1158                 self.visit_teardown_kw(keyword)
1159             else:
1160                 self.visit_test_kw(keyword)
1161         except AttributeError:
1162             pass
1163
1164     def end_keyword(self, keyword):
1165         """Called when keyword ends. Default implementation does nothing.
1166
1167         :param keyword: Keyword to process.
1168         :type keyword: Keyword
1169         :returns: Nothing.
1170         """
1171
1172     def visit_test_kw(self, test_kw):
1173         """Implements traversing through the test keyword and its child
1174         keywords.
1175
1176         :param test_kw: Keyword to process.
1177         :type test_kw: Keyword
1178         :returns: Nothing.
1179         """
1180         for keyword in test_kw.keywords:
1181             if self.start_test_kw(keyword) is not False:
1182                 self.visit_test_kw(keyword)
1183                 self.end_test_kw(keyword)
1184
1185     def start_test_kw(self, test_kw):
1186         """Called when test keyword starts. Default implementation does
1187         nothing.
1188
1189         :param test_kw: Keyword to process.
1190         :type test_kw: Keyword
1191         :returns: Nothing.
1192         """
1193         if test_kw.name.count(u"Show Runtime On All Duts") or \
1194                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
1195             self._msg_type = u"test-show-runtime"
1196             self._sh_run_counter += 1
1197         elif test_kw.name.count(u"Install Dpdk Test On All Duts") and \
1198                 not self._version:
1199             self._msg_type = u"dpdk-version"
1200         else:
1201             return
1202         test_kw.messages.visit(self)
1203
1204     def end_test_kw(self, test_kw):
1205         """Called when keyword ends. Default implementation does nothing.
1206
1207         :param test_kw: Keyword to process.
1208         :type test_kw: Keyword
1209         :returns: Nothing.
1210         """
1211
1212     def visit_setup_kw(self, setup_kw):
1213         """Implements traversing through the teardown keyword and its child
1214         keywords.
1215
1216         :param setup_kw: Keyword to process.
1217         :type setup_kw: Keyword
1218         :returns: Nothing.
1219         """
1220         for keyword in setup_kw.keywords:
1221             if self.start_setup_kw(keyword) is not False:
1222                 self.visit_setup_kw(keyword)
1223                 self.end_setup_kw(keyword)
1224
1225     def start_setup_kw(self, setup_kw):
1226         """Called when teardown keyword starts. Default implementation does
1227         nothing.
1228
1229         :param setup_kw: Keyword to process.
1230         :type setup_kw: Keyword
1231         :returns: Nothing.
1232         """
1233         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1234                 and not self._version:
1235             self._msg_type = u"vpp-version"
1236         elif setup_kw.name.count(u"Set Global Variable") \
1237                 and not self._timestamp:
1238             self._msg_type = u"timestamp"
1239         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1240             self._msg_type = u"testbed"
1241         else:
1242             return
1243         setup_kw.messages.visit(self)
1244
1245     def end_setup_kw(self, setup_kw):
1246         """Called when keyword ends. Default implementation does nothing.
1247
1248         :param setup_kw: Keyword to process.
1249         :type setup_kw: Keyword
1250         :returns: Nothing.
1251         """
1252
1253     def visit_teardown_kw(self, teardown_kw):
1254         """Implements traversing through the teardown keyword and its child
1255         keywords.
1256
1257         :param teardown_kw: Keyword to process.
1258         :type teardown_kw: Keyword
1259         :returns: Nothing.
1260         """
1261         for keyword in teardown_kw.keywords:
1262             if self.start_teardown_kw(keyword) is not False:
1263                 self.visit_teardown_kw(keyword)
1264                 self.end_teardown_kw(keyword)
1265
1266     def start_teardown_kw(self, teardown_kw):
1267         """Called when teardown keyword starts
1268
1269         :param teardown_kw: Keyword to process.
1270         :type teardown_kw: Keyword
1271         :returns: Nothing.
1272         """
1273
1274         if teardown_kw.name.count(u"Show Vat History On All Duts"):
1275             # TODO: Remove when not needed:
1276             self._conf_history_lookup_nr = 0
1277             self._msg_type = u"teardown-vat-history"
1278             teardown_kw.messages.visit(self)
1279         elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1280             self._conf_history_lookup_nr = 0
1281             self._msg_type = u"teardown-papi-history"
1282             teardown_kw.messages.visit(self)
1283
1284     def end_teardown_kw(self, teardown_kw):
1285         """Called when keyword ends. Default implementation does nothing.
1286
1287         :param teardown_kw: Keyword to process.
1288         :type teardown_kw: Keyword
1289         :returns: Nothing.
1290         """
1291
1292     def visit_message(self, msg):
1293         """Implements visiting the message.
1294
1295         :param msg: Message to process.
1296         :type msg: Message
1297         :returns: Nothing.
1298         """
1299         if self.start_message(msg) is not False:
1300             self.end_message(msg)
1301
1302     def start_message(self, msg):
1303         """Called when message starts. Get required information from messages:
1304         - VPP version.
1305
1306         :param msg: Message to process.
1307         :type msg: Message
1308         :returns: Nothing.
1309         """
1310         if self._msg_type:
1311             self.parse_msg[self._msg_type](msg)
1312
1313     def end_message(self, msg):
1314         """Called when message ends. Default implementation does nothing.
1315
1316         :param msg: Message to process.
1317         :type msg: Message
1318         :returns: Nothing.
1319         """
1320
1321
1322 class InputData:
1323     """Input data
1324
1325     The data is extracted from output.xml files generated by Jenkins jobs and
1326     stored in pandas' DataFrames.
1327
1328     The data structure:
1329     - job name
1330       - build number
1331         - metadata
1332           (as described in ExecutionChecker documentation)
1333         - suites
1334           (as described in ExecutionChecker documentation)
1335         - tests
1336           (as described in ExecutionChecker documentation)
1337     """
1338
1339     def __init__(self, spec):
1340         """Initialization.
1341
1342         :param spec: Specification.
1343         :type spec: Specification
1344         """
1345
1346         # Specification:
1347         self._cfg = spec
1348
1349         # Data store:
1350         self._input_data = pd.Series()
1351
1352     @property
1353     def data(self):
1354         """Getter - Input data.
1355
1356         :returns: Input data
1357         :rtype: pandas.Series
1358         """
1359         return self._input_data
1360
1361     def metadata(self, job, build):
1362         """Getter - metadata
1363
1364         :param job: Job which metadata we want.
1365         :param build: Build which metadata we want.
1366         :type job: str
1367         :type build: str
1368         :returns: Metadata
1369         :rtype: pandas.Series
1370         """
1371         return self.data[job][build][u"metadata"]
1372
1373     def suites(self, job, build):
1374         """Getter - suites
1375
1376         :param job: Job which suites we want.
1377         :param build: Build which suites we want.
1378         :type job: str
1379         :type build: str
1380         :returns: Suites.
1381         :rtype: pandas.Series
1382         """
1383         return self.data[job][str(build)][u"suites"]
1384
1385     def tests(self, job, build):
1386         """Getter - tests
1387
1388         :param job: Job which tests we want.
1389         :param build: Build which tests we want.
1390         :type job: str
1391         :type build: str
1392         :returns: Tests.
1393         :rtype: pandas.Series
1394         """
1395         return self.data[job][build][u"tests"]
1396
1397     def _parse_tests(self, job, build):
1398         """Process data from robot output.xml file and return JSON structured
1399         data.
1400
1401         :param job: The name of job which build output data will be processed.
1402         :param build: The build which output data will be processed.
1403         :type job: str
1404         :type build: dict
1405         :returns: JSON data structure.
1406         :rtype: dict
1407         """
1408
1409         metadata = {
1410             u"job": job,
1411             u"build": build
1412         }
1413
1414         with open(build[u"file-name"], u'r') as data_file:
1415             try:
1416                 result = ExecutionResult(data_file)
1417             except errors.DataError as err:
1418                 logging.error(
1419                     f"Error occurred while parsing output.xml: {repr(err)}"
1420                 )
1421                 return None
1422         checker = ExecutionChecker(metadata, self._cfg.mapping,
1423                                    self._cfg.ignore)
1424         result.visit(checker)
1425
1426         return checker.data
1427
1428     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1429         """Download and parse the input data file.
1430
1431         :param pid: PID of the process executing this method.
1432         :param job: Name of the Jenkins job which generated the processed input
1433             file.
1434         :param build: Information about the Jenkins build which generated the
1435             processed input file.
1436         :param repeat: Repeat the download specified number of times if not
1437             successful.
1438         :type pid: int
1439         :type job: str
1440         :type build: dict
1441         :type repeat: int
1442         """
1443
1444         logging.info(f"  Processing the job/build: {job}: {build[u'build']}")
1445
1446         state = u"failed"
1447         success = False
1448         data = None
1449         do_repeat = repeat
1450         while do_repeat:
1451             success = download_and_unzip_data_file(self._cfg, job, build, pid)
1452             if success:
1453                 break
1454             do_repeat -= 1
1455         if not success:
1456             logging.error(
1457                 f"It is not possible to download the input data file from the "
1458                 f"job {job}, build {build[u'build']}, or it is damaged. "
1459                 f"Skipped."
1460             )
1461         if success:
1462             logging.info(f"    Processing data from build {build[u'build']}")
1463             data = self._parse_tests(job, build)
1464             if data is None:
1465                 logging.error(
1466                     f"Input data file from the job {job}, build "
1467                     f"{build[u'build']} is damaged. Skipped."
1468                 )
1469             else:
1470                 state = u"processed"
1471
1472             try:
1473                 remove(build[u"file-name"])
1474             except OSError as err:
1475                 logging.error(
1476                     f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1477                 )
1478
1479         # If the time-period is defined in the specification file, remove all
1480         # files which are outside the time period.
1481         is_last = False
1482         timeperiod = self._cfg.input.get(u"time-period", None)
1483         if timeperiod and data:
1484             now = dt.utcnow()
1485             timeperiod = timedelta(int(timeperiod))
1486             metadata = data.get(u"metadata", None)
1487             if metadata:
1488                 generated = metadata.get(u"generated", None)
1489                 if generated:
1490                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1491                     if (now - generated) > timeperiod:
1492                         # Remove the data and the file:
1493                         state = u"removed"
1494                         data = None
1495                         is_last = True
1496                         logging.info(
1497                             f"    The build {job}/{build[u'build']} is "
1498                             f"outdated, will be removed."
1499                         )
1500         logging.info(u"  Done.")
1501
1502         return {
1503             u"data": data,
1504             u"state": state,
1505             u"job": job,
1506             u"build": build,
1507             u"last": is_last
1508         }
1509
1510     def download_and_parse_data(self, repeat=1):
1511         """Download the input data files, parse input data from input files and
1512         store in pandas' Series.
1513
1514         :param repeat: Repeat the download specified number of times if not
1515             successful.
1516         :type repeat: int
1517         """
1518
1519         logging.info(u"Downloading and parsing input files ...")
1520
1521         for job, builds in self._cfg.builds.items():
1522             for build in builds:
1523
1524                 result = self._download_and_parse_build(job, build, repeat)
1525                 if result[u"last"]:
1526                     break
1527                 build_nr = result[u"build"][u"build"]
1528
1529                 if result[u"data"]:
1530                     data = result[u"data"]
1531                     build_data = pd.Series({
1532                         u"metadata": pd.Series(
1533                             list(data[u"metadata"].values()),
1534                             index=list(data[u"metadata"].keys())
1535                         ),
1536                         u"suites": pd.Series(
1537                             list(data[u"suites"].values()),
1538                             index=list(data[u"suites"].keys())
1539                         ),
1540                         u"tests": pd.Series(
1541                             list(data[u"tests"].values()),
1542                             index=list(data[u"tests"].keys())
1543                         )
1544                     })
1545
1546                     if self._input_data.get(job, None) is None:
1547                         self._input_data[job] = pd.Series()
1548                     self._input_data[job][str(build_nr)] = build_data
1549
1550                     self._cfg.set_input_file_name(
1551                         job, build_nr, result[u"build"][u"file-name"])
1552
1553                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1554
1555                 mem_alloc = \
1556                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1557                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1558
1559         logging.info(u"Done.")
1560
1561     def process_local_file(self, local_file, job=u"local", build_nr=1,
1562                            replace=True):
1563         """Process local XML file given as a command-line parameter.
1564
1565         :param local_file: The file to process.
1566         :param job: Job name.
1567         :param build_nr: Build number.
1568         :param replace: If True, the information about jobs and builds is
1569             replaced by the new one, otherwise the new jobs and builds are
1570             added.
1571         :type local_file: str
1572         :type job: str
1573         :type build_nr: int
1574         :type replace: bool
1575         :raises: PresentationError if an error occurs.
1576         """
1577         if not isfile(local_file):
1578             raise PresentationError(f"The file {local_file} does not exist.")
1579
1580         try:
1581             build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1582         except (IndexError, ValueError):
1583             pass
1584
1585         build = {
1586             u"build": build_nr,
1587             u"status": u"failed",
1588             u"file-name": local_file
1589         }
1590         if replace:
1591             self._cfg.builds = dict()
1592         self._cfg.add_build(job, build)
1593
1594         logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1595         data = self._parse_tests(job, build)
1596         if data is None:
1597             raise PresentationError(
1598                 f"Error occurred while parsing the file {local_file}"
1599             )
1600
1601         build_data = pd.Series({
1602             u"metadata": pd.Series(
1603                 list(data[u"metadata"].values()),
1604                 index=list(data[u"metadata"].keys())
1605             ),
1606             u"suites": pd.Series(
1607                 list(data[u"suites"].values()),
1608                 index=list(data[u"suites"].keys())
1609             ),
1610             u"tests": pd.Series(
1611                 list(data[u"tests"].values()),
1612                 index=list(data[u"tests"].keys())
1613             )
1614         })
1615
1616         if self._input_data.get(job, None) is None:
1617             self._input_data[job] = pd.Series()
1618         self._input_data[job][str(build_nr)] = build_data
1619
1620         self._cfg.set_input_state(job, build_nr, u"processed")
1621
1622     def process_local_directory(self, local_dir, replace=True):
1623         """Process local directory with XML file(s). The directory is processed
1624         as a 'job' and the XML files in it as builds.
1625         If the given directory contains only sub-directories, these
1626         sub-directories processed as jobs and corresponding XML files as builds
1627         of their job.
1628
1629         :param local_dir: Local directory to process.
1630         :param replace: If True, the information about jobs and builds is
1631             replaced by the new one, otherwise the new jobs and builds are
1632             added.
1633         :type local_dir: str
1634         :type replace: bool
1635         """
1636         if not isdir(local_dir):
1637             raise PresentationError(
1638                 f"The directory {local_dir} does not exist."
1639             )
1640
1641         # Check if the given directory includes only files, or only directories
1642         _, dirnames, filenames = next(walk(local_dir))
1643
1644         if filenames and not dirnames:
1645             filenames.sort()
1646             # local_builds:
1647             # key: dir (job) name, value: list of file names (builds)
1648             local_builds = {
1649                 local_dir: [join(local_dir, name) for name in filenames]
1650             }
1651
1652         elif dirnames and not filenames:
1653             dirnames.sort()
1654             # local_builds:
1655             # key: dir (job) name, value: list of file names (builds)
1656             local_builds = dict()
1657             for dirname in dirnames:
1658                 builds = [
1659                     join(local_dir, dirname, name)
1660                     for name in listdir(join(local_dir, dirname))
1661                     if isfile(join(local_dir, dirname, name))
1662                 ]
1663                 if builds:
1664                     local_builds[dirname] = sorted(builds)
1665
1666         elif not filenames and not dirnames:
1667             raise PresentationError(f"The directory {local_dir} is empty.")
1668         else:
1669             raise PresentationError(
1670                 f"The directory {local_dir} can include only files or only "
1671                 f"directories, not both.\nThe directory {local_dir} includes "
1672                 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1673             )
1674
1675         if replace:
1676             self._cfg.builds = dict()
1677
1678         for job, files in local_builds.items():
1679             for idx, local_file in enumerate(files):
1680                 self.process_local_file(local_file, job, idx + 1, replace=False)
1681
1682     @staticmethod
1683     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1684         """Return the index of character in the string which is the end of tag.
1685
1686         :param tag_filter: The string where the end of tag is being searched.
1687         :param start: The index where the searching is stated.
1688         :param closer: The character which is the tag closer.
1689         :type tag_filter: str
1690         :type start: int
1691         :type closer: str
1692         :returns: The index of the tag closer.
1693         :rtype: int
1694         """
1695         try:
1696             idx_opener = tag_filter.index(closer, start)
1697             return tag_filter.index(closer, idx_opener + 1)
1698         except ValueError:
1699             return None
1700
1701     @staticmethod
1702     def _condition(tag_filter):
1703         """Create a conditional statement from the given tag filter.
1704
1705         :param tag_filter: Filter based on tags from the element specification.
1706         :type tag_filter: str
1707         :returns: Conditional statement which can be evaluated.
1708         :rtype: str
1709         """
1710         index = 0
1711         while True:
1712             index = InputData._end_of_tag(tag_filter, index)
1713             if index is None:
1714                 return tag_filter
1715             index += 1
1716             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1717
1718     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1719                     continue_on_error=False):
1720         """Filter required data from the given jobs and builds.
1721
1722         The output data structure is:
1723         - job 1
1724           - build 1
1725             - test (or suite) 1 ID:
1726               - param 1
1727               - param 2
1728               ...
1729               - param n
1730             ...
1731             - test (or suite) n ID:
1732             ...
1733           ...
1734           - build n
1735         ...
1736         - job n
1737
1738         :param element: Element which will use the filtered data.
1739         :param params: Parameters which will be included in the output. If None,
1740             all parameters are included.
1741         :param data: If not None, this data is used instead of data specified
1742             in the element.
1743         :param data_set: The set of data to be filtered: tests, suites,
1744             metadata.
1745         :param continue_on_error: Continue if there is error while reading the
1746             data. The Item will be empty then
1747         :type element: pandas.Series
1748         :type params: list
1749         :type data: dict
1750         :type data_set: str
1751         :type continue_on_error: bool
1752         :returns: Filtered data.
1753         :rtype pandas.Series
1754         """
1755
1756         try:
1757             if data_set == "suites":
1758                 cond = u"True"
1759             elif element[u"filter"] in (u"all", u"template"):
1760                 cond = u"True"
1761             else:
1762                 cond = InputData._condition(element[u"filter"])
1763             logging.debug(f"   Filter: {cond}")
1764         except KeyError:
1765             logging.error(u"  No filter defined.")
1766             return None
1767
1768         if params is None:
1769             params = element.get(u"parameters", None)
1770             if params:
1771                 params.append(u"type")
1772
1773         data_to_filter = data if data else element[u"data"]
1774         data = pd.Series()
1775         try:
1776             for job, builds in data_to_filter.items():
1777                 data[job] = pd.Series()
1778                 for build in builds:
1779                     data[job][str(build)] = pd.Series()
1780                     try:
1781                         data_dict = dict(
1782                             self.data[job][str(build)][data_set].items())
1783                     except KeyError:
1784                         if continue_on_error:
1785                             continue
1786                         return None
1787
1788                     for test_id, test_data in data_dict.items():
1789                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1790                             data[job][str(build)][test_id] = pd.Series()
1791                             if params is None:
1792                                 for param, val in test_data.items():
1793                                     data[job][str(build)][test_id][param] = val
1794                             else:
1795                                 for param in params:
1796                                     try:
1797                                         data[job][str(build)][test_id][param] =\
1798                                             test_data[param]
1799                                     except KeyError:
1800                                         data[job][str(build)][test_id][param] =\
1801                                             u"No Data"
1802             return data
1803
1804         except (KeyError, IndexError, ValueError) as err:
1805             logging.error(
1806                 f"Missing mandatory parameter in the element specification: "
1807                 f"{repr(err)}"
1808             )
1809             return None
1810         except AttributeError as err:
1811             logging.error(repr(err))
1812             return None
1813         except SyntaxError as err:
1814             logging.error(
1815                 f"The filter {cond} is not correct. Check if all tags are "
1816                 f"enclosed by apostrophes.\n{repr(err)}"
1817             )
1818             return None
1819
1820     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1821                              continue_on_error=False):
1822         """Filter required data from the given jobs and builds.
1823
1824         The output data structure is:
1825         - job 1
1826           - build 1
1827             - test (or suite) 1 ID:
1828               - param 1
1829               - param 2
1830               ...
1831               - param n
1832             ...
1833             - test (or suite) n ID:
1834             ...
1835           ...
1836           - build n
1837         ...
1838         - job n
1839
1840         :param element: Element which will use the filtered data.
1841         :param params: Parameters which will be included in the output. If None,
1842         all parameters are included.
1843         :param data_set: The set of data to be filtered: tests, suites,
1844         metadata.
1845         :param continue_on_error: Continue if there is error while reading the
1846         data. The Item will be empty then
1847         :type element: pandas.Series
1848         :type params: list
1849         :type data_set: str
1850         :type continue_on_error: bool
1851         :returns: Filtered data.
1852         :rtype pandas.Series
1853         """
1854
1855         include = element.get(u"include", None)
1856         if not include:
1857             logging.warning(u"No tests to include, skipping the element.")
1858             return None
1859
1860         if params is None:
1861             params = element.get(u"parameters", None)
1862             if params:
1863                 params.append(u"type")
1864
1865         data = pd.Series()
1866         try:
1867             for job, builds in element[u"data"].items():
1868                 data[job] = pd.Series()
1869                 for build in builds:
1870                     data[job][str(build)] = pd.Series()
1871                     for test in include:
1872                         try:
1873                             reg_ex = re.compile(str(test).lower())
1874                             for test_id in self.data[job][
1875                                     str(build)][data_set].keys():
1876                                 if re.match(reg_ex, str(test_id).lower()):
1877                                     test_data = self.data[job][
1878                                         str(build)][data_set][test_id]
1879                                     data[job][str(build)][test_id] = pd.Series()
1880                                     if params is None:
1881                                         for param, val in test_data.items():
1882                                             data[job][str(build)][test_id]\
1883                                                 [param] = val
1884                                     else:
1885                                         for param in params:
1886                                             try:
1887                                                 data[job][str(build)][
1888                                                     test_id][param] = \
1889                                                     test_data[param]
1890                                             except KeyError:
1891                                                 data[job][str(build)][
1892                                                     test_id][param] = u"No Data"
1893                         except KeyError as err:
1894                             if continue_on_error:
1895                                 logging.debug(repr(err))
1896                                 continue
1897                             logging.error(repr(err))
1898                             return None
1899             return data
1900
1901         except (KeyError, IndexError, ValueError) as err:
1902             logging.error(
1903                 f"Missing mandatory parameter in the element "
1904                 f"specification: {repr(err)}"
1905             )
1906             return None
1907         except AttributeError as err:
1908             logging.error(repr(err))
1909             return None
1910
1911     @staticmethod
1912     def merge_data(data):
1913         """Merge data from more jobs and builds to a simple data structure.
1914
1915         The output data structure is:
1916
1917         - test (suite) 1 ID:
1918           - param 1
1919           - param 2
1920           ...
1921           - param n
1922         ...
1923         - test (suite) n ID:
1924         ...
1925
1926         :param data: Data to merge.
1927         :type data: pandas.Series
1928         :returns: Merged data.
1929         :rtype: pandas.Series
1930         """
1931
1932         logging.info(u"    Merging data ...")
1933
1934         merged_data = pd.Series()
1935         for builds in data.values:
1936             for item in builds.values:
1937                 for item_id, item_data in item.items():
1938                     merged_data[item_id] = item_data
1939         return merged_data
1940
1941     def print_all_oper_data(self):
1942         """Print all operational data to console.
1943         """
1944
1945         tbl_hdr = (
1946             u"Name",
1947             u"Nr of Vectors",
1948             u"Nr of Packets",
1949             u"Suspends",
1950             u"Cycles per Packet",
1951             u"Average Vector Size"
1952         )
1953
1954         for job in self._input_data.values:
1955             for build in job.values:
1956                 for test_id, test_data in build[u"tests"].items():
1957                     print(f"{test_id}")
1958                     if test_data.get(u"show-run", None) is None:
1959                         continue
1960                     for dut_name, data in test_data[u"show-run"].items():
1961                         if data.get(u"threads", None) is None:
1962                             continue
1963                         print(f"Host IP: {data.get(u'host', '')}, "
1964                               f"Socket: {data.get(u'socket', '')}")
1965                         for thread_nr, thread in data[u"threads"].items():
1966                             txt_table = prettytable.PrettyTable(tbl_hdr)
1967                             avg = 0.0
1968                             for row in thread:
1969                                 txt_table.add_row(row)
1970                                 avg += row[-1]
1971                             if len(thread) == 0:
1972                                 avg = u""
1973                             else:
1974                                 avg = f", Average Vector Size per Node: " \
1975                                       f"{(avg / len(thread)):.2f}"
1976                             th_name = u"main" if thread_nr == 0 \
1977                                 else f"worker_{thread_nr}"
1978                             print(f"{dut_name}, {th_name}{avg}")
1979                             txt_table.float_format = u".2"
1980                             txt_table.align = u"r"
1981                             txt_table.align[u"Name"] = u"l"
1982                             print(f"{txt_table.get_string()}\n")