resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove, walk, listdir
  29 from os.path import isfile, isdir, join
  30 from datetime import datetime as dt
  31 from datetime import timedelta
  32 from json import loads
  33 from json.decoder import JSONDecodeError
  34
  35 import hdrh.histogram
  36 import hdrh.codec
  37 import prettytable
  38 import pandas as pd
  39
  40 from robot.api import ExecutionResult, ResultVisitor
  41 from robot import errors
  42
  43 from resources.libraries.python import jumpavg
  44 from input_data_files import download_and_unzip_data_file
  45 from pal_errors import PresentationError
  46
  47
  48 # Separator used in file names
  49 SEPARATOR = u"__"
  50
  51
  52 class ExecutionChecker(ResultVisitor):
  53     """Class to traverse through the test suite structure.
  54
  55     The functionality implemented in this class generates a json structure:
  56
  57     Performance tests:
  58
  59     {
  60         "metadata": {
  61             "generated": "Timestamp",
  62             "version": "SUT version",
  63             "job": "Jenkins job name",
  64             "build": "Information about the build"
  65         },
  66         "suites": {
  67             "Suite long name 1": {
  68                 "name": Suite name,
  69                 "doc": "Suite 1 documentation",
  70                 "parent": "Suite 1 parent",
  71                 "level": "Level of the suite in the suite hierarchy"
  72             }
  73             "Suite long name N": {
  74                 "name": Suite name,
  75                 "doc": "Suite N documentation",
  76                 "parent": "Suite 2 parent",
  77                 "level": "Level of the suite in the suite hierarchy"
  78             }
  79         }
  80         "tests": {
  81             # NDRPDR tests:
  82             "ID": {
  83                 "name": "Test name",
  84                 "parent": "Name of the parent of the test",
  85                 "doc": "Test documentation",
  86                 "msg": "Test message",
  87                 "conf-history": "DUT1 and DUT2 VAT History",
  88                 "show-run": "Show Run",
  89                 "tags": ["tag 1", "tag 2", "tag n"],
  90                 "type": "NDRPDR",
  91                 "status": "PASS" | "FAIL",
  92                 "throughput": {
  93                     "NDR": {
  94                         "LOWER": float,
  95                         "UPPER": float
  96                     },
  97                     "PDR": {
  98                         "LOWER": float,
  99                         "UPPER": float
 100                     }
 101                 },
 102                 "latency": {
 103                     "NDR": {
 104                         "direction1": {
 105                             "min": float,
 106                             "avg": float,
 107                             "max": float,
 108                             "hdrh": str
 109                         },
 110                         "direction2": {
 111                             "min": float,
 112                             "avg": float,
 113                             "max": float,
 114                             "hdrh": str
 115                         }
 116                     },
 117                     "PDR": {
 118                         "direction1": {
 119                             "min": float,
 120                             "avg": float,
 121                             "max": float,
 122                             "hdrh": str
 123                         },
 124                         "direction2": {
 125                             "min": float,
 126                             "avg": float,
 127                             "max": float,
 128                             "hdrh": str
 129                         }
 130                     }
 131                 }
 132             }
 133
 134             # TCP tests:
 135             "ID": {
 136                 "name": "Test name",
 137                 "parent": "Name of the parent of the test",
 138                 "doc": "Test documentation",
 139                 "msg": "Test message",
 140                 "tags": ["tag 1", "tag 2", "tag n"],
 141                 "type": "TCP",
 142                 "status": "PASS" | "FAIL",
 143                 "result": int
 144             }
 145
 146             # MRR, BMRR tests:
 147             "ID": {
 148                 "name": "Test name",
 149                 "parent": "Name of the parent of the test",
 150                 "doc": "Test documentation",
 151                 "msg": "Test message",
 152                 "tags": ["tag 1", "tag 2", "tag n"],
 153                 "type": "MRR" | "BMRR",
 154                 "status": "PASS" | "FAIL",
 155                 "result": {
 156                     "receive-rate": float,
 157                     # Average of a list, computed using AvgStdevStats.
 158                     # In CSIT-1180, replace with List[float].
 159                 }
 160             }
 161
 162             "ID" {
 163                 # next test
 164             }
 165         }
 166     }
 167
 168
 169     Functional tests:
 170
 171     {
 172         "metadata": {  # Optional
 173             "version": "VPP version",
 174             "job": "Jenkins job name",
 175             "build": "Information about the build"
 176         },
 177         "suites": {
 178             "Suite name 1": {
 179                 "doc": "Suite 1 documentation",
 180                 "parent": "Suite 1 parent",
 181                 "level": "Level of the suite in the suite hierarchy"
 182             }
 183             "Suite name N": {
 184                 "doc": "Suite N documentation",
 185                 "parent": "Suite 2 parent",
 186                 "level": "Level of the suite in the suite hierarchy"
 187             }
 188         }
 189         "tests": {
 190             "ID": {
 191                 "name": "Test name",
 192                 "parent": "Name of the parent of the test",
 193                 "doc": "Test documentation"
 194                 "msg": "Test message"
 195                 "tags": ["tag 1", "tag 2", "tag n"],
 196                 "conf-history": "DUT1 and DUT2 VAT History"
 197                 "show-run": "Show Run"
 198                 "status": "PASS" | "FAIL"
 199             },
 200             "ID" {
 201                 # next test
 202             }
 203         }
 204     }
 205
 206     .. note:: ID is the lowercase full path to the test.
 207     """
 208
 209     REGEX_PLR_RATE = re.compile(
 210         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 211         r'PLRsearch upper bound::?\s(\d+.\d+)'
 212     )
 213     REGEX_NDRPDR_RATE = re.compile(
 214         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 215         r'NDR_UPPER:\s(\d+.\d+).*\n'
 216         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 217         r'PDR_UPPER:\s(\d+.\d+)'
 218     )
 219     REGEX_NDRPDR_GBPS = re.compile(
 220         r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
 221         r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
 222         r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
 223         r'PDR_UPPER:.*,\s(\d+.\d+)'
 224     )
 225     REGEX_PERF_MSG_INFO = re.compile(
 226         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 227         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 228         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 229         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 230         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 231     )
 232     REGEX_CPS_MSG_INFO = re.compile(
 233         r'NDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*\n'
 234         r'PDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*'
 235     )
 236     REGEX_PPS_MSG_INFO = re.compile(
 237         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 238         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*'
 239     )
 240     REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
 241
 242     REGEX_VSAP_MSG_INFO = re.compile(
 243         r'Transfer Rate: (\d*.\d*).*\n'
 244         r'Latency: (\d*.\d*).*\n'
 245         r'Completed requests: (\d*).*\n'
 246         r'Failed requests: (\d*).*\n'
 247         r'Total data transferred: (\d*).*\n'
 248         r'Connection [cr]ps rate:\s*(\d*.\d*)'
 249     )
 250
 251     # Needed for CPS and PPS tests
 252     REGEX_NDRPDR_LAT_BASE = re.compile(
 253         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 254         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 255     )
 256     REGEX_NDRPDR_LAT = re.compile(
 257         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 258         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 259         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 260         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 261         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 262         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 263     )
 264
 265     REGEX_VERSION_VPP = re.compile(
 266         r"(return STDOUT Version:\s*|"
 267         r"VPP Version:\s*|VPP version:\s*)(.*)"
 268     )
 269     REGEX_VERSION_DPDK = re.compile(
 270         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 271     )
 272     REGEX_TCP = re.compile(
 273         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 274     )
 275     REGEX_MRR = re.compile(
 276         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 277         r'tx\s(\d*),\srx\s(\d*)'
 278     )
 279     REGEX_BMRR = re.compile(
 280         r'.*trial results.*: \[(.*)\]'
 281     )
 282     REGEX_RECONF_LOSS = re.compile(
 283         r'Packets lost due to reconfig: (\d*)'
 284     )
 285     REGEX_RECONF_TIME = re.compile(
 286         r'Implied time lost: (\d*.[\de-]*)'
 287     )
 288     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 289
 290     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 291
 292     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 293
 294     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 295
 296     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 297
 298     def __init__(self, metadata, mapping, ignore):
 299         """Initialisation.
 300
 301         :param metadata: Key-value pairs to be included in "metadata" part of
 302             JSON structure.
 303         :param mapping: Mapping of the old names of test cases to the new
 304             (actual) one.
 305         :param ignore: List of TCs to be ignored.
 306         :type metadata: dict
 307         :type mapping: dict
 308         :type ignore: list
 309         """
 310
 311         # Type of message to parse out from the test messages
 312         self._msg_type = None
 313
 314         # VPP version
 315         self._version = None
 316
 317         # Timestamp
 318         self._timestamp = None
 319
 320         # Testbed. The testbed is identified by TG node IP address.
 321         self._testbed = None
 322
 323         # Mapping of TCs long names
 324         self._mapping = mapping
 325
 326         # Ignore list
 327         self._ignore = ignore
 328
 329         # Number of PAPI History messages found:
 330         # 0 - no message
 331         # 1 - PAPI History of DUT1
 332         # 2 - PAPI History of DUT2
 333         self._conf_history_lookup_nr = 0
 334
 335         self._sh_run_counter = 0
 336
 337         # Test ID of currently processed test- the lowercase full path to the
 338         # test
 339         self._test_id = None
 340
 341         # The main data structure
 342         self._data = {
 343             u"metadata": OrderedDict(),
 344             u"suites": OrderedDict(),
 345             u"tests": OrderedDict()
 346         }
 347
 348         # Save the provided metadata
 349         for key, val in metadata.items():
 350             self._data[u"metadata"][key] = val
 351
 352         # Dictionary defining the methods used to parse different types of
 353         # messages
 354         self.parse_msg = {
 355             u"timestamp": self._get_timestamp,
 356             u"vpp-version": self._get_vpp_version,
 357             u"dpdk-version": self._get_dpdk_version,
 358             u"teardown-papi-history": self._get_papi_history,
 359             u"test-show-runtime": self._get_show_run,
 360             u"testbed": self._get_testbed
 361         }
 362
 363     @property
 364     def data(self):
 365         """Getter - Data parsed from the XML file.
 366
 367         :returns: Data parsed from the XML file.
 368         :rtype: dict
 369         """
 370         return self._data
 371
 372     def _get_data_from_mrr_test_msg(self, msg):
 373         """Get info from message of MRR performance tests.
 374
 375         :param msg: Message to be processed.
 376         :type msg: str
 377         :returns: Processed message or original message if a problem occurs.
 378         :rtype: str
 379         """
 380
 381         groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
 382         if not groups or groups.lastindex != 1:
 383             return u"Test Failed."
 384
 385         try:
 386             data = groups.group(1).split(u", ")
 387         except (AttributeError, IndexError, ValueError, KeyError):
 388             return u"Test Failed."
 389
 390         out_str = u"["
 391         try:
 392             for item in data:
 393                 out_str += f"{(float(item) / 1e6):.2f}, "
 394             return out_str[:-2] + u"]"
 395         except (AttributeError, IndexError, ValueError, KeyError):
 396             return u"Test Failed."
 397
 398     def _get_data_from_cps_test_msg(self, msg):
 399         """Get info from message of NDRPDR CPS tests.
 400
 401         :param msg: Message to be processed.
 402         :type msg: str
 403         :returns: Processed message or "Test Failed." if a problem occurs.
 404         :rtype: str
 405         """
 406
 407         groups = re.search(self.REGEX_CPS_MSG_INFO, msg)
 408         if not groups or groups.lastindex != 2:
 409             return u"Test Failed."
 410
 411         try:
 412             return (
 413                 f"1. {(float(groups.group(1)) / 1e6):5.2f}\n"
 414                 f"2. {(float(groups.group(2)) / 1e6):5.2f}"
 415             )
 416         except (AttributeError, IndexError, ValueError, KeyError):
 417             return u"Test Failed."
 418
 419     def _get_data_from_pps_test_msg(self, msg):
 420         """Get info from message of NDRPDR PPS tests.
 421
 422         :param msg: Message to be processed.
 423         :type msg: str
 424         :returns: Processed message or "Test Failed." if a problem occurs.
 425         :rtype: str
 426         """
 427
 428         groups = re.search(self.REGEX_PPS_MSG_INFO, msg)
 429         if not groups or groups.lastindex != 4:
 430             return u"Test Failed."
 431
 432         try:
 433             return (
 434                 f"1. {(float(groups.group(1)) / 1e6):5.2f}      "
 435                 f"{float(groups.group(2)):5.2f}\n"
 436                 f"2. {(float(groups.group(3)) / 1e6):5.2f}      "
 437                 f"{float(groups.group(4)):5.2f}"
 438             )
 439         except (AttributeError, IndexError, ValueError, KeyError):
 440             return u"Test Failed."
 441
 442     def _get_data_from_perf_test_msg(self, msg):
 443         """Get info from message of NDRPDR performance tests.
 444
 445         :param msg: Message to be processed.
 446         :type msg: str
 447         :returns: Processed message or "Test Failed." if a problem occurs.
 448         :rtype: str
 449         """
 450
 451         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 452         if not groups or groups.lastindex != 10:
 453             return u"Test Failed."
 454
 455         try:
 456             data = {
 457                 u"ndr_low": float(groups.group(1)),
 458                 u"ndr_low_b": float(groups.group(2)),
 459                 u"pdr_low": float(groups.group(3)),
 460                 u"pdr_low_b": float(groups.group(4)),
 461                 u"pdr_lat_90_1": groups.group(5),
 462                 u"pdr_lat_90_2": groups.group(6),
 463                 u"pdr_lat_50_1": groups.group(7),
 464                 u"pdr_lat_50_2": groups.group(8),
 465                 u"pdr_lat_10_1": groups.group(9),
 466                 u"pdr_lat_10_2": groups.group(10),
 467             }
 468         except (AttributeError, IndexError, ValueError, KeyError):
 469             return u"Test Failed."
 470
 471         def _process_lat(in_str_1, in_str_2):
 472             """Extract min, avg, max values from latency string.
 473
 474             :param in_str_1: Latency string for one direction produced by robot
 475                 framework.
 476             :param in_str_2: Latency string for second direction produced by
 477                 robot framework.
 478             :type in_str_1: str
 479             :type in_str_2: str
 480             :returns: Processed latency string or None if a problem occurs.
 481             :rtype: tuple
 482             """
 483             in_list_1 = in_str_1.split('/', 3)
 484             in_list_2 = in_str_2.split('/', 3)
 485
 486             if len(in_list_1) != 4 and len(in_list_2) != 4:
 487                 return None
 488
 489             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 490             try:
 491                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 492             except hdrh.codec.HdrLengthException:
 493                 return None
 494
 495             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 496             try:
 497                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 498             except hdrh.codec.HdrLengthException:
 499                 return None
 500
 501             if hdr_lat_1 and hdr_lat_2:
 502                 hdr_lat = (
 503                     hdr_lat_1.get_value_at_percentile(50.0),
 504                     hdr_lat_1.get_value_at_percentile(90.0),
 505                     hdr_lat_1.get_value_at_percentile(99.0),
 506                     hdr_lat_2.get_value_at_percentile(50.0),
 507                     hdr_lat_2.get_value_at_percentile(90.0),
 508                     hdr_lat_2.get_value_at_percentile(99.0)
 509                 )
 510
 511                 if all(hdr_lat):
 512                     return hdr_lat
 513
 514             return None
 515
 516         try:
 517             out_msg = (
 518                 f"1. {(data[u'ndr_low'] / 1e6):5.2f}      "
 519                 f"{data[u'ndr_low_b']:5.2f}"
 520                 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f}      "
 521                 f"{data[u'pdr_low_b']:5.2f}"
 522             )
 523             latency = (
 524                 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
 525                 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
 526                 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
 527             )
 528             if all(latency):
 529                 max_len = len(str(max((max(item) for item in latency))))
 530                 max_len = 4 if max_len < 4 else max_len
 531
 532                 for idx, lat in enumerate(latency):
 533                     if not idx:
 534                         out_msg += u"\n"
 535                     out_msg += (
 536                         f"\n{idx + 3}. "
 537                         f"{lat[0]:{max_len}d} "
 538                         f"{lat[1]:{max_len}d} "
 539                         f"{lat[2]:{max_len}d}      "
 540                         f"{lat[3]:{max_len}d} "
 541                         f"{lat[4]:{max_len}d} "
 542                         f"{lat[5]:{max_len}d} "
 543                     )
 544
 545             return out_msg
 546
 547         except (AttributeError, IndexError, ValueError, KeyError):
 548             return u"Test Failed."
 549
 550     def _get_testbed(self, msg):
 551         """Called when extraction of testbed IP is required.
 552         The testbed is identified by TG node IP address.
 553
 554         :param msg: Message to process.
 555         :type msg: Message
 556         :returns: Nothing.
 557         """
 558
 559         if msg.message.count(u"Setup of TG node") or \
 560                 msg.message.count(u"Setup of node TG host"):
 561             reg_tg_ip = re.compile(
 562                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 563             try:
 564                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 565             except (KeyError, ValueError, IndexError, AttributeError):
 566                 pass
 567             finally:
 568                 self._data[u"metadata"][u"testbed"] = self._testbed
 569                 self._msg_type = None
 570
 571     def _get_vpp_version(self, msg):
 572         """Called when extraction of VPP version is required.
 573
 574         :param msg: Message to process.
 575         :type msg: Message
 576         :returns: Nothing.
 577         """
 578
 579         if msg.message.count(u"return STDOUT Version:") or \
 580                 msg.message.count(u"VPP Version:") or \
 581                 msg.message.count(u"VPP version:"):
 582             self._version = str(
 583                 re.search(self.REGEX_VERSION_VPP, msg.message).group(2)
 584             )
 585             self._data[u"metadata"][u"version"] = self._version
 586             self._msg_type = None
 587
 588     def _get_dpdk_version(self, msg):
 589         """Called when extraction of DPDK version is required.
 590
 591         :param msg: Message to process.
 592         :type msg: Message
 593         :returns: Nothing.
 594         """
 595
 596         if msg.message.count(u"DPDK Version:"):
 597             try:
 598                 self._version = str(re.search(
 599                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 600                 self._data[u"metadata"][u"version"] = self._version
 601             except IndexError:
 602                 pass
 603             finally:
 604                 self._msg_type = None
 605
 606     def _get_timestamp(self, msg):
 607         """Called when extraction of timestamp is required.
 608
 609         :param msg: Message to process.
 610         :type msg: Message
 611         :returns: Nothing.
 612         """
 613
 614         self._timestamp = msg.timestamp[:14]
 615         self._data[u"metadata"][u"generated"] = self._timestamp
 616         self._msg_type = None
 617
 618     def _get_papi_history(self, msg):
 619         """Called when extraction of PAPI command history is required.
 620
 621         :param msg: Message to process.
 622         :type msg: Message
 623         :returns: Nothing.
 624         """
 625         if msg.message.count(u"PAPI command history:"):
 626             self._conf_history_lookup_nr += 1
 627             if self._conf_history_lookup_nr == 1:
 628                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 629             else:
 630                 self._msg_type = None
 631             text = re.sub(
 632                 r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} PAPI command history:",
 633                 u"",
 634                 msg.message,
 635                 count=1
 636             ).replace(u'"', u"'")
 637             self._data[u"tests"][self._test_id][u"conf-history"] += (
 638                 f"**DUT{str(self._conf_history_lookup_nr)}:** {text}"
 639             )
 640
 641     def _get_show_run(self, msg):
 642         """Called when extraction of VPP operational data (output of CLI command
 643         Show Runtime) is required.
 644
 645         :param msg: Message to process.
 646         :type msg: Message
 647         :returns: Nothing.
 648         """
 649
 650         if not msg.message.count(u"stats runtime"):
 651             return
 652
 653         # Temporary solution
 654         if self._sh_run_counter > 1:
 655             return
 656
 657         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 658             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 659
 660         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 661         if not groups:
 662             return
 663         try:
 664             host = groups.group(1)
 665         except (AttributeError, IndexError):
 666             host = u""
 667         try:
 668             sock = groups.group(2)
 669         except (AttributeError, IndexError):
 670             sock = u""
 671
 672         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 673                         replace(u"'", u'"').replace(u'b"', u'"').
 674                         replace(u'u"', u'"').split(u":", 1)[1])
 675
 676         try:
 677             threads_nr = len(runtime[0][u"clocks"])
 678         except (IndexError, KeyError):
 679             return
 680
 681         dut = u"dut{nr}".format(
 682             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 683
 684         oper = {
 685             u"host": host,
 686             u"socket": sock,
 687             # Needed for json converter, enable when 'threads' is gone.
 688             # u"runtime": runtime,
 689             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 690         }
 691
 692         for item in runtime:
 693             for idx in range(threads_nr):
 694                 if item[u"vectors"][idx] > 0:
 695                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 696                 elif item[u"calls"][idx] > 0:
 697                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 698                 elif item[u"suspends"][idx] > 0:
 699                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 700                 else:
 701                     clocks = 0.0
 702
 703                 if item[u"calls"][idx] > 0:
 704                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 705                 else:
 706                     vectors_call = 0.0
 707
 708                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 709                         int(item[u"suspends"][idx]):
 710                     oper[u"threads"][idx].append([
 711                         item[u"name"],
 712                         item[u"calls"][idx],
 713                         item[u"vectors"][idx],
 714                         item[u"suspends"][idx],
 715                         clocks,
 716                         vectors_call
 717                     ])
 718
 719         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 720
 721     def _get_ndrpdr_throughput(self, msg):
 722         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 723         message.
 724
 725         :param msg: The test message to be parsed.
 726         :type msg: str
 727         :returns: Parsed data as a dict and the status (PASS/FAIL).
 728         :rtype: tuple(dict, str)
 729         """
 730
 731         throughput = {
 732             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 733             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 734         }
 735         status = u"FAIL"
 736         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 737
 738         if groups is not None:
 739             try:
 740                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 741                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 742                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 743                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 744                 status = u"PASS"
 745             except (IndexError, ValueError):
 746                 pass
 747
 748         return throughput, status
 749
 750     def _get_ndrpdr_throughput_gbps(self, msg):
 751         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
 752         test message.
 753
 754         :param msg: The test message to be parsed.
 755         :type msg: str
 756         :returns: Parsed data as a dict and the status (PASS/FAIL).
 757         :rtype: tuple(dict, str)
 758         """
 759
 760         gbps = {
 761             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 762             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 763         }
 764         status = u"FAIL"
 765         groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
 766
 767         if groups is not None:
 768             try:
 769                 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
 770                 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
 771                 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
 772                 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
 773                 status = u"PASS"
 774             except (IndexError, ValueError):
 775                 pass
 776
 777         return gbps, status
 778
 779     def _get_plr_throughput(self, msg):
 780         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 781         message.
 782
 783         :param msg: The test message to be parsed.
 784         :type msg: str
 785         :returns: Parsed data as a dict and the status (PASS/FAIL).
 786         :rtype: tuple(dict, str)
 787         """
 788
 789         throughput = {
 790             u"LOWER": -1.0,
 791             u"UPPER": -1.0
 792         }
 793         status = u"FAIL"
 794         groups = re.search(self.REGEX_PLR_RATE, msg)
 795
 796         if groups is not None:
 797             try:
 798                 throughput[u"LOWER"] = float(groups.group(1))
 799                 throughput[u"UPPER"] = float(groups.group(2))
 800                 status = u"PASS"
 801             except (IndexError, ValueError):
 802                 pass
 803
 804         return throughput, status
 805
 806     def _get_ndrpdr_latency(self, msg):
 807         """Get LATENCY from the test message.
 808
 809         :param msg: The test message to be parsed.
 810         :type msg: str
 811         :returns: Parsed data as a dict and the status (PASS/FAIL).
 812         :rtype: tuple(dict, str)
 813         """
 814         latency_default = {
 815             u"min": -1.0,
 816             u"avg": -1.0,
 817             u"max": -1.0,
 818             u"hdrh": u""
 819         }
 820         latency = {
 821             u"NDR": {
 822                 u"direction1": copy.copy(latency_default),
 823                 u"direction2": copy.copy(latency_default)
 824             },
 825             u"PDR": {
 826                 u"direction1": copy.copy(latency_default),
 827                 u"direction2": copy.copy(latency_default)
 828             },
 829             u"LAT0": {
 830                 u"direction1": copy.copy(latency_default),
 831                 u"direction2": copy.copy(latency_default)
 832             },
 833             u"PDR10": {
 834                 u"direction1": copy.copy(latency_default),
 835                 u"direction2": copy.copy(latency_default)
 836             },
 837             u"PDR50": {
 838                 u"direction1": copy.copy(latency_default),
 839                 u"direction2": copy.copy(latency_default)
 840             },
 841             u"PDR90": {
 842                 u"direction1": copy.copy(latency_default),
 843                 u"direction2": copy.copy(latency_default)
 844             },
 845         }
 846
 847         groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 848         if groups is None:
 849             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 850         if groups is None:
 851             return latency, u"FAIL"
 852
 853         def process_latency(in_str):
 854             """Return object with parsed latency values.
 855
 856             TODO: Define class for the return type.
 857
 858             :param in_str: Input string, min/avg/max/hdrh format.
 859             :type in_str: str
 860             :returns: Dict with corresponding keys, except hdrh float values.
 861             :rtype dict:
 862             :throws IndexError: If in_str does not have enough substrings.
 863             :throws ValueError: If a substring does not convert to float.
 864             """
 865             in_list = in_str.split('/', 3)
 866
 867             rval = {
 868                 u"min": float(in_list[0]),
 869                 u"avg": float(in_list[1]),
 870                 u"max": float(in_list[2]),
 871                 u"hdrh": u""
 872             }
 873
 874             if len(in_list) == 4:
 875                 rval[u"hdrh"] = str(in_list[3])
 876
 877             return rval
 878
 879         try:
 880             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 881             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 882             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 883             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 884             if groups.lastindex == 4:
 885                 return latency, u"PASS"
 886         except (IndexError, ValueError):
 887             pass
 888
 889         try:
 890             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 891             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 892             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 893             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 894             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 895             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 896             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 897             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 898             if groups.lastindex == 12:
 899                 return latency, u"PASS"
 900         except (IndexError, ValueError):
 901             pass
 902
 903         return latency, u"FAIL"
 904
 905     @staticmethod
 906     def _get_hoststack_data(msg, tags):
 907         """Get data from the hoststack test message.
 908
 909         :param msg: The test message to be parsed.
 910         :param tags: Test tags.
 911         :type msg: str
 912         :type tags: list
 913         :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
 914         :rtype: tuple(dict, str)
 915         """
 916         result = dict()
 917         status = u"FAIL"
 918
 919         msg = msg.replace(u"'", u'"').replace(u" ", u"")
 920         if u"LDPRELOAD" in tags:
 921             try:
 922                 result = loads(msg)
 923                 status = u"PASS"
 924             except JSONDecodeError:
 925                 pass
 926         elif u"VPPECHO" in tags:
 927             try:
 928                 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
 929                 result = dict(
 930                     client=loads(msg_lst[0]),
 931                     server=loads(msg_lst[1])
 932                 )
 933                 status = u"PASS"
 934             except (JSONDecodeError, IndexError):
 935                 pass
 936
 937         return result, status
 938
 939     def _get_vsap_data(self, msg, tags):
 940         """Get data from the vsap test message.
 941
 942         :param msg: The test message to be parsed.
 943         :param tags: Test tags.
 944         :type msg: str
 945         :type tags: list
 946         :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
 947         :rtype: tuple(dict, str)
 948         """
 949         result = dict()
 950         status = u"FAIL"
 951
 952         groups = re.search(self.REGEX_VSAP_MSG_INFO, msg)
 953         if groups is not None:
 954             try:
 955                 result[u"transfer-rate"] = float(groups.group(1)) * 1e3
 956                 result[u"latency"] = float(groups.group(2))
 957                 result[u"completed-requests"] = int(groups.group(3))
 958                 result[u"failed-requests"] = int(groups.group(4))
 959                 result[u"bytes-transferred"] = int(groups.group(5))
 960                 if u"TCP_CPS"in tags:
 961                     result[u"cps"] = float(groups.group(6))
 962                 elif u"TCP_RPS" in tags:
 963                     result[u"rps"] = float(groups.group(6))
 964                 else:
 965                     return result, status
 966                 status = u"PASS"
 967             except (IndexError, ValueError):
 968                 pass
 969
 970         return result, status
 971
 972     def visit_suite(self, suite):
 973         """Implements traversing through the suite and its direct children.
 974
 975         :param suite: Suite to process.
 976         :type suite: Suite
 977         :returns: Nothing.
 978         """
 979         if self.start_suite(suite) is not False:
 980             suite.suites.visit(self)
 981             suite.tests.visit(self)
 982             self.end_suite(suite)
 983
 984     def start_suite(self, suite):
 985         """Called when suite starts.
 986
 987         :param suite: Suite to process.
 988         :type suite: Suite
 989         :returns: Nothing.
 990         """
 991
 992         try:
 993             parent_name = suite.parent.name
 994         except AttributeError:
 995             return
 996
 997         self._data[u"suites"][suite.longname.lower().
 998                               replace(u'"', u"'").
 999                               replace(u" ", u"_")] = {
1000                                   u"name": suite.name.lower(),
1001                                   u"doc": suite.doc,
1002                                   u"parent": parent_name,
1003                                   u"level": len(suite.longname.split(u"."))
1004                               }
1005
1006         suite.keywords.visit(self)
1007
1008     def end_suite(self, suite):
1009         """Called when suite ends.
1010
1011         :param suite: Suite to process.
1012         :type suite: Suite
1013         :returns: Nothing.
1014         """
1015
1016     def visit_test(self, test):
1017         """Implements traversing through the test.
1018
1019         :param test: Test to process.
1020         :type test: Test
1021         :returns: Nothing.
1022         """
1023         if self.start_test(test) is not False:
1024             test.keywords.visit(self)
1025             self.end_test(test)
1026
1027     def start_test(self, test):
1028         """Called when test starts.
1029
1030         :param test: Test to process.
1031         :type test: Test
1032         :returns: Nothing.
1033         """
1034
1035         self._sh_run_counter = 0
1036
1037         longname_orig = test.longname.lower()
1038
1039         # Check the ignore list
1040         if longname_orig in self._ignore:
1041             return
1042
1043         tags = [str(tag) for tag in test.tags]
1044         test_result = dict()
1045
1046         # Change the TC long name and name if defined in the mapping table
1047         longname = self._mapping.get(longname_orig, None)
1048         if longname is not None:
1049             name = longname.split(u'.')[-1]
1050             logging.debug(
1051                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1052                 f"{name}"
1053             )
1054         else:
1055             longname = longname_orig
1056             name = test.name.lower()
1057
1058         # Remove TC number from the TC long name (backward compatibility):
1059         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1060         # Remove TC number from the TC name (not needed):
1061         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1062
1063         test_result[u"parent"] = test.parent.name.lower()
1064         test_result[u"tags"] = tags
1065         test_result["doc"] = test.doc
1066         test_result[u"type"] = u""
1067         test_result[u"status"] = test.status
1068         test_result[u"starttime"] = test.starttime
1069         test_result[u"endtime"] = test.endtime
1070
1071         if test.status == u"PASS":
1072             if u"NDRPDR" in tags:
1073                 if u"TCP_PPS" in tags or u"UDP_PPS" in tags:
1074                     test_result[u"msg"] = self._get_data_from_pps_test_msg(
1075                         test.message)
1076                 elif u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1077                     test_result[u"msg"] = self._get_data_from_cps_test_msg(
1078                         test.message)
1079                 else:
1080                     test_result[u"msg"] = self._get_data_from_perf_test_msg(
1081                         test.message)
1082             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1083                 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1084                     test.message)
1085             else:
1086                 test_result[u"msg"] = test.message
1087         else:
1088             test_result[u"msg"] = test.message
1089
1090         if u"PERFTEST" in tags:
1091             # Replace info about cores (e.g. -1c-) with the info about threads
1092             # and cores (e.g. -1t1c-) in the long test case names and in the
1093             # test case names if necessary.
1094             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1095             if not groups:
1096                 tag_count = 0
1097                 tag_tc = str()
1098                 for tag in test_result[u"tags"]:
1099                     groups = re.search(self.REGEX_TC_TAG, tag)
1100                     if groups:
1101                         tag_count += 1
1102                         tag_tc = tag
1103
1104                 if tag_count == 1:
1105                     self._test_id = re.sub(
1106                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1107                         self._test_id, count=1
1108                     )
1109                     test_result[u"name"] = re.sub(
1110                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1111                         test_result["name"], count=1
1112                     )
1113                 else:
1114                     test_result[u"status"] = u"FAIL"
1115                     self._data[u"tests"][self._test_id] = test_result
1116                     logging.debug(
1117                         f"The test {self._test_id} has no or more than one "
1118                         f"multi-threading tags.\n"
1119                         f"Tags: {test_result[u'tags']}"
1120                     )
1121                     return
1122
1123         if u"DEVICETEST" in tags:
1124             test_result[u"type"] = u"DEVICETEST"
1125         elif u"NDRPDR" in tags:
1126             if u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1127                 test_result[u"type"] = u"CPS"
1128             else:
1129                 test_result[u"type"] = u"NDRPDR"
1130             if test.status == u"PASS":
1131                 test_result[u"throughput"], test_result[u"status"] = \
1132                     self._get_ndrpdr_throughput(test.message)
1133                 test_result[u"gbps"], test_result[u"status"] = \
1134                     self._get_ndrpdr_throughput_gbps(test.message)
1135                 test_result[u"latency"], test_result[u"status"] = \
1136                     self._get_ndrpdr_latency(test.message)
1137         elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1138             if u"MRR" in tags:
1139                 test_result[u"type"] = u"MRR"
1140             else:
1141                 test_result[u"type"] = u"BMRR"
1142             if test.status == u"PASS":
1143                 test_result[u"result"] = dict()
1144                 groups = re.search(self.REGEX_BMRR, test.message)
1145                 if groups is not None:
1146                     items_str = groups.group(1)
1147                     items_float = [
1148                         float(item.strip().replace(u"'", u""))
1149                         for item in items_str.split(",")
1150                     ]
1151                     # Use whole list in CSIT-1180.
1152                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1153                     test_result[u"result"][u"samples"] = items_float
1154                     test_result[u"result"][u"receive-rate"] = stats.avg
1155                     test_result[u"result"][u"receive-stdev"] = stats.stdev
1156                 else:
1157                     groups = re.search(self.REGEX_MRR, test.message)
1158                     test_result[u"result"][u"receive-rate"] = \
1159                         float(groups.group(3)) / float(groups.group(1))
1160         elif u"SOAK" in tags:
1161             test_result[u"type"] = u"SOAK"
1162             if test.status == u"PASS":
1163                 test_result[u"throughput"], test_result[u"status"] = \
1164                     self._get_plr_throughput(test.message)
1165         elif u"HOSTSTACK" in tags:
1166             test_result[u"type"] = u"HOSTSTACK"
1167             if test.status == u"PASS":
1168                 test_result[u"result"], test_result[u"status"] = \
1169                     self._get_hoststack_data(test.message, tags)
1170         elif u"LDP_NGINX" in tags:
1171             test_result[u"type"] = u"LDP_NGINX"
1172             test_result[u"result"], test_result[u"status"] = \
1173                 self._get_vsap_data(test.message, tags)
1174         # elif u"TCP" in tags:  # This might be not used
1175         #     test_result[u"type"] = u"TCP"
1176         #     if test.status == u"PASS":
1177         #         groups = re.search(self.REGEX_TCP, test.message)
1178         #         test_result[u"result"] = int(groups.group(2))
1179         elif u"RECONF" in tags:
1180             test_result[u"type"] = u"RECONF"
1181             if test.status == u"PASS":
1182                 test_result[u"result"] = None
1183                 try:
1184                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1185                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1186                     test_result[u"result"] = {
1187                         u"loss": int(grps_loss.group(1)),
1188                         u"time": float(grps_time.group(1))
1189                     }
1190                 except (AttributeError, IndexError, ValueError, TypeError):
1191                     test_result[u"status"] = u"FAIL"
1192         else:
1193             test_result[u"status"] = u"FAIL"
1194
1195         self._data[u"tests"][self._test_id] = test_result
1196
1197     def end_test(self, test):
1198         """Called when test ends.
1199
1200         :param test: Test to process.
1201         :type test: Test
1202         :returns: Nothing.
1203         """
1204
1205     def visit_keyword(self, keyword):
1206         """Implements traversing through the keyword and its child keywords.
1207
1208         :param keyword: Keyword to process.
1209         :type keyword: Keyword
1210         :returns: Nothing.
1211         """
1212         if self.start_keyword(keyword) is not False:
1213             self.end_keyword(keyword)
1214
1215     def start_keyword(self, keyword):
1216         """Called when keyword starts. Default implementation does nothing.
1217
1218         :param keyword: Keyword to process.
1219         :type keyword: Keyword
1220         :returns: Nothing.
1221         """
1222         try:
1223             if keyword.type == u"setup":
1224                 self.visit_setup_kw(keyword)
1225             elif keyword.type == u"teardown":
1226                 self.visit_teardown_kw(keyword)
1227             else:
1228                 self.visit_test_kw(keyword)
1229         except AttributeError:
1230             pass
1231
1232     def end_keyword(self, keyword):
1233         """Called when keyword ends. Default implementation does nothing.
1234
1235         :param keyword: Keyword to process.
1236         :type keyword: Keyword
1237         :returns: Nothing.
1238         """
1239
1240     def visit_test_kw(self, test_kw):
1241         """Implements traversing through the test keyword and its child
1242         keywords.
1243
1244         :param test_kw: Keyword to process.
1245         :type test_kw: Keyword
1246         :returns: Nothing.
1247         """
1248         for keyword in test_kw.keywords:
1249             if self.start_test_kw(keyword) is not False:
1250                 self.visit_test_kw(keyword)
1251                 self.end_test_kw(keyword)
1252
1253     def start_test_kw(self, test_kw):
1254         """Called when test keyword starts. Default implementation does
1255         nothing.
1256
1257         :param test_kw: Keyword to process.
1258         :type test_kw: Keyword
1259         :returns: Nothing.
1260         """
1261         if test_kw.name.count(u"Show Runtime On All Duts") or \
1262                 test_kw.name.count(u"Show Runtime Counters On All Duts") or \
1263                 test_kw.name.count(u"Vpp Show Runtime On All Duts"):
1264             self._msg_type = u"test-show-runtime"
1265             self._sh_run_counter += 1
1266         else:
1267             return
1268         test_kw.messages.visit(self)
1269
1270     def end_test_kw(self, test_kw):
1271         """Called when keyword ends. Default implementation does nothing.
1272
1273         :param test_kw: Keyword to process.
1274         :type test_kw: Keyword
1275         :returns: Nothing.
1276         """
1277
1278     def visit_setup_kw(self, setup_kw):
1279         """Implements traversing through the teardown keyword and its child
1280         keywords.
1281
1282         :param setup_kw: Keyword to process.
1283         :type setup_kw: Keyword
1284         :returns: Nothing.
1285         """
1286         for keyword in setup_kw.keywords:
1287             if self.start_setup_kw(keyword) is not False:
1288                 self.visit_setup_kw(keyword)
1289                 self.end_setup_kw(keyword)
1290
1291     def start_setup_kw(self, setup_kw):
1292         """Called when teardown keyword starts. Default implementation does
1293         nothing.
1294
1295         :param setup_kw: Keyword to process.
1296         :type setup_kw: Keyword
1297         :returns: Nothing.
1298         """
1299         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1300                 and not self._version:
1301             self._msg_type = u"vpp-version"
1302         elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1303                 not self._version:
1304             self._msg_type = u"dpdk-version"
1305         elif setup_kw.name.count(u"Set Global Variable") \
1306                 and not self._timestamp:
1307             self._msg_type = u"timestamp"
1308         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1309             self._msg_type = u"testbed"
1310         else:
1311             return
1312         setup_kw.messages.visit(self)
1313
1314     def end_setup_kw(self, setup_kw):
1315         """Called when keyword ends. Default implementation does nothing.
1316
1317         :param setup_kw: Keyword to process.
1318         :type setup_kw: Keyword
1319         :returns: Nothing.
1320         """
1321
1322     def visit_teardown_kw(self, teardown_kw):
1323         """Implements traversing through the teardown keyword and its child
1324         keywords.
1325
1326         :param teardown_kw: Keyword to process.
1327         :type teardown_kw: Keyword
1328         :returns: Nothing.
1329         """
1330         for keyword in teardown_kw.keywords:
1331             if self.start_teardown_kw(keyword) is not False:
1332                 self.visit_teardown_kw(keyword)
1333                 self.end_teardown_kw(keyword)
1334
1335     def start_teardown_kw(self, teardown_kw):
1336         """Called when teardown keyword starts
1337
1338         :param teardown_kw: Keyword to process.
1339         :type teardown_kw: Keyword
1340         :returns: Nothing.
1341         """
1342         if teardown_kw.name.count(u"Show Papi History On All Duts"):
1343             self._conf_history_lookup_nr = 0
1344             self._msg_type = u"teardown-papi-history"
1345             teardown_kw.messages.visit(self)
1346
1347     def end_teardown_kw(self, teardown_kw):
1348         """Called when keyword ends. Default implementation does nothing.
1349
1350         :param teardown_kw: Keyword to process.
1351         :type teardown_kw: Keyword
1352         :returns: Nothing.
1353         """
1354
1355     def visit_message(self, msg):
1356         """Implements visiting the message.
1357
1358         :param msg: Message to process.
1359         :type msg: Message
1360         :returns: Nothing.
1361         """
1362         if self.start_message(msg) is not False:
1363             self.end_message(msg)
1364
1365     def start_message(self, msg):
1366         """Called when message starts. Get required information from messages:
1367         - VPP version.
1368
1369         :param msg: Message to process.
1370         :type msg: Message
1371         :returns: Nothing.
1372         """
1373         if self._msg_type:
1374             self.parse_msg[self._msg_type](msg)
1375
1376     def end_message(self, msg):
1377         """Called when message ends. Default implementation does nothing.
1378
1379         :param msg: Message to process.
1380         :type msg: Message
1381         :returns: Nothing.
1382         """
1383
1384
1385 class InputData:
1386     """Input data
1387
1388     The data is extracted from output.xml files generated by Jenkins jobs and
1389     stored in pandas' DataFrames.
1390
1391     The data structure:
1392     - job name
1393       - build number
1394         - metadata
1395           (as described in ExecutionChecker documentation)
1396         - suites
1397           (as described in ExecutionChecker documentation)
1398         - tests
1399           (as described in ExecutionChecker documentation)
1400     """
1401
1402     def __init__(self, spec):
1403         """Initialization.
1404
1405         :param spec: Specification.
1406         :type spec: Specification
1407         """
1408
1409         # Specification:
1410         self._cfg = spec
1411
1412         # Data store:
1413         self._input_data = pd.Series()
1414
1415     @property
1416     def data(self):
1417         """Getter - Input data.
1418
1419         :returns: Input data
1420         :rtype: pandas.Series
1421         """
1422         return self._input_data
1423
1424     def metadata(self, job, build):
1425         """Getter - metadata
1426
1427         :param job: Job which metadata we want.
1428         :param build: Build which metadata we want.
1429         :type job: str
1430         :type build: str
1431         :returns: Metadata
1432         :rtype: pandas.Series
1433         """
1434         return self.data[job][build][u"metadata"]
1435
1436     def suites(self, job, build):
1437         """Getter - suites
1438
1439         :param job: Job which suites we want.
1440         :param build: Build which suites we want.
1441         :type job: str
1442         :type build: str
1443         :returns: Suites.
1444         :rtype: pandas.Series
1445         """
1446         return self.data[job][str(build)][u"suites"]
1447
1448     def tests(self, job, build):
1449         """Getter - tests
1450
1451         :param job: Job which tests we want.
1452         :param build: Build which tests we want.
1453         :type job: str
1454         :type build: str
1455         :returns: Tests.
1456         :rtype: pandas.Series
1457         """
1458         return self.data[job][build][u"tests"]
1459
1460     def _parse_tests(self, job, build):
1461         """Process data from robot output.xml file and return JSON structured
1462         data.
1463
1464         :param job: The name of job which build output data will be processed.
1465         :param build: The build which output data will be processed.
1466         :type job: str
1467         :type build: dict
1468         :returns: JSON data structure.
1469         :rtype: dict
1470         """
1471
1472         metadata = {
1473             u"job": job,
1474             u"build": build
1475         }
1476
1477         with open(build[u"file-name"], u'r') as data_file:
1478             try:
1479                 result = ExecutionResult(data_file)
1480             except errors.DataError as err:
1481                 logging.error(
1482                     f"Error occurred while parsing output.xml: {repr(err)}"
1483                 )
1484                 return None
1485         checker = ExecutionChecker(
1486             metadata, self._cfg.mapping, self._cfg.ignore
1487         )
1488         result.visit(checker)
1489
1490         return checker.data
1491
1492     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1493         """Download and parse the input data file.
1494
1495         :param pid: PID of the process executing this method.
1496         :param job: Name of the Jenkins job which generated the processed input
1497             file.
1498         :param build: Information about the Jenkins build which generated the
1499             processed input file.
1500         :param repeat: Repeat the download specified number of times if not
1501             successful.
1502         :type pid: int
1503         :type job: str
1504         :type build: dict
1505         :type repeat: int
1506         """
1507
1508         logging.info(f"Processing the job/build: {job}: {build[u'build']}")
1509
1510         state = u"failed"
1511         success = False
1512         data = None
1513         do_repeat = repeat
1514         while do_repeat:
1515             success = download_and_unzip_data_file(self._cfg, job, build, pid)
1516             if success:
1517                 break
1518             do_repeat -= 1
1519         if not success:
1520             logging.error(
1521                 f"It is not possible to download the input data file from the "
1522                 f"job {job}, build {build[u'build']}, or it is damaged. "
1523                 f"Skipped."
1524             )
1525         if success:
1526             logging.info(f"  Processing data from build {build[u'build']}")
1527             data = self._parse_tests(job, build)
1528             if data is None:
1529                 logging.error(
1530                     f"Input data file from the job {job}, build "
1531                     f"{build[u'build']} is damaged. Skipped."
1532                 )
1533             else:
1534                 state = u"processed"
1535
1536             try:
1537                 remove(build[u"file-name"])
1538             except OSError as err:
1539                 logging.error(
1540                     f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1541                 )
1542
1543         # If the time-period is defined in the specification file, remove all
1544         # files which are outside the time period.
1545         is_last = False
1546         timeperiod = self._cfg.environment.get(u"time-period", None)
1547         if timeperiod and data:
1548             now = dt.utcnow()
1549             timeperiod = timedelta(int(timeperiod))
1550             metadata = data.get(u"metadata", None)
1551             if metadata:
1552                 generated = metadata.get(u"generated", None)
1553                 if generated:
1554                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1555                     if (now - generated) > timeperiod:
1556                         # Remove the data and the file:
1557                         state = u"removed"
1558                         data = None
1559                         is_last = True
1560                         logging.info(
1561                             f"  The build {job}/{build[u'build']} is "
1562                             f"outdated, will be removed."
1563                         )
1564         return {
1565             u"data": data,
1566             u"state": state,
1567             u"job": job,
1568             u"build": build,
1569             u"last": is_last
1570         }
1571
1572     def download_and_parse_data(self, repeat=1):
1573         """Download the input data files, parse input data from input files and
1574         store in pandas' Series.
1575
1576         :param repeat: Repeat the download specified number of times if not
1577             successful.
1578         :type repeat: int
1579         """
1580
1581         logging.info(u"Downloading and parsing input files ...")
1582
1583         for job, builds in self._cfg.input.items():
1584             for build in builds:
1585
1586                 result = self._download_and_parse_build(job, build, repeat)
1587                 if result[u"last"]:
1588                     break
1589                 build_nr = result[u"build"][u"build"]
1590
1591                 if result[u"data"]:
1592                     data = result[u"data"]
1593                     build_data = pd.Series({
1594                         u"metadata": pd.Series(
1595                             list(data[u"metadata"].values()),
1596                             index=list(data[u"metadata"].keys())
1597                         ),
1598                         u"suites": pd.Series(
1599                             list(data[u"suites"].values()),
1600                             index=list(data[u"suites"].keys())
1601                         ),
1602                         u"tests": pd.Series(
1603                             list(data[u"tests"].values()),
1604                             index=list(data[u"tests"].keys())
1605                         )
1606                     })
1607
1608                     if self._input_data.get(job, None) is None:
1609                         self._input_data[job] = pd.Series()
1610                     self._input_data[job][str(build_nr)] = build_data
1611                     self._cfg.set_input_file_name(
1612                         job, build_nr, result[u"build"][u"file-name"]
1613                     )
1614                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1615
1616                 mem_alloc = \
1617                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1618                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1619
1620         logging.info(u"Done.")
1621
1622         msg = f"Successful downloads from the sources:\n"
1623         for source in self._cfg.environment[u"data-sources"]:
1624             if source[u"successful-downloads"]:
1625                 msg += (
1626                     f"{source[u'url']}/{source[u'path']}/"
1627                     f"{source[u'file-name']}: "
1628                     f"{source[u'successful-downloads']}\n"
1629                 )
1630         logging.info(msg)
1631
1632     def process_local_file(self, local_file, job=u"local", build_nr=1,
1633                            replace=True):
1634         """Process local XML file given as a command-line parameter.
1635
1636         :param local_file: The file to process.
1637         :param job: Job name.
1638         :param build_nr: Build number.
1639         :param replace: If True, the information about jobs and builds is
1640             replaced by the new one, otherwise the new jobs and builds are
1641             added.
1642         :type local_file: str
1643         :type job: str
1644         :type build_nr: int
1645         :type replace: bool
1646         :raises: PresentationError if an error occurs.
1647         """
1648         if not isfile(local_file):
1649             raise PresentationError(f"The file {local_file} does not exist.")
1650
1651         try:
1652             build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1653         except (IndexError, ValueError):
1654             pass
1655
1656         build = {
1657             u"build": build_nr,
1658             u"status": u"failed",
1659             u"file-name": local_file
1660         }
1661         if replace:
1662             self._cfg.input = dict()
1663         self._cfg.add_build(job, build)
1664
1665         logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1666         data = self._parse_tests(job, build)
1667         if data is None:
1668             raise PresentationError(
1669                 f"Error occurred while parsing the file {local_file}"
1670             )
1671
1672         build_data = pd.Series({
1673             u"metadata": pd.Series(
1674                 list(data[u"metadata"].values()),
1675                 index=list(data[u"metadata"].keys())
1676             ),
1677             u"suites": pd.Series(
1678                 list(data[u"suites"].values()),
1679                 index=list(data[u"suites"].keys())
1680             ),
1681             u"tests": pd.Series(
1682                 list(data[u"tests"].values()),
1683                 index=list(data[u"tests"].keys())
1684             )
1685         })
1686
1687         if self._input_data.get(job, None) is None:
1688             self._input_data[job] = pd.Series()
1689         self._input_data[job][str(build_nr)] = build_data
1690
1691         self._cfg.set_input_state(job, build_nr, u"processed")
1692
1693     def process_local_directory(self, local_dir, replace=True):
1694         """Process local directory with XML file(s). The directory is processed
1695         as a 'job' and the XML files in it as builds.
1696         If the given directory contains only sub-directories, these
1697         sub-directories processed as jobs and corresponding XML files as builds
1698         of their job.
1699
1700         :param local_dir: Local directory to process.
1701         :param replace: If True, the information about jobs and builds is
1702             replaced by the new one, otherwise the new jobs and builds are
1703             added.
1704         :type local_dir: str
1705         :type replace: bool
1706         """
1707         if not isdir(local_dir):
1708             raise PresentationError(
1709                 f"The directory {local_dir} does not exist."
1710             )
1711
1712         # Check if the given directory includes only files, or only directories
1713         _, dirnames, filenames = next(walk(local_dir))
1714
1715         if filenames and not dirnames:
1716             filenames.sort()
1717             # local_builds:
1718             # key: dir (job) name, value: list of file names (builds)
1719             local_builds = {
1720                 local_dir: [join(local_dir, name) for name in filenames]
1721             }
1722
1723         elif dirnames and not filenames:
1724             dirnames.sort()
1725             # local_builds:
1726             # key: dir (job) name, value: list of file names (builds)
1727             local_builds = dict()
1728             for dirname in dirnames:
1729                 builds = [
1730                     join(local_dir, dirname, name)
1731                     for name in listdir(join(local_dir, dirname))
1732                     if isfile(join(local_dir, dirname, name))
1733                 ]
1734                 if builds:
1735                     local_builds[dirname] = sorted(builds)
1736
1737         elif not filenames and not dirnames:
1738             raise PresentationError(f"The directory {local_dir} is empty.")
1739         else:
1740             raise PresentationError(
1741                 f"The directory {local_dir} can include only files or only "
1742                 f"directories, not both.\nThe directory {local_dir} includes "
1743                 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1744             )
1745
1746         if replace:
1747             self._cfg.input = dict()
1748
1749         for job, files in local_builds.items():
1750             for idx, local_file in enumerate(files):
1751                 self.process_local_file(local_file, job, idx + 1, replace=False)
1752
1753     @staticmethod
1754     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1755         """Return the index of character in the string which is the end of tag.
1756
1757         :param tag_filter: The string where the end of tag is being searched.
1758         :param start: The index where the searching is stated.
1759         :param closer: The character which is the tag closer.
1760         :type tag_filter: str
1761         :type start: int
1762         :type closer: str
1763         :returns: The index of the tag closer.
1764         :rtype: int
1765         """
1766         try:
1767             idx_opener = tag_filter.index(closer, start)
1768             return tag_filter.index(closer, idx_opener + 1)
1769         except ValueError:
1770             return None
1771
1772     @staticmethod
1773     def _condition(tag_filter):
1774         """Create a conditional statement from the given tag filter.
1775
1776         :param tag_filter: Filter based on tags from the element specification.
1777         :type tag_filter: str
1778         :returns: Conditional statement which can be evaluated.
1779         :rtype: str
1780         """
1781         index = 0
1782         while True:
1783             index = InputData._end_of_tag(tag_filter, index)
1784             if index is None:
1785                 return tag_filter
1786             index += 1
1787             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1788
1789     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1790                     continue_on_error=False):
1791         """Filter required data from the given jobs and builds.
1792
1793         The output data structure is:
1794         - job 1
1795           - build 1
1796             - test (or suite) 1 ID:
1797               - param 1
1798               - param 2
1799               ...
1800               - param n
1801             ...
1802             - test (or suite) n ID:
1803             ...
1804           ...
1805           - build n
1806         ...
1807         - job n
1808
1809         :param element: Element which will use the filtered data.
1810         :param params: Parameters which will be included in the output. If None,
1811             all parameters are included.
1812         :param data: If not None, this data is used instead of data specified
1813             in the element.
1814         :param data_set: The set of data to be filtered: tests, suites,
1815             metadata.
1816         :param continue_on_error: Continue if there is error while reading the
1817             data. The Item will be empty then
1818         :type element: pandas.Series
1819         :type params: list
1820         :type data: dict
1821         :type data_set: str
1822         :type continue_on_error: bool
1823         :returns: Filtered data.
1824         :rtype pandas.Series
1825         """
1826
1827         try:
1828             if data_set == "suites":
1829                 cond = u"True"
1830             elif element[u"filter"] in (u"all", u"template"):
1831                 cond = u"True"
1832             else:
1833                 cond = InputData._condition(element[u"filter"])
1834             logging.debug(f"   Filter: {cond}")
1835         except KeyError:
1836             logging.error(u"  No filter defined.")
1837             return None
1838
1839         if params is None:
1840             params = element.get(u"parameters", None)
1841             if params:
1842                 params.extend((u"type", u"status"))
1843
1844         data_to_filter = data if data else element[u"data"]
1845         data = pd.Series()
1846         try:
1847             for job, builds in data_to_filter.items():
1848                 data[job] = pd.Series()
1849                 for build in builds:
1850                     data[job][str(build)] = pd.Series()
1851                     try:
1852                         data_dict = dict(
1853                             self.data[job][str(build)][data_set].items())
1854                     except KeyError:
1855                         if continue_on_error:
1856                             continue
1857                         return None
1858
1859                     for test_id, test_data in data_dict.items():
1860                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1861                             data[job][str(build)][test_id] = pd.Series()
1862                             if params is None:
1863                                 for param, val in test_data.items():
1864                                     data[job][str(build)][test_id][param] = val
1865                             else:
1866                                 for param in params:
1867                                     try:
1868                                         data[job][str(build)][test_id][param] =\
1869                                             test_data[param]
1870                                     except KeyError:
1871                                         data[job][str(build)][test_id][param] =\
1872                                             u"No Data"
1873             return data
1874
1875         except (KeyError, IndexError, ValueError) as err:
1876             logging.error(
1877                 f"Missing mandatory parameter in the element specification: "
1878                 f"{repr(err)}"
1879             )
1880             return None
1881         except AttributeError as err:
1882             logging.error(repr(err))
1883             return None
1884         except SyntaxError as err:
1885             logging.error(
1886                 f"The filter {cond} is not correct. Check if all tags are "
1887                 f"enclosed by apostrophes.\n{repr(err)}"
1888             )
1889             return None
1890
1891     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1892                              continue_on_error=False):
1893         """Filter required data from the given jobs and builds.
1894
1895         The output data structure is:
1896         - job 1
1897           - build 1
1898             - test (or suite) 1 ID:
1899               - param 1
1900               - param 2
1901               ...
1902               - param n
1903             ...
1904             - test (or suite) n ID:
1905             ...
1906           ...
1907           - build n
1908         ...
1909         - job n
1910
1911         :param element: Element which will use the filtered data.
1912         :param params: Parameters which will be included in the output. If None,
1913         all parameters are included.
1914         :param data_set: The set of data to be filtered: tests, suites,
1915         metadata.
1916         :param continue_on_error: Continue if there is error while reading the
1917         data. The Item will be empty then
1918         :type element: pandas.Series
1919         :type params: list
1920         :type data_set: str
1921         :type continue_on_error: bool
1922         :returns: Filtered data.
1923         :rtype pandas.Series
1924         """
1925
1926         include = element.get(u"include", None)
1927         if not include:
1928             logging.warning(u"No tests to include, skipping the element.")
1929             return None
1930
1931         if params is None:
1932             params = element.get(u"parameters", None)
1933             if params and u"type" not in params:
1934                 params.append(u"type")
1935
1936         cores = element.get(u"core", None)
1937         if cores:
1938             tests = list()
1939             for core in cores:
1940                 for test in include:
1941                     tests.append(test.format(core=core))
1942         else:
1943             tests = include
1944
1945         data = pd.Series()
1946         try:
1947             for job, builds in element[u"data"].items():
1948                 data[job] = pd.Series()
1949                 for build in builds:
1950                     data[job][str(build)] = pd.Series()
1951                     for test in tests:
1952                         try:
1953                             reg_ex = re.compile(str(test).lower())
1954                             for test_id in self.data[job][
1955                                     str(build)][data_set].keys():
1956                                 if re.match(reg_ex, str(test_id).lower()):
1957                                     test_data = self.data[job][
1958                                         str(build)][data_set][test_id]
1959                                     data[job][str(build)][test_id] = pd.Series()
1960                                     if params is None:
1961                                         for param, val in test_data.items():
1962                                             data[job][str(build)][test_id]\
1963                                                 [param] = val
1964                                     else:
1965                                         for param in params:
1966                                             try:
1967                                                 data[job][str(build)][
1968                                                     test_id][param] = \
1969                                                     test_data[param]
1970                                             except KeyError:
1971                                                 data[job][str(build)][
1972                                                     test_id][param] = u"No Data"
1973                         except KeyError as err:
1974                             if continue_on_error:
1975                                 logging.debug(repr(err))
1976                                 continue
1977                             logging.error(repr(err))
1978                             return None
1979             return data
1980
1981         except (KeyError, IndexError, ValueError) as err:
1982             logging.error(
1983                 f"Missing mandatory parameter in the element "
1984                 f"specification: {repr(err)}"
1985             )
1986             return None
1987         except AttributeError as err:
1988             logging.error(repr(err))
1989             return None
1990
1991     @staticmethod
1992     def merge_data(data):
1993         """Merge data from more jobs and builds to a simple data structure.
1994
1995         The output data structure is:
1996
1997         - test (suite) 1 ID:
1998           - param 1
1999           - param 2
2000           ...
2001           - param n
2002         ...
2003         - test (suite) n ID:
2004         ...
2005
2006         :param data: Data to merge.
2007         :type data: pandas.Series
2008         :returns: Merged data.
2009         :rtype: pandas.Series
2010         """
2011
2012         logging.info(u"    Merging data ...")
2013
2014         merged_data = pd.Series()
2015         for builds in data.values:
2016             for item in builds.values:
2017                 for item_id, item_data in item.items():
2018                     merged_data[item_id] = item_data
2019         return merged_data
2020
2021     def print_all_oper_data(self):
2022         """Print all operational data to console.
2023         """
2024
2025         tbl_hdr = (
2026             u"Name",
2027             u"Nr of Vectors",
2028             u"Nr of Packets",
2029             u"Suspends",
2030             u"Cycles per Packet",
2031             u"Average Vector Size"
2032         )
2033
2034         for job in self._input_data.values:
2035             for build in job.values:
2036                 for test_id, test_data in build[u"tests"].items():
2037                     print(f"{test_id}")
2038                     if test_data.get(u"show-run", None) is None:
2039                         continue
2040                     for dut_name, data in test_data[u"show-run"].items():
2041                         if data.get(u"threads", None) is None:
2042                             continue
2043                         print(f"Host IP: {data.get(u'host', '')}, "
2044                               f"Socket: {data.get(u'socket', '')}")
2045                         for thread_nr, thread in data[u"threads"].items():
2046                             txt_table = prettytable.PrettyTable(tbl_hdr)
2047                             avg = 0.0
2048                             for row in thread:
2049                                 txt_table.add_row(row)
2050                                 avg += row[-1]
2051                             if len(thread) == 0:
2052                                 avg = u""
2053                             else:
2054                                 avg = f", Average Vector Size per Node: " \
2055                                       f"{(avg / len(thread)):.2f}"
2056                             th_name = u"main" if thread_nr == 0 \
2057                                 else f"worker_{thread_nr}"
2058                             print(f"{dut_name}, {th_name}{avg}")
2059                             txt_table.float_format = u".2"
2060                             txt_table.align = u"r"
2061                             txt_table.align[u"Name"] = u"l"
2062                             print(f"{txt_table.get_string()}\n")