resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove, walk, listdir
  29 from os.path import isfile, isdir, join
  30 from datetime import datetime as dt
  31 from datetime import timedelta
  32 from json import loads
  33 from json.decoder import JSONDecodeError
  34
  35 import hdrh.histogram
  36 import hdrh.codec
  37 import prettytable
  38 import pandas as pd
  39
  40 from robot.api import ExecutionResult, ResultVisitor
  41 from robot import errors
  42
  43 from resources.libraries.python import jumpavg
  44 from input_data_files import download_and_unzip_data_file
  45 from pal_errors import PresentationError
  46
  47
  48 # Separator used in file names
  49 SEPARATOR = u"__"
  50
  51
  52 class ExecutionChecker(ResultVisitor):
  53     """Class to traverse through the test suite structure.
  54
  55     The functionality implemented in this class generates a json structure:
  56
  57     Performance tests:
  58
  59     {
  60         "metadata": {
  61             "generated": "Timestamp",
  62             "version": "SUT version",
  63             "job": "Jenkins job name",
  64             "build": "Information about the build"
  65         },
  66         "suites": {
  67             "Suite long name 1": {
  68                 "name": Suite name,
  69                 "doc": "Suite 1 documentation",
  70                 "parent": "Suite 1 parent",
  71                 "level": "Level of the suite in the suite hierarchy"
  72             }
  73             "Suite long name N": {
  74                 "name": Suite name,
  75                 "doc": "Suite N documentation",
  76                 "parent": "Suite 2 parent",
  77                 "level": "Level of the suite in the suite hierarchy"
  78             }
  79         }
  80         "tests": {
  81             # NDRPDR tests:
  82             "ID": {
  83                 "name": "Test name",
  84                 "parent": "Name of the parent of the test",
  85                 "doc": "Test documentation",
  86                 "msg": "Test message",
  87                 "conf-history": "DUT1 and DUT2 VAT History",
  88                 "show-run": "Show Run",
  89                 "tags": ["tag 1", "tag 2", "tag n"],
  90                 "type": "NDRPDR",
  91                 "status": "PASS" | "FAIL",
  92                 "throughput": {
  93                     "NDR": {
  94                         "LOWER": float,
  95                         "UPPER": float
  96                     },
  97                     "PDR": {
  98                         "LOWER": float,
  99                         "UPPER": float
 100                     }
 101                 },
 102                 "latency": {
 103                     "NDR": {
 104                         "direction1": {
 105                             "min": float,
 106                             "avg": float,
 107                             "max": float,
 108                             "hdrh": str
 109                         },
 110                         "direction2": {
 111                             "min": float,
 112                             "avg": float,
 113                             "max": float,
 114                             "hdrh": str
 115                         }
 116                     },
 117                     "PDR": {
 118                         "direction1": {
 119                             "min": float,
 120                             "avg": float,
 121                             "max": float,
 122                             "hdrh": str
 123                         },
 124                         "direction2": {
 125                             "min": float,
 126                             "avg": float,
 127                             "max": float,
 128                             "hdrh": str
 129                         }
 130                     }
 131                 }
 132             }
 133
 134             # TCP tests:
 135             "ID": {
 136                 "name": "Test name",
 137                 "parent": "Name of the parent of the test",
 138                 "doc": "Test documentation",
 139                 "msg": "Test message",
 140                 "tags": ["tag 1", "tag 2", "tag n"],
 141                 "type": "TCP",
 142                 "status": "PASS" | "FAIL",
 143                 "result": int
 144             }
 145
 146             # MRR, BMRR tests:
 147             "ID": {
 148                 "name": "Test name",
 149                 "parent": "Name of the parent of the test",
 150                 "doc": "Test documentation",
 151                 "msg": "Test message",
 152                 "tags": ["tag 1", "tag 2", "tag n"],
 153                 "type": "MRR" | "BMRR",
 154                 "status": "PASS" | "FAIL",
 155                 "result": {
 156                     "receive-rate": float,
 157                     # Average of a list, computed using AvgStdevStats.
 158                     # In CSIT-1180, replace with List[float].
 159                 }
 160             }
 161
 162             "ID" {
 163                 # next test
 164             }
 165         }
 166     }
 167
 168
 169     Functional tests:
 170
 171     {
 172         "metadata": {  # Optional
 173             "version": "VPP version",
 174             "job": "Jenkins job name",
 175             "build": "Information about the build"
 176         },
 177         "suites": {
 178             "Suite name 1": {
 179                 "doc": "Suite 1 documentation",
 180                 "parent": "Suite 1 parent",
 181                 "level": "Level of the suite in the suite hierarchy"
 182             }
 183             "Suite name N": {
 184                 "doc": "Suite N documentation",
 185                 "parent": "Suite 2 parent",
 186                 "level": "Level of the suite in the suite hierarchy"
 187             }
 188         }
 189         "tests": {
 190             "ID": {
 191                 "name": "Test name",
 192                 "parent": "Name of the parent of the test",
 193                 "doc": "Test documentation"
 194                 "msg": "Test message"
 195                 "tags": ["tag 1", "tag 2", "tag n"],
 196                 "conf-history": "DUT1 and DUT2 VAT History"
 197                 "show-run": "Show Run"
 198                 "status": "PASS" | "FAIL"
 199             },
 200             "ID" {
 201                 # next test
 202             }
 203         }
 204     }
 205
 206     .. note:: ID is the lowercase full path to the test.
 207     """
 208
 209     REGEX_PLR_RATE = re.compile(
 210         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 211         r'PLRsearch upper bound::?\s(\d+.\d+)'
 212     )
 213     REGEX_NDRPDR_RATE = re.compile(
 214         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 215         r'NDR_UPPER:\s(\d+.\d+).*\n'
 216         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 217         r'PDR_UPPER:\s(\d+.\d+)'
 218     )
 219     REGEX_NDRPDR_GBPS = re.compile(
 220         r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
 221         r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
 222         r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
 223         r'PDR_UPPER:.*,\s(\d+.\d+)'
 224     )
 225     REGEX_PERF_MSG_INFO = re.compile(
 226         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 227         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 228         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 229         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 230         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 231     )
 232     REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
 233
 234     # TODO: Remove when not needed
 235     REGEX_NDRPDR_LAT_BASE = re.compile(
 236         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 237         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 238     )
 239     REGEX_NDRPDR_LAT = re.compile(
 240         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 241         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 242         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 243         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 244         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 245         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 246     )
 247     # TODO: Remove when not needed
 248     REGEX_NDRPDR_LAT_LONG = re.compile(
 249         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 250         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 251         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 252         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 253         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 254         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 255         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 256         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 257         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 258     )
 259     REGEX_VERSION_VPP = re.compile(
 260         r"(return STDOUT Version:\s*|"
 261         r"VPP Version:\s*|VPP version:\s*)(.*)"
 262     )
 263     REGEX_VERSION_DPDK = re.compile(
 264         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 265     )
 266     REGEX_TCP = re.compile(
 267         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 268     )
 269     REGEX_MRR = re.compile(
 270         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 271         r'tx\s(\d*),\srx\s(\d*)'
 272     )
 273     REGEX_BMRR = re.compile(
 274         r'Maximum Receive Rate trial results'
 275         r' in packets per second: \[(.*)\]'
 276     )
 277     REGEX_RECONF_LOSS = re.compile(
 278         r'Packets lost due to reconfig: (\d*)'
 279     )
 280     REGEX_RECONF_TIME = re.compile(
 281         r'Implied time lost: (\d*.[\de-]*)'
 282     )
 283     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 284
 285     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 286
 287     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 288
 289     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 290
 291     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 292
 293     def __init__(self, metadata, mapping, ignore):
 294         """Initialisation.
 295
 296         :param metadata: Key-value pairs to be included in "metadata" part of
 297             JSON structure.
 298         :param mapping: Mapping of the old names of test cases to the new
 299             (actual) one.
 300         :param ignore: List of TCs to be ignored.
 301         :type metadata: dict
 302         :type mapping: dict
 303         :type ignore: list
 304         """
 305
 306         # Type of message to parse out from the test messages
 307         self._msg_type = None
 308
 309         # VPP version
 310         self._version = None
 311
 312         # Timestamp
 313         self._timestamp = None
 314
 315         # Testbed. The testbed is identified by TG node IP address.
 316         self._testbed = None
 317
 318         # Mapping of TCs long names
 319         self._mapping = mapping
 320
 321         # Ignore list
 322         self._ignore = ignore
 323
 324         # Number of PAPI History messages found:
 325         # 0 - no message
 326         # 1 - PAPI History of DUT1
 327         # 2 - PAPI History of DUT2
 328         self._conf_history_lookup_nr = 0
 329
 330         self._sh_run_counter = 0
 331
 332         # Test ID of currently processed test- the lowercase full path to the
 333         # test
 334         self._test_id = None
 335
 336         # The main data structure
 337         self._data = {
 338             u"metadata": OrderedDict(),
 339             u"suites": OrderedDict(),
 340             u"tests": OrderedDict()
 341         }
 342
 343         # Save the provided metadata
 344         for key, val in metadata.items():
 345             self._data[u"metadata"][key] = val
 346
 347         # Dictionary defining the methods used to parse different types of
 348         # messages
 349         self.parse_msg = {
 350             u"timestamp": self._get_timestamp,
 351             u"vpp-version": self._get_vpp_version,
 352             u"dpdk-version": self._get_dpdk_version,
 353             # TODO: Remove when not needed:
 354             u"teardown-vat-history": self._get_vat_history,
 355             u"teardown-papi-history": self._get_papi_history,
 356             u"test-show-runtime": self._get_show_run,
 357             u"testbed": self._get_testbed
 358         }
 359
 360     @property
 361     def data(self):
 362         """Getter - Data parsed from the XML file.
 363
 364         :returns: Data parsed from the XML file.
 365         :rtype: dict
 366         """
 367         return self._data
 368
 369     def _get_data_from_mrr_test_msg(self, msg):
 370         """Get info from message of MRR performance tests.
 371
 372         :param msg: Message to be processed.
 373         :type msg: str
 374         :returns: Processed message or original message if a problem occurs.
 375         :rtype: str
 376         """
 377
 378         groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
 379         if not groups or groups.lastindex != 1:
 380             return u"Test Failed."
 381
 382         try:
 383             data = groups.group(1).split(u", ")
 384         except (AttributeError, IndexError, ValueError, KeyError):
 385             return u"Test Failed."
 386
 387         out_str = u"["
 388         try:
 389             for item in data:
 390                 out_str += f"{(float(item) / 1e6):.2f}, "
 391             return out_str[:-2] + u"]"
 392         except (AttributeError, IndexError, ValueError, KeyError):
 393             return u"Test Failed."
 394
 395     def _get_data_from_perf_test_msg(self, msg):
 396         """Get info from message of NDRPDR performance tests.
 397
 398         :param msg: Message to be processed.
 399         :type msg: str
 400         :returns: Processed message or original message if a problem occurs.
 401         :rtype: str
 402         """
 403
 404         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 405         if not groups or groups.lastindex != 10:
 406             return u"Test Failed."
 407
 408         try:
 409             data = {
 410                 u"ndr_low": float(groups.group(1)),
 411                 u"ndr_low_b": float(groups.group(2)),
 412                 u"pdr_low": float(groups.group(3)),
 413                 u"pdr_low_b": float(groups.group(4)),
 414                 u"pdr_lat_90_1": groups.group(5),
 415                 u"pdr_lat_90_2": groups.group(6),
 416                 u"pdr_lat_50_1": groups.group(7),
 417                 u"pdr_lat_50_2": groups.group(8),
 418                 u"pdr_lat_10_1": groups.group(9),
 419                 u"pdr_lat_10_2": groups.group(10),
 420             }
 421         except (AttributeError, IndexError, ValueError, KeyError):
 422             return u"Test Failed."
 423
 424         def _process_lat(in_str_1, in_str_2):
 425             """Extract min, avg, max values from latency string.
 426
 427             :param in_str_1: Latency string for one direction produced by robot
 428                 framework.
 429             :param in_str_2: Latency string for second direction produced by
 430                 robot framework.
 431             :type in_str_1: str
 432             :type in_str_2: str
 433             :returns: Processed latency string or None if a problem occurs.
 434             :rtype: tuple
 435             """
 436             in_list_1 = in_str_1.split('/', 3)
 437             in_list_2 = in_str_2.split('/', 3)
 438
 439             if len(in_list_1) != 4 and len(in_list_2) != 4:
 440                 return None
 441
 442             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 443             try:
 444                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 445             except hdrh.codec.HdrLengthException:
 446                 return None
 447
 448             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 449             try:
 450                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 451             except hdrh.codec.HdrLengthException:
 452                 return None
 453
 454             if hdr_lat_1 and hdr_lat_2:
 455                 hdr_lat = (
 456                     hdr_lat_1.get_value_at_percentile(50.0),
 457                     hdr_lat_1.get_value_at_percentile(90.0),
 458                     hdr_lat_1.get_value_at_percentile(99.0),
 459                     hdr_lat_2.get_value_at_percentile(50.0),
 460                     hdr_lat_2.get_value_at_percentile(90.0),
 461                     hdr_lat_2.get_value_at_percentile(99.0)
 462                 )
 463
 464                 if all(hdr_lat):
 465                     return hdr_lat
 466
 467             return None
 468
 469         try:
 470             out_msg = (
 471                 f"1. {(data[u'ndr_low'] / 1e6):5.2f}      "
 472                 f"{data[u'ndr_low_b']:5.2f}"
 473                 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f}      "
 474                 f"{data[u'pdr_low_b']:5.2f}"
 475             )
 476             latency = (
 477                 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
 478                 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
 479                 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
 480             )
 481             if all(latency):
 482                 max_len = len(str(max((max(item) for item in latency))))
 483                 max_len = 4 if max_len < 4 else max_len
 484
 485                 for idx, lat in enumerate(latency):
 486                     if not idx:
 487                         out_msg += u"\n"
 488                     out_msg += (
 489                         f"\n{idx + 3}. "
 490                         f"{lat[0]:{max_len}d} "
 491                         f"{lat[1]:{max_len}d} "
 492                         f"{lat[2]:{max_len}d}      "
 493                         f"{lat[3]:{max_len}d} "
 494                         f"{lat[4]:{max_len}d} "
 495                         f"{lat[5]:{max_len}d} "
 496                     )
 497
 498             return out_msg
 499
 500         except (AttributeError, IndexError, ValueError, KeyError):
 501             return u"Test Failed."
 502
 503     def _get_testbed(self, msg):
 504         """Called when extraction of testbed IP is required.
 505         The testbed is identified by TG node IP address.
 506
 507         :param msg: Message to process.
 508         :type msg: Message
 509         :returns: Nothing.
 510         """
 511
 512         if msg.message.count(u"Setup of TG node") or \
 513                 msg.message.count(u"Setup of node TG host"):
 514             reg_tg_ip = re.compile(
 515                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 516             try:
 517                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 518             except (KeyError, ValueError, IndexError, AttributeError):
 519                 pass
 520             finally:
 521                 self._data[u"metadata"][u"testbed"] = self._testbed
 522                 self._msg_type = None
 523
 524     def _get_vpp_version(self, msg):
 525         """Called when extraction of VPP version is required.
 526
 527         :param msg: Message to process.
 528         :type msg: Message
 529         :returns: Nothing.
 530         """
 531
 532         if msg.message.count(u"return STDOUT Version:") or \
 533                 msg.message.count(u"VPP Version:") or \
 534                 msg.message.count(u"VPP version:"):
 535             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 536                                 group(2))
 537             self._data[u"metadata"][u"version"] = self._version
 538             self._msg_type = None
 539
 540     def _get_dpdk_version(self, msg):
 541         """Called when extraction of DPDK version is required.
 542
 543         :param msg: Message to process.
 544         :type msg: Message
 545         :returns: Nothing.
 546         """
 547
 548         if msg.message.count(u"DPDK Version:"):
 549             try:
 550                 self._version = str(re.search(
 551                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 552                 self._data[u"metadata"][u"version"] = self._version
 553             except IndexError:
 554                 pass
 555             finally:
 556                 self._msg_type = None
 557
 558     def _get_timestamp(self, msg):
 559         """Called when extraction of timestamp is required.
 560
 561         :param msg: Message to process.
 562         :type msg: Message
 563         :returns: Nothing.
 564         """
 565
 566         self._timestamp = msg.timestamp[:14]
 567         self._data[u"metadata"][u"generated"] = self._timestamp
 568         self._msg_type = None
 569
 570     def _get_vat_history(self, msg):
 571         """Called when extraction of VAT command history is required.
 572
 573         TODO: Remove when not needed.
 574
 575         :param msg: Message to process.
 576         :type msg: Message
 577         :returns: Nothing.
 578         """
 579         if msg.message.count(u"VAT command history:"):
 580             self._conf_history_lookup_nr += 1
 581             if self._conf_history_lookup_nr == 1:
 582                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 583             else:
 584                 self._msg_type = None
 585             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 586                           r"VAT command history:", u"",
 587                           msg.message, count=1).replace(u'\n', u' |br| ').\
 588                 replace(u'"', u"'")
 589
 590             self._data[u"tests"][self._test_id][u"conf-history"] += (
 591                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 592             )
 593
 594     def _get_papi_history(self, msg):
 595         """Called when extraction of PAPI command history is required.
 596
 597         :param msg: Message to process.
 598         :type msg: Message
 599         :returns: Nothing.
 600         """
 601         if msg.message.count(u"PAPI command history:"):
 602             self._conf_history_lookup_nr += 1
 603             if self._conf_history_lookup_nr == 1:
 604                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 605             else:
 606                 self._msg_type = None
 607             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 608                           r"PAPI command history:", u"",
 609                           msg.message, count=1).replace(u'\n', u' |br| ').\
 610                 replace(u'"', u"'")
 611             self._data[u"tests"][self._test_id][u"conf-history"] += (
 612                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 613             )
 614
 615     def _get_show_run(self, msg):
 616         """Called when extraction of VPP operational data (output of CLI command
 617         Show Runtime) is required.
 618
 619         :param msg: Message to process.
 620         :type msg: Message
 621         :returns: Nothing.
 622         """
 623
 624         if not msg.message.count(u"stats runtime"):
 625             return
 626
 627         # Temporary solution
 628         if self._sh_run_counter > 1:
 629             return
 630
 631         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 632             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 633
 634         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 635         if not groups:
 636             return
 637         try:
 638             host = groups.group(1)
 639         except (AttributeError, IndexError):
 640             host = u""
 641         try:
 642             sock = groups.group(2)
 643         except (AttributeError, IndexError):
 644             sock = u""
 645
 646         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 647                         replace(u"'", u'"').replace(u'b"', u'"').
 648                         replace(u'u"', u'"').split(u":", 1)[1])
 649
 650         try:
 651             threads_nr = len(runtime[0][u"clocks"])
 652         except (IndexError, KeyError):
 653             return
 654
 655         dut = u"DUT{nr}".format(
 656             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 657
 658         oper = {
 659             u"host": host,
 660             u"socket": sock,
 661             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 662         }
 663
 664         for item in runtime:
 665             for idx in range(threads_nr):
 666                 if item[u"vectors"][idx] > 0:
 667                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 668                 elif item[u"calls"][idx] > 0:
 669                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 670                 elif item[u"suspends"][idx] > 0:
 671                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 672                 else:
 673                     clocks = 0.0
 674
 675                 if item[u"calls"][idx] > 0:
 676                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 677                 else:
 678                     vectors_call = 0.0
 679
 680                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 681                         int(item[u"suspends"][idx]):
 682                     oper[u"threads"][idx].append([
 683                         item[u"name"],
 684                         item[u"calls"][idx],
 685                         item[u"vectors"][idx],
 686                         item[u"suspends"][idx],
 687                         clocks,
 688                         vectors_call
 689                     ])
 690
 691         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 692
 693     def _get_ndrpdr_throughput(self, msg):
 694         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 695         message.
 696
 697         :param msg: The test message to be parsed.
 698         :type msg: str
 699         :returns: Parsed data as a dict and the status (PASS/FAIL).
 700         :rtype: tuple(dict, str)
 701         """
 702
 703         throughput = {
 704             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 705             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 706         }
 707         status = u"FAIL"
 708         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 709
 710         if groups is not None:
 711             try:
 712                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 713                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 714                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 715                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 716                 status = u"PASS"
 717             except (IndexError, ValueError):
 718                 pass
 719
 720         return throughput, status
 721
 722     def _get_ndrpdr_throughput_gbps(self, msg):
 723         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
 724         test message.
 725
 726         :param msg: The test message to be parsed.
 727         :type msg: str
 728         :returns: Parsed data as a dict and the status (PASS/FAIL).
 729         :rtype: tuple(dict, str)
 730         """
 731
 732         gbps = {
 733             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 734             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 735         }
 736         status = u"FAIL"
 737         groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
 738
 739         if groups is not None:
 740             try:
 741                 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
 742                 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
 743                 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
 744                 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
 745                 status = u"PASS"
 746             except (IndexError, ValueError):
 747                 pass
 748
 749         return gbps, status
 750
 751     def _get_plr_throughput(self, msg):
 752         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 753         message.
 754
 755         :param msg: The test message to be parsed.
 756         :type msg: str
 757         :returns: Parsed data as a dict and the status (PASS/FAIL).
 758         :rtype: tuple(dict, str)
 759         """
 760
 761         throughput = {
 762             u"LOWER": -1.0,
 763             u"UPPER": -1.0
 764         }
 765         status = u"FAIL"
 766         groups = re.search(self.REGEX_PLR_RATE, msg)
 767
 768         if groups is not None:
 769             try:
 770                 throughput[u"LOWER"] = float(groups.group(1))
 771                 throughput[u"UPPER"] = float(groups.group(2))
 772                 status = u"PASS"
 773             except (IndexError, ValueError):
 774                 pass
 775
 776         return throughput, status
 777
 778     def _get_ndrpdr_latency(self, msg):
 779         """Get LATENCY from the test message.
 780
 781         :param msg: The test message to be parsed.
 782         :type msg: str
 783         :returns: Parsed data as a dict and the status (PASS/FAIL).
 784         :rtype: tuple(dict, str)
 785         """
 786         latency_default = {
 787             u"min": -1.0,
 788             u"avg": -1.0,
 789             u"max": -1.0,
 790             u"hdrh": u""
 791         }
 792         latency = {
 793             u"NDR": {
 794                 u"direction1": copy.copy(latency_default),
 795                 u"direction2": copy.copy(latency_default)
 796             },
 797             u"PDR": {
 798                 u"direction1": copy.copy(latency_default),
 799                 u"direction2": copy.copy(latency_default)
 800             },
 801             u"LAT0": {
 802                 u"direction1": copy.copy(latency_default),
 803                 u"direction2": copy.copy(latency_default)
 804             },
 805             u"PDR10": {
 806                 u"direction1": copy.copy(latency_default),
 807                 u"direction2": copy.copy(latency_default)
 808             },
 809             u"PDR50": {
 810                 u"direction1": copy.copy(latency_default),
 811                 u"direction2": copy.copy(latency_default)
 812             },
 813             u"PDR90": {
 814                 u"direction1": copy.copy(latency_default),
 815                 u"direction2": copy.copy(latency_default)
 816             },
 817         }
 818
 819         # TODO: Rewrite when long and base are not needed
 820         groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
 821         if groups is None:
 822             groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 823         if groups is None:
 824             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 825         if groups is None:
 826             return latency, u"FAIL"
 827
 828         def process_latency(in_str):
 829             """Return object with parsed latency values.
 830
 831             TODO: Define class for the return type.
 832
 833             :param in_str: Input string, min/avg/max/hdrh format.
 834             :type in_str: str
 835             :returns: Dict with corresponding keys, except hdrh float values.
 836             :rtype dict:
 837             :throws IndexError: If in_str does not have enough substrings.
 838             :throws ValueError: If a substring does not convert to float.
 839             """
 840             in_list = in_str.split('/', 3)
 841
 842             rval = {
 843                 u"min": float(in_list[0]),
 844                 u"avg": float(in_list[1]),
 845                 u"max": float(in_list[2]),
 846                 u"hdrh": u""
 847             }
 848
 849             if len(in_list) == 4:
 850                 rval[u"hdrh"] = str(in_list[3])
 851
 852             return rval
 853
 854         try:
 855             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 856             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 857             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 858             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 859             if groups.lastindex == 4:
 860                 return latency, u"PASS"
 861         except (IndexError, ValueError):
 862             pass
 863
 864         try:
 865             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 866             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 867             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 868             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 869             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 870             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 871             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 872             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 873             if groups.lastindex == 12:
 874                 return latency, u"PASS"
 875         except (IndexError, ValueError):
 876             pass
 877
 878         # TODO: Remove when not needed
 879         latency[u"NDR10"] = {
 880             u"direction1": copy.copy(latency_default),
 881             u"direction2": copy.copy(latency_default)
 882         }
 883         latency[u"NDR50"] = {
 884             u"direction1": copy.copy(latency_default),
 885             u"direction2": copy.copy(latency_default)
 886         }
 887         latency[u"NDR90"] = {
 888             u"direction1": copy.copy(latency_default),
 889             u"direction2": copy.copy(latency_default)
 890         }
 891         try:
 892             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
 893             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
 894             latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
 895             latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
 896             latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
 897             latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
 898             latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
 899             latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
 900             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
 901             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
 902             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
 903             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
 904             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
 905             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
 906             return latency, u"PASS"
 907         except (IndexError, ValueError):
 908             pass
 909
 910         return latency, u"FAIL"
 911
 912     @staticmethod
 913     def _get_hoststack_data(msg, tags):
 914         """Get data from the hoststack test message.
 915
 916         :param msg: The test message to be parsed.
 917         :param tags: Test tags.
 918         :type msg: str
 919         :type tags: list
 920         :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
 921         :rtype: tuple(dict, str)
 922         """
 923         result = dict()
 924         status = u"FAIL"
 925
 926         msg = msg.replace(u"'", u'"').replace(u" ", u"")
 927         if u"LDPRELOAD" in tags:
 928             try:
 929                 result = loads(msg)
 930                 status = u"PASS"
 931             except JSONDecodeError:
 932                 pass
 933         elif u"VPPECHO" in tags:
 934             try:
 935                 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
 936                 result = dict(
 937                     client=loads(msg_lst[0]),
 938                     server=loads(msg_lst[1])
 939                 )
 940                 status = u"PASS"
 941             except (JSONDecodeError, IndexError):
 942                 pass
 943
 944         return result, status
 945
 946     def visit_suite(self, suite):
 947         """Implements traversing through the suite and its direct children.
 948
 949         :param suite: Suite to process.
 950         :type suite: Suite
 951         :returns: Nothing.
 952         """
 953         if self.start_suite(suite) is not False:
 954             suite.suites.visit(self)
 955             suite.tests.visit(self)
 956             self.end_suite(suite)
 957
 958     def start_suite(self, suite):
 959         """Called when suite starts.
 960
 961         :param suite: Suite to process.
 962         :type suite: Suite
 963         :returns: Nothing.
 964         """
 965
 966         try:
 967             parent_name = suite.parent.name
 968         except AttributeError:
 969             return
 970
 971         doc_str = suite.doc.\
 972             replace(u'"', u"'").\
 973             replace(u'\n', u' ').\
 974             replace(u'\r', u'').\
 975             replace(u'*[', u' |br| *[').\
 976             replace(u"*", u"**").\
 977             replace(u' |br| *[', u'*[', 1)
 978
 979         self._data[u"suites"][suite.longname.lower().
 980                               replace(u'"', u"'").
 981                               replace(u" ", u"_")] = {
 982                                   u"name": suite.name.lower(),
 983                                   u"doc": doc_str,
 984                                   u"parent": parent_name,
 985                                   u"level": len(suite.longname.split(u"."))
 986                               }
 987
 988         suite.keywords.visit(self)
 989
 990     def end_suite(self, suite):
 991         """Called when suite ends.
 992
 993         :param suite: Suite to process.
 994         :type suite: Suite
 995         :returns: Nothing.
 996         """
 997
 998     def visit_test(self, test):
 999         """Implements traversing through the test.
1000
1001         :param test: Test to process.
1002         :type test: Test
1003         :returns: Nothing.
1004         """
1005         if self.start_test(test) is not False:
1006             test.keywords.visit(self)
1007             self.end_test(test)
1008
1009     def start_test(self, test):
1010         """Called when test starts.
1011
1012         :param test: Test to process.
1013         :type test: Test
1014         :returns: Nothing.
1015         """
1016
1017         self._sh_run_counter = 0
1018
1019         longname_orig = test.longname.lower()
1020
1021         # Check the ignore list
1022         if longname_orig in self._ignore:
1023             return
1024
1025         tags = [str(tag) for tag in test.tags]
1026         test_result = dict()
1027
1028         # Change the TC long name and name if defined in the mapping table
1029         longname = self._mapping.get(longname_orig, None)
1030         if longname is not None:
1031             name = longname.split(u'.')[-1]
1032             logging.debug(
1033                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1034                 f"{name}"
1035             )
1036         else:
1037             longname = longname_orig
1038             name = test.name.lower()
1039
1040         # Remove TC number from the TC long name (backward compatibility):
1041         self._test_id = re.sub(
1042             self.REGEX_TC_NUMBER, u"", longname.replace(u"snat", u"nat")
1043         )
1044         # Remove TC number from the TC name (not needed):
1045         test_result[u"name"] = re.sub(
1046             self.REGEX_TC_NUMBER, "", name.replace(u"snat", u"nat")
1047         )
1048
1049         test_result[u"parent"] = test.parent.name.lower().\
1050             replace(u"snat", u"nat")
1051         test_result[u"tags"] = tags
1052         test_result["doc"] = test.doc.\
1053             replace(u'"', u"'").\
1054             replace(u'\n', u' ').\
1055             replace(u'\r', u'').\
1056             replace(u'[', u' |br| [').\
1057             replace(u' |br| [', u'[', 1)
1058         test_result[u"type"] = u"FUNC"
1059         test_result[u"status"] = test.status
1060
1061         if test.status == u"PASS":
1062             if u"NDRPDR" in tags:
1063                 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1064                     test.message).replace(u'\n', u' |br| ').\
1065                     replace(u'\r', u'').replace(u'"', u"'")
1066             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1067                 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1068                     test.message).replace(u'\n', u' |br| ').\
1069                     replace(u'\r', u'').replace(u'"', u"'")
1070             else:
1071                 test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\
1072                     replace(u'\r', u'').replace(u'"', u"'")
1073         else:
1074             test_result[u"msg"] = u"Test Failed."
1075
1076         if u"PERFTEST" in tags:
1077             # Replace info about cores (e.g. -1c-) with the info about threads
1078             # and cores (e.g. -1t1c-) in the long test case names and in the
1079             # test case names if necessary.
1080             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1081             if not groups:
1082                 tag_count = 0
1083                 tag_tc = str()
1084                 for tag in test_result[u"tags"]:
1085                     groups = re.search(self.REGEX_TC_TAG, tag)
1086                     if groups:
1087                         tag_count += 1
1088                         tag_tc = tag
1089
1090                 if tag_count == 1:
1091                     self._test_id = re.sub(
1092                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1093                         self._test_id, count=1
1094                     )
1095                     test_result[u"name"] = re.sub(
1096                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1097                         test_result["name"], count=1
1098                     )
1099                 else:
1100                     test_result[u"status"] = u"FAIL"
1101                     self._data[u"tests"][self._test_id] = test_result
1102                     logging.debug(
1103                         f"The test {self._test_id} has no or more than one "
1104                         f"multi-threading tags.\n"
1105                         f"Tags: {test_result[u'tags']}"
1106                     )
1107                     return
1108
1109         if test.status == u"PASS":
1110             if u"NDRPDR" in tags:
1111                 test_result[u"type"] = u"NDRPDR"
1112                 test_result[u"throughput"], test_result[u"status"] = \
1113                     self._get_ndrpdr_throughput(test.message)
1114                 test_result[u"gbps"], test_result[u"status"] = \
1115                     self._get_ndrpdr_throughput_gbps(test.message)
1116                 test_result[u"latency"], test_result[u"status"] = \
1117                     self._get_ndrpdr_latency(test.message)
1118             elif u"SOAK" in tags:
1119                 test_result[u"type"] = u"SOAK"
1120                 test_result[u"throughput"], test_result[u"status"] = \
1121                     self._get_plr_throughput(test.message)
1122             elif u"HOSTSTACK" in tags:
1123                 test_result[u"type"] = u"HOSTSTACK"
1124                 test_result[u"result"], test_result[u"status"] = \
1125                     self._get_hoststack_data(test.message, tags)
1126             elif u"TCP" in tags:
1127                 test_result[u"type"] = u"TCP"
1128                 groups = re.search(self.REGEX_TCP, test.message)
1129                 test_result[u"result"] = int(groups.group(2))
1130             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1131                 if u"MRR" in tags:
1132                     test_result[u"type"] = u"MRR"
1133                 else:
1134                     test_result[u"type"] = u"BMRR"
1135
1136                 test_result[u"result"] = dict()
1137                 groups = re.search(self.REGEX_BMRR, test.message)
1138                 if groups is not None:
1139                     items_str = groups.group(1)
1140                     items_float = [
1141                         float(item.strip()) for item in items_str.split(",")
1142                     ]
1143                     # Use whole list in CSIT-1180.
1144                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1145                     test_result[u"result"][u"receive-rate"] = stats.avg
1146                     test_result[u"result"][u"receive-stdev"] = stats.stdev
1147                 else:
1148                     groups = re.search(self.REGEX_MRR, test.message)
1149                     test_result[u"result"][u"receive-rate"] = \
1150                         float(groups.group(3)) / float(groups.group(1))
1151             elif u"RECONF" in tags:
1152                 test_result[u"type"] = u"RECONF"
1153                 test_result[u"result"] = None
1154                 try:
1155                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1156                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1157                     test_result[u"result"] = {
1158                         u"loss": int(grps_loss.group(1)),
1159                         u"time": float(grps_time.group(1))
1160                     }
1161                 except (AttributeError, IndexError, ValueError, TypeError):
1162                     test_result[u"status"] = u"FAIL"
1163             elif u"DEVICETEST" in tags:
1164                 test_result[u"type"] = u"DEVICETEST"
1165             else:
1166                 test_result[u"status"] = u"FAIL"
1167                 self._data[u"tests"][self._test_id] = test_result
1168                 return
1169
1170         self._data[u"tests"][self._test_id] = test_result
1171
1172     def end_test(self, test):
1173         """Called when test ends.
1174
1175         :param test: Test to process.
1176         :type test: Test
1177         :returns: Nothing.
1178         """
1179
1180     def visit_keyword(self, keyword):
1181         """Implements traversing through the keyword and its child keywords.
1182
1183         :param keyword: Keyword to process.
1184         :type keyword: Keyword
1185         :returns: Nothing.
1186         """
1187         if self.start_keyword(keyword) is not False:
1188             self.end_keyword(keyword)
1189
1190     def start_keyword(self, keyword):
1191         """Called when keyword starts. Default implementation does nothing.
1192
1193         :param keyword: Keyword to process.
1194         :type keyword: Keyword
1195         :returns: Nothing.
1196         """
1197         try:
1198             if keyword.type == u"setup":
1199                 self.visit_setup_kw(keyword)
1200             elif keyword.type == u"teardown":
1201                 self.visit_teardown_kw(keyword)
1202             else:
1203                 self.visit_test_kw(keyword)
1204         except AttributeError:
1205             pass
1206
1207     def end_keyword(self, keyword):
1208         """Called when keyword ends. Default implementation does nothing.
1209
1210         :param keyword: Keyword to process.
1211         :type keyword: Keyword
1212         :returns: Nothing.
1213         """
1214
1215     def visit_test_kw(self, test_kw):
1216         """Implements traversing through the test keyword and its child
1217         keywords.
1218
1219         :param test_kw: Keyword to process.
1220         :type test_kw: Keyword
1221         :returns: Nothing.
1222         """
1223         for keyword in test_kw.keywords:
1224             if self.start_test_kw(keyword) is not False:
1225                 self.visit_test_kw(keyword)
1226                 self.end_test_kw(keyword)
1227
1228     def start_test_kw(self, test_kw):
1229         """Called when test keyword starts. Default implementation does
1230         nothing.
1231
1232         :param test_kw: Keyword to process.
1233         :type test_kw: Keyword
1234         :returns: Nothing.
1235         """
1236         if test_kw.name.count(u"Show Runtime On All Duts") or \
1237                 test_kw.name.count(u"Show Runtime Counters On All Duts") or \
1238                 test_kw.name.count(u"Vpp Show Runtime On All Duts"):
1239             self._msg_type = u"test-show-runtime"
1240             self._sh_run_counter += 1
1241         else:
1242             return
1243         test_kw.messages.visit(self)
1244
1245     def end_test_kw(self, test_kw):
1246         """Called when keyword ends. Default implementation does nothing.
1247
1248         :param test_kw: Keyword to process.
1249         :type test_kw: Keyword
1250         :returns: Nothing.
1251         """
1252
1253     def visit_setup_kw(self, setup_kw):
1254         """Implements traversing through the teardown keyword and its child
1255         keywords.
1256
1257         :param setup_kw: Keyword to process.
1258         :type setup_kw: Keyword
1259         :returns: Nothing.
1260         """
1261         for keyword in setup_kw.keywords:
1262             if self.start_setup_kw(keyword) is not False:
1263                 self.visit_setup_kw(keyword)
1264                 self.end_setup_kw(keyword)
1265
1266     def start_setup_kw(self, setup_kw):
1267         """Called when teardown keyword starts. Default implementation does
1268         nothing.
1269
1270         :param setup_kw: Keyword to process.
1271         :type setup_kw: Keyword
1272         :returns: Nothing.
1273         """
1274         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1275                 and not self._version:
1276             self._msg_type = u"vpp-version"
1277         elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1278                 not self._version:
1279             self._msg_type = u"dpdk-version"
1280         elif setup_kw.name.count(u"Set Global Variable") \
1281                 and not self._timestamp:
1282             self._msg_type = u"timestamp"
1283         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1284             self._msg_type = u"testbed"
1285         else:
1286             return
1287         setup_kw.messages.visit(self)
1288
1289     def end_setup_kw(self, setup_kw):
1290         """Called when keyword ends. Default implementation does nothing.
1291
1292         :param setup_kw: Keyword to process.
1293         :type setup_kw: Keyword
1294         :returns: Nothing.
1295         """
1296
1297     def visit_teardown_kw(self, teardown_kw):
1298         """Implements traversing through the teardown keyword and its child
1299         keywords.
1300
1301         :param teardown_kw: Keyword to process.
1302         :type teardown_kw: Keyword
1303         :returns: Nothing.
1304         """
1305         for keyword in teardown_kw.keywords:
1306             if self.start_teardown_kw(keyword) is not False:
1307                 self.visit_teardown_kw(keyword)
1308                 self.end_teardown_kw(keyword)
1309
1310     def start_teardown_kw(self, teardown_kw):
1311         """Called when teardown keyword starts
1312
1313         :param teardown_kw: Keyword to process.
1314         :type teardown_kw: Keyword
1315         :returns: Nothing.
1316         """
1317
1318         if teardown_kw.name.count(u"Show Vat History On All Duts"):
1319             # TODO: Remove when not needed:
1320             self._conf_history_lookup_nr = 0
1321             self._msg_type = u"teardown-vat-history"
1322             teardown_kw.messages.visit(self)
1323         elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1324             self._conf_history_lookup_nr = 0
1325             self._msg_type = u"teardown-papi-history"
1326             teardown_kw.messages.visit(self)
1327
1328     def end_teardown_kw(self, teardown_kw):
1329         """Called when keyword ends. Default implementation does nothing.
1330
1331         :param teardown_kw: Keyword to process.
1332         :type teardown_kw: Keyword
1333         :returns: Nothing.
1334         """
1335
1336     def visit_message(self, msg):
1337         """Implements visiting the message.
1338
1339         :param msg: Message to process.
1340         :type msg: Message
1341         :returns: Nothing.
1342         """
1343         if self.start_message(msg) is not False:
1344             self.end_message(msg)
1345
1346     def start_message(self, msg):
1347         """Called when message starts. Get required information from messages:
1348         - VPP version.
1349
1350         :param msg: Message to process.
1351         :type msg: Message
1352         :returns: Nothing.
1353         """
1354         if self._msg_type:
1355             self.parse_msg[self._msg_type](msg)
1356
1357     def end_message(self, msg):
1358         """Called when message ends. Default implementation does nothing.
1359
1360         :param msg: Message to process.
1361         :type msg: Message
1362         :returns: Nothing.
1363         """
1364
1365
1366 class InputData:
1367     """Input data
1368
1369     The data is extracted from output.xml files generated by Jenkins jobs and
1370     stored in pandas' DataFrames.
1371
1372     The data structure:
1373     - job name
1374       - build number
1375         - metadata
1376           (as described in ExecutionChecker documentation)
1377         - suites
1378           (as described in ExecutionChecker documentation)
1379         - tests
1380           (as described in ExecutionChecker documentation)
1381     """
1382
1383     def __init__(self, spec):
1384         """Initialization.
1385
1386         :param spec: Specification.
1387         :type spec: Specification
1388         """
1389
1390         # Specification:
1391         self._cfg = spec
1392
1393         # Data store:
1394         self._input_data = pd.Series()
1395
1396     @property
1397     def data(self):
1398         """Getter - Input data.
1399
1400         :returns: Input data
1401         :rtype: pandas.Series
1402         """
1403         return self._input_data
1404
1405     def metadata(self, job, build):
1406         """Getter - metadata
1407
1408         :param job: Job which metadata we want.
1409         :param build: Build which metadata we want.
1410         :type job: str
1411         :type build: str
1412         :returns: Metadata
1413         :rtype: pandas.Series
1414         """
1415         return self.data[job][build][u"metadata"]
1416
1417     def suites(self, job, build):
1418         """Getter - suites
1419
1420         :param job: Job which suites we want.
1421         :param build: Build which suites we want.
1422         :type job: str
1423         :type build: str
1424         :returns: Suites.
1425         :rtype: pandas.Series
1426         """
1427         return self.data[job][str(build)][u"suites"]
1428
1429     def tests(self, job, build):
1430         """Getter - tests
1431
1432         :param job: Job which tests we want.
1433         :param build: Build which tests we want.
1434         :type job: str
1435         :type build: str
1436         :returns: Tests.
1437         :rtype: pandas.Series
1438         """
1439         return self.data[job][build][u"tests"]
1440
1441     def _parse_tests(self, job, build):
1442         """Process data from robot output.xml file and return JSON structured
1443         data.
1444
1445         :param job: The name of job which build output data will be processed.
1446         :param build: The build which output data will be processed.
1447         :type job: str
1448         :type build: dict
1449         :returns: JSON data structure.
1450         :rtype: dict
1451         """
1452
1453         metadata = {
1454             u"job": job,
1455             u"build": build
1456         }
1457
1458         with open(build[u"file-name"], u'r') as data_file:
1459             try:
1460                 result = ExecutionResult(data_file)
1461             except errors.DataError as err:
1462                 logging.error(
1463                     f"Error occurred while parsing output.xml: {repr(err)}"
1464                 )
1465                 return None
1466         checker = ExecutionChecker(metadata, self._cfg.mapping,
1467                                    self._cfg.ignore)
1468         result.visit(checker)
1469
1470         return checker.data
1471
1472     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1473         """Download and parse the input data file.
1474
1475         :param pid: PID of the process executing this method.
1476         :param job: Name of the Jenkins job which generated the processed input
1477             file.
1478         :param build: Information about the Jenkins build which generated the
1479             processed input file.
1480         :param repeat: Repeat the download specified number of times if not
1481             successful.
1482         :type pid: int
1483         :type job: str
1484         :type build: dict
1485         :type repeat: int
1486         """
1487
1488         logging.info(f"  Processing the job/build: {job}: {build[u'build']}")
1489
1490         state = u"failed"
1491         success = False
1492         data = None
1493         do_repeat = repeat
1494         while do_repeat:
1495             success = download_and_unzip_data_file(self._cfg, job, build, pid)
1496             if success:
1497                 break
1498             do_repeat -= 1
1499         if not success:
1500             logging.error(
1501                 f"It is not possible to download the input data file from the "
1502                 f"job {job}, build {build[u'build']}, or it is damaged. "
1503                 f"Skipped."
1504             )
1505         if success:
1506             logging.info(f"    Processing data from build {build[u'build']}")
1507             data = self._parse_tests(job, build)
1508             if data is None:
1509                 logging.error(
1510                     f"Input data file from the job {job}, build "
1511                     f"{build[u'build']} is damaged. Skipped."
1512                 )
1513             else:
1514                 state = u"processed"
1515
1516             try:
1517                 remove(build[u"file-name"])
1518             except OSError as err:
1519                 logging.error(
1520                     f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1521                 )
1522
1523         # If the time-period is defined in the specification file, remove all
1524         # files which are outside the time period.
1525         is_last = False
1526         timeperiod = self._cfg.input.get(u"time-period", None)
1527         if timeperiod and data:
1528             now = dt.utcnow()
1529             timeperiod = timedelta(int(timeperiod))
1530             metadata = data.get(u"metadata", None)
1531             if metadata:
1532                 generated = metadata.get(u"generated", None)
1533                 if generated:
1534                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1535                     if (now - generated) > timeperiod:
1536                         # Remove the data and the file:
1537                         state = u"removed"
1538                         data = None
1539                         is_last = True
1540                         logging.info(
1541                             f"    The build {job}/{build[u'build']} is "
1542                             f"outdated, will be removed."
1543                         )
1544         logging.info(u"  Done.")
1545
1546         return {
1547             u"data": data,
1548             u"state": state,
1549             u"job": job,
1550             u"build": build,
1551             u"last": is_last
1552         }
1553
1554     def download_and_parse_data(self, repeat=1):
1555         """Download the input data files, parse input data from input files and
1556         store in pandas' Series.
1557
1558         :param repeat: Repeat the download specified number of times if not
1559             successful.
1560         :type repeat: int
1561         """
1562
1563         logging.info(u"Downloading and parsing input files ...")
1564
1565         for job, builds in self._cfg.builds.items():
1566             for build in builds:
1567
1568                 result = self._download_and_parse_build(job, build, repeat)
1569                 if result[u"last"]:
1570                     break
1571                 build_nr = result[u"build"][u"build"]
1572
1573                 if result[u"data"]:
1574                     data = result[u"data"]
1575                     build_data = pd.Series({
1576                         u"metadata": pd.Series(
1577                             list(data[u"metadata"].values()),
1578                             index=list(data[u"metadata"].keys())
1579                         ),
1580                         u"suites": pd.Series(
1581                             list(data[u"suites"].values()),
1582                             index=list(data[u"suites"].keys())
1583                         ),
1584                         u"tests": pd.Series(
1585                             list(data[u"tests"].values()),
1586                             index=list(data[u"tests"].keys())
1587                         )
1588                     })
1589
1590                     if self._input_data.get(job, None) is None:
1591                         self._input_data[job] = pd.Series()
1592                     self._input_data[job][str(build_nr)] = build_data
1593
1594                     self._cfg.set_input_file_name(
1595                         job, build_nr, result[u"build"][u"file-name"])
1596
1597                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1598
1599                 mem_alloc = \
1600                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1601                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1602
1603         logging.info(u"Done.")
1604
1605     def process_local_file(self, local_file, job=u"local", build_nr=1,
1606                            replace=True):
1607         """Process local XML file given as a command-line parameter.
1608
1609         :param local_file: The file to process.
1610         :param job: Job name.
1611         :param build_nr: Build number.
1612         :param replace: If True, the information about jobs and builds is
1613             replaced by the new one, otherwise the new jobs and builds are
1614             added.
1615         :type local_file: str
1616         :type job: str
1617         :type build_nr: int
1618         :type replace: bool
1619         :raises: PresentationError if an error occurs.
1620         """
1621         if not isfile(local_file):
1622             raise PresentationError(f"The file {local_file} does not exist.")
1623
1624         try:
1625             build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1626         except (IndexError, ValueError):
1627             pass
1628
1629         build = {
1630             u"build": build_nr,
1631             u"status": u"failed",
1632             u"file-name": local_file
1633         }
1634         if replace:
1635             self._cfg.builds = dict()
1636         self._cfg.add_build(job, build)
1637
1638         logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1639         data = self._parse_tests(job, build)
1640         if data is None:
1641             raise PresentationError(
1642                 f"Error occurred while parsing the file {local_file}"
1643             )
1644
1645         build_data = pd.Series({
1646             u"metadata": pd.Series(
1647                 list(data[u"metadata"].values()),
1648                 index=list(data[u"metadata"].keys())
1649             ),
1650             u"suites": pd.Series(
1651                 list(data[u"suites"].values()),
1652                 index=list(data[u"suites"].keys())
1653             ),
1654             u"tests": pd.Series(
1655                 list(data[u"tests"].values()),
1656                 index=list(data[u"tests"].keys())
1657             )
1658         })
1659
1660         if self._input_data.get(job, None) is None:
1661             self._input_data[job] = pd.Series()
1662         self._input_data[job][str(build_nr)] = build_data
1663
1664         self._cfg.set_input_state(job, build_nr, u"processed")
1665
1666     def process_local_directory(self, local_dir, replace=True):
1667         """Process local directory with XML file(s). The directory is processed
1668         as a 'job' and the XML files in it as builds.
1669         If the given directory contains only sub-directories, these
1670         sub-directories processed as jobs and corresponding XML files as builds
1671         of their job.
1672
1673         :param local_dir: Local directory to process.
1674         :param replace: If True, the information about jobs and builds is
1675             replaced by the new one, otherwise the new jobs and builds are
1676             added.
1677         :type local_dir: str
1678         :type replace: bool
1679         """
1680         if not isdir(local_dir):
1681             raise PresentationError(
1682                 f"The directory {local_dir} does not exist."
1683             )
1684
1685         # Check if the given directory includes only files, or only directories
1686         _, dirnames, filenames = next(walk(local_dir))
1687
1688         if filenames and not dirnames:
1689             filenames.sort()
1690             # local_builds:
1691             # key: dir (job) name, value: list of file names (builds)
1692             local_builds = {
1693                 local_dir: [join(local_dir, name) for name in filenames]
1694             }
1695
1696         elif dirnames and not filenames:
1697             dirnames.sort()
1698             # local_builds:
1699             # key: dir (job) name, value: list of file names (builds)
1700             local_builds = dict()
1701             for dirname in dirnames:
1702                 builds = [
1703                     join(local_dir, dirname, name)
1704                     for name in listdir(join(local_dir, dirname))
1705                     if isfile(join(local_dir, dirname, name))
1706                 ]
1707                 if builds:
1708                     local_builds[dirname] = sorted(builds)
1709
1710         elif not filenames and not dirnames:
1711             raise PresentationError(f"The directory {local_dir} is empty.")
1712         else:
1713             raise PresentationError(
1714                 f"The directory {local_dir} can include only files or only "
1715                 f"directories, not both.\nThe directory {local_dir} includes "
1716                 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1717             )
1718
1719         if replace:
1720             self._cfg.builds = dict()
1721
1722         for job, files in local_builds.items():
1723             for idx, local_file in enumerate(files):
1724                 self.process_local_file(local_file, job, idx + 1, replace=False)
1725
1726     @staticmethod
1727     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1728         """Return the index of character in the string which is the end of tag.
1729
1730         :param tag_filter: The string where the end of tag is being searched.
1731         :param start: The index where the searching is stated.
1732         :param closer: The character which is the tag closer.
1733         :type tag_filter: str
1734         :type start: int
1735         :type closer: str
1736         :returns: The index of the tag closer.
1737         :rtype: int
1738         """
1739         try:
1740             idx_opener = tag_filter.index(closer, start)
1741             return tag_filter.index(closer, idx_opener + 1)
1742         except ValueError:
1743             return None
1744
1745     @staticmethod
1746     def _condition(tag_filter):
1747         """Create a conditional statement from the given tag filter.
1748
1749         :param tag_filter: Filter based on tags from the element specification.
1750         :type tag_filter: str
1751         :returns: Conditional statement which can be evaluated.
1752         :rtype: str
1753         """
1754         index = 0
1755         while True:
1756             index = InputData._end_of_tag(tag_filter, index)
1757             if index is None:
1758                 return tag_filter
1759             index += 1
1760             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1761
1762     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1763                     continue_on_error=False):
1764         """Filter required data from the given jobs and builds.
1765
1766         The output data structure is:
1767         - job 1
1768           - build 1
1769             - test (or suite) 1 ID:
1770               - param 1
1771               - param 2
1772               ...
1773               - param n
1774             ...
1775             - test (or suite) n ID:
1776             ...
1777           ...
1778           - build n
1779         ...
1780         - job n
1781
1782         :param element: Element which will use the filtered data.
1783         :param params: Parameters which will be included in the output. If None,
1784             all parameters are included.
1785         :param data: If not None, this data is used instead of data specified
1786             in the element.
1787         :param data_set: The set of data to be filtered: tests, suites,
1788             metadata.
1789         :param continue_on_error: Continue if there is error while reading the
1790             data. The Item will be empty then
1791         :type element: pandas.Series
1792         :type params: list
1793         :type data: dict
1794         :type data_set: str
1795         :type continue_on_error: bool
1796         :returns: Filtered data.
1797         :rtype pandas.Series
1798         """
1799
1800         try:
1801             if data_set == "suites":
1802                 cond = u"True"
1803             elif element[u"filter"] in (u"all", u"template"):
1804                 cond = u"True"
1805             else:
1806                 cond = InputData._condition(element[u"filter"])
1807             logging.debug(f"   Filter: {cond}")
1808         except KeyError:
1809             logging.error(u"  No filter defined.")
1810             return None
1811
1812         if params is None:
1813             params = element.get(u"parameters", None)
1814             if params:
1815                 params.append(u"type")
1816
1817         data_to_filter = data if data else element[u"data"]
1818         data = pd.Series()
1819         try:
1820             for job, builds in data_to_filter.items():
1821                 data[job] = pd.Series()
1822                 for build in builds:
1823                     data[job][str(build)] = pd.Series()
1824                     try:
1825                         data_dict = dict(
1826                             self.data[job][str(build)][data_set].items())
1827                     except KeyError:
1828                         if continue_on_error:
1829                             continue
1830                         return None
1831
1832                     for test_id, test_data in data_dict.items():
1833                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1834                             data[job][str(build)][test_id] = pd.Series()
1835                             if params is None:
1836                                 for param, val in test_data.items():
1837                                     data[job][str(build)][test_id][param] = val
1838                             else:
1839                                 for param in params:
1840                                     try:
1841                                         data[job][str(build)][test_id][param] =\
1842                                             test_data[param]
1843                                     except KeyError:
1844                                         data[job][str(build)][test_id][param] =\
1845                                             u"No Data"
1846             return data
1847
1848         except (KeyError, IndexError, ValueError) as err:
1849             logging.error(
1850                 f"Missing mandatory parameter in the element specification: "
1851                 f"{repr(err)}"
1852             )
1853             return None
1854         except AttributeError as err:
1855             logging.error(repr(err))
1856             return None
1857         except SyntaxError as err:
1858             logging.error(
1859                 f"The filter {cond} is not correct. Check if all tags are "
1860                 f"enclosed by apostrophes.\n{repr(err)}"
1861             )
1862             return None
1863
1864     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1865                              continue_on_error=False):
1866         """Filter required data from the given jobs and builds.
1867
1868         The output data structure is:
1869         - job 1
1870           - build 1
1871             - test (or suite) 1 ID:
1872               - param 1
1873               - param 2
1874               ...
1875               - param n
1876             ...
1877             - test (or suite) n ID:
1878             ...
1879           ...
1880           - build n
1881         ...
1882         - job n
1883
1884         :param element: Element which will use the filtered data.
1885         :param params: Parameters which will be included in the output. If None,
1886         all parameters are included.
1887         :param data_set: The set of data to be filtered: tests, suites,
1888         metadata.
1889         :param continue_on_error: Continue if there is error while reading the
1890         data. The Item will be empty then
1891         :type element: pandas.Series
1892         :type params: list
1893         :type data_set: str
1894         :type continue_on_error: bool
1895         :returns: Filtered data.
1896         :rtype pandas.Series
1897         """
1898
1899         include = element.get(u"include", None)
1900         if not include:
1901             logging.warning(u"No tests to include, skipping the element.")
1902             return None
1903
1904         if params is None:
1905             params = element.get(u"parameters", None)
1906             if params:
1907                 params.append(u"type")
1908
1909         data = pd.Series()
1910         try:
1911             for job, builds in element[u"data"].items():
1912                 data[job] = pd.Series()
1913                 for build in builds:
1914                     data[job][str(build)] = pd.Series()
1915                     for test in include:
1916                         try:
1917                             reg_ex = re.compile(str(test).lower())
1918                             for test_id in self.data[job][
1919                                     str(build)][data_set].keys():
1920                                 if re.match(reg_ex, str(test_id).lower()):
1921                                     test_data = self.data[job][
1922                                         str(build)][data_set][test_id]
1923                                     data[job][str(build)][test_id] = pd.Series()
1924                                     if params is None:
1925                                         for param, val in test_data.items():
1926                                             data[job][str(build)][test_id]\
1927                                                 [param] = val
1928                                     else:
1929                                         for param in params:
1930                                             try:
1931                                                 data[job][str(build)][
1932                                                     test_id][param] = \
1933                                                     test_data[param]
1934                                             except KeyError:
1935                                                 data[job][str(build)][
1936                                                     test_id][param] = u"No Data"
1937                         except KeyError as err:
1938                             if continue_on_error:
1939                                 logging.debug(repr(err))
1940                                 continue
1941                             logging.error(repr(err))
1942                             return None
1943             return data
1944
1945         except (KeyError, IndexError, ValueError) as err:
1946             logging.error(
1947                 f"Missing mandatory parameter in the element "
1948                 f"specification: {repr(err)}"
1949             )
1950             return None
1951         except AttributeError as err:
1952             logging.error(repr(err))
1953             return None
1954
1955     @staticmethod
1956     def merge_data(data):
1957         """Merge data from more jobs and builds to a simple data structure.
1958
1959         The output data structure is:
1960
1961         - test (suite) 1 ID:
1962           - param 1
1963           - param 2
1964           ...
1965           - param n
1966         ...
1967         - test (suite) n ID:
1968         ...
1969
1970         :param data: Data to merge.
1971         :type data: pandas.Series
1972         :returns: Merged data.
1973         :rtype: pandas.Series
1974         """
1975
1976         logging.info(u"    Merging data ...")
1977
1978         merged_data = pd.Series()
1979         for builds in data.values:
1980             for item in builds.values:
1981                 for item_id, item_data in item.items():
1982                     merged_data[item_id] = item_data
1983         return merged_data
1984
1985     def print_all_oper_data(self):
1986         """Print all operational data to console.
1987         """
1988
1989         tbl_hdr = (
1990             u"Name",
1991             u"Nr of Vectors",
1992             u"Nr of Packets",
1993             u"Suspends",
1994             u"Cycles per Packet",
1995             u"Average Vector Size"
1996         )
1997
1998         for job in self._input_data.values:
1999             for build in job.values:
2000                 for test_id, test_data in build[u"tests"].items():
2001                     print(f"{test_id}")
2002                     if test_data.get(u"show-run", None) is None:
2003                         continue
2004                     for dut_name, data in test_data[u"show-run"].items():
2005                         if data.get(u"threads", None) is None:
2006                             continue
2007                         print(f"Host IP: {data.get(u'host', '')}, "
2008                               f"Socket: {data.get(u'socket', '')}")
2009                         for thread_nr, thread in data[u"threads"].items():
2010                             txt_table = prettytable.PrettyTable(tbl_hdr)
2011                             avg = 0.0
2012                             for row in thread:
2013                                 txt_table.add_row(row)
2014                                 avg += row[-1]
2015                             if len(thread) == 0:
2016                                 avg = u""
2017                             else:
2018                                 avg = f", Average Vector Size per Node: " \
2019                                       f"{(avg / len(thread)):.2f}"
2020                             th_name = u"main" if thread_nr == 0 \
2021                                 else f"worker_{thread_nr}"
2022                             print(f"{dut_name}, {th_name}{avg}")
2023                             txt_table.float_format = u".2"
2024                             txt_table.align = u"r"
2025                             txt_table.align[u"Name"] = u"l"
2026                             print(f"{txt_table.get_string()}\n")