resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove, walk, listdir
  29 from os.path import isfile, isdir, join
  30 from datetime import datetime as dt
  31 from datetime import timedelta
  32 from json import loads
  33 from json.decoder import JSONDecodeError
  34
  35 import hdrh.histogram
  36 import hdrh.codec
  37 import prettytable
  38 import pandas as pd
  39
  40 from robot.api import ExecutionResult, ResultVisitor
  41 from robot import errors
  42
  43 from resources.libraries.python import jumpavg
  44 from input_data_files import download_and_unzip_data_file
  45 from pal_errors import PresentationError
  46
  47
  48 # Separator used in file names
  49 SEPARATOR = u"__"
  50
  51
  52 class ExecutionChecker(ResultVisitor):
  53     """Class to traverse through the test suite structure.
  54
  55     The functionality implemented in this class generates a json structure:
  56
  57     Performance tests:
  58
  59     {
  60         "metadata": {
  61             "generated": "Timestamp",
  62             "version": "SUT version",
  63             "job": "Jenkins job name",
  64             "build": "Information about the build"
  65         },
  66         "suites": {
  67             "Suite long name 1": {
  68                 "name": Suite name,
  69                 "doc": "Suite 1 documentation",
  70                 "parent": "Suite 1 parent",
  71                 "level": "Level of the suite in the suite hierarchy"
  72             }
  73             "Suite long name N": {
  74                 "name": Suite name,
  75                 "doc": "Suite N documentation",
  76                 "parent": "Suite 2 parent",
  77                 "level": "Level of the suite in the suite hierarchy"
  78             }
  79         }
  80         "tests": {
  81             # NDRPDR tests:
  82             "ID": {
  83                 "name": "Test name",
  84                 "parent": "Name of the parent of the test",
  85                 "doc": "Test documentation",
  86                 "msg": "Test message",
  87                 "conf-history": "DUT1 and DUT2 VAT History",
  88                 "show-run": "Show Run",
  89                 "tags": ["tag 1", "tag 2", "tag n"],
  90                 "type": "NDRPDR",
  91                 "status": "PASS" | "FAIL",
  92                 "throughput": {
  93                     "NDR": {
  94                         "LOWER": float,
  95                         "UPPER": float
  96                     },
  97                     "PDR": {
  98                         "LOWER": float,
  99                         "UPPER": float
 100                     }
 101                 },
 102                 "latency": {
 103                     "NDR": {
 104                         "direction1": {
 105                             "min": float,
 106                             "avg": float,
 107                             "max": float,
 108                             "hdrh": str
 109                         },
 110                         "direction2": {
 111                             "min": float,
 112                             "avg": float,
 113                             "max": float,
 114                             "hdrh": str
 115                         }
 116                     },
 117                     "PDR": {
 118                         "direction1": {
 119                             "min": float,
 120                             "avg": float,
 121                             "max": float,
 122                             "hdrh": str
 123                         },
 124                         "direction2": {
 125                             "min": float,
 126                             "avg": float,
 127                             "max": float,
 128                             "hdrh": str
 129                         }
 130                     }
 131                 }
 132             }
 133
 134             # TCP tests:
 135             "ID": {
 136                 "name": "Test name",
 137                 "parent": "Name of the parent of the test",
 138                 "doc": "Test documentation",
 139                 "msg": "Test message",
 140                 "tags": ["tag 1", "tag 2", "tag n"],
 141                 "type": "TCP",
 142                 "status": "PASS" | "FAIL",
 143                 "result": int
 144             }
 145
 146             # MRR, BMRR tests:
 147             "ID": {
 148                 "name": "Test name",
 149                 "parent": "Name of the parent of the test",
 150                 "doc": "Test documentation",
 151                 "msg": "Test message",
 152                 "tags": ["tag 1", "tag 2", "tag n"],
 153                 "type": "MRR" | "BMRR",
 154                 "status": "PASS" | "FAIL",
 155                 "result": {
 156                     "receive-rate": float,
 157                     # Average of a list, computed using AvgStdevStats.
 158                     # In CSIT-1180, replace with List[float].
 159                 }
 160             }
 161
 162             "ID" {
 163                 # next test
 164             }
 165         }
 166     }
 167
 168
 169     Functional tests:
 170
 171     {
 172         "metadata": {  # Optional
 173             "version": "VPP version",
 174             "job": "Jenkins job name",
 175             "build": "Information about the build"
 176         },
 177         "suites": {
 178             "Suite name 1": {
 179                 "doc": "Suite 1 documentation",
 180                 "parent": "Suite 1 parent",
 181                 "level": "Level of the suite in the suite hierarchy"
 182             }
 183             "Suite name N": {
 184                 "doc": "Suite N documentation",
 185                 "parent": "Suite 2 parent",
 186                 "level": "Level of the suite in the suite hierarchy"
 187             }
 188         }
 189         "tests": {
 190             "ID": {
 191                 "name": "Test name",
 192                 "parent": "Name of the parent of the test",
 193                 "doc": "Test documentation"
 194                 "msg": "Test message"
 195                 "tags": ["tag 1", "tag 2", "tag n"],
 196                 "conf-history": "DUT1 and DUT2 VAT History"
 197                 "show-run": "Show Run"
 198                 "status": "PASS" | "FAIL"
 199             },
 200             "ID" {
 201                 # next test
 202             }
 203         }
 204     }
 205
 206     .. note:: ID is the lowercase full path to the test.
 207     """
 208
 209     REGEX_PLR_RATE = re.compile(
 210         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 211         r'PLRsearch upper bound::?\s(\d+.\d+)'
 212     )
 213     REGEX_NDRPDR_RATE = re.compile(
 214         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 215         r'NDR_UPPER:\s(\d+.\d+).*\n'
 216         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 217         r'PDR_UPPER:\s(\d+.\d+)'
 218     )
 219     REGEX_NDRPDR_GBPS = re.compile(
 220         r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
 221         r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
 222         r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
 223         r'PDR_UPPER:.*,\s(\d+.\d+)'
 224     )
 225     REGEX_PERF_MSG_INFO = re.compile(
 226         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 227         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 228         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 229         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 230         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 231     )
 232     REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
 233
 234     # TODO: Remove when not needed
 235     REGEX_NDRPDR_LAT_BASE = re.compile(
 236         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 237         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 238     )
 239     REGEX_NDRPDR_LAT = re.compile(
 240         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 241         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 242         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 243         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 244         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 245         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 246     )
 247     # TODO: Remove when not needed
 248     REGEX_NDRPDR_LAT_LONG = re.compile(
 249         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 250         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 251         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 252         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 253         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 254         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 255         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 256         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 257         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 258     )
 259     REGEX_VERSION_VPP = re.compile(
 260         r"(return STDOUT Version:\s*|"
 261         r"VPP Version:\s*|VPP version:\s*)(.*)"
 262     )
 263     REGEX_VERSION_DPDK = re.compile(
 264         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 265     )
 266     REGEX_TCP = re.compile(
 267         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 268     )
 269     REGEX_MRR = re.compile(
 270         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 271         r'tx\s(\d*),\srx\s(\d*)'
 272     )
 273     REGEX_BMRR = re.compile(
 274         r'Maximum Receive Rate trial results'
 275         r' in packets per second: \[(.*)\]'
 276     )
 277     REGEX_RECONF_LOSS = re.compile(
 278         r'Packets lost due to reconfig: (\d*)'
 279     )
 280     REGEX_RECONF_TIME = re.compile(
 281         r'Implied time lost: (\d*.[\de-]*)'
 282     )
 283     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 284
 285     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 286
 287     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 288
 289     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 290
 291     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 292
 293     def __init__(self, metadata, mapping, ignore):
 294         """Initialisation.
 295
 296         :param metadata: Key-value pairs to be included in "metadata" part of
 297             JSON structure.
 298         :param mapping: Mapping of the old names of test cases to the new
 299             (actual) one.
 300         :param ignore: List of TCs to be ignored.
 301         :type metadata: dict
 302         :type mapping: dict
 303         :type ignore: list
 304         """
 305
 306         # Type of message to parse out from the test messages
 307         self._msg_type = None
 308
 309         # VPP version
 310         self._version = None
 311
 312         # Timestamp
 313         self._timestamp = None
 314
 315         # Testbed. The testbed is identified by TG node IP address.
 316         self._testbed = None
 317
 318         # Mapping of TCs long names
 319         self._mapping = mapping
 320
 321         # Ignore list
 322         self._ignore = ignore
 323
 324         # Number of PAPI History messages found:
 325         # 0 - no message
 326         # 1 - PAPI History of DUT1
 327         # 2 - PAPI History of DUT2
 328         self._conf_history_lookup_nr = 0
 329
 330         self._sh_run_counter = 0
 331
 332         # Test ID of currently processed test- the lowercase full path to the
 333         # test
 334         self._test_id = None
 335
 336         # The main data structure
 337         self._data = {
 338             u"metadata": OrderedDict(),
 339             u"suites": OrderedDict(),
 340             u"tests": OrderedDict()
 341         }
 342
 343         # Save the provided metadata
 344         for key, val in metadata.items():
 345             self._data[u"metadata"][key] = val
 346
 347         # Dictionary defining the methods used to parse different types of
 348         # messages
 349         self.parse_msg = {
 350             u"timestamp": self._get_timestamp,
 351             u"vpp-version": self._get_vpp_version,
 352             u"dpdk-version": self._get_dpdk_version,
 353             # TODO: Remove when not needed:
 354             u"teardown-vat-history": self._get_vat_history,
 355             u"teardown-papi-history": self._get_papi_history,
 356             u"test-show-runtime": self._get_show_run,
 357             u"testbed": self._get_testbed
 358         }
 359
 360     @property
 361     def data(self):
 362         """Getter - Data parsed from the XML file.
 363
 364         :returns: Data parsed from the XML file.
 365         :rtype: dict
 366         """
 367         return self._data
 368
 369     def _get_data_from_mrr_test_msg(self, msg):
 370         """Get info from message of MRR performance tests.
 371
 372         :param msg: Message to be processed.
 373         :type msg: str
 374         :returns: Processed message or original message if a problem occurs.
 375         :rtype: str
 376         """
 377
 378         groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
 379         if not groups or groups.lastindex != 1:
 380             return u"Test Failed."
 381
 382         try:
 383             data = groups.group(1).split(u", ")
 384         except (AttributeError, IndexError, ValueError, KeyError):
 385             return u"Test Failed."
 386
 387         out_str = u"["
 388         try:
 389             for item in data:
 390                 out_str += f"{(float(item) / 1e6):.2f}, "
 391             return out_str[:-2] + u"]"
 392         except (AttributeError, IndexError, ValueError, KeyError):
 393             return u"Test Failed."
 394
 395     def _get_data_from_perf_test_msg(self, msg):
 396         """Get info from message of NDRPDR performance tests.
 397
 398         :param msg: Message to be processed.
 399         :type msg: str
 400         :returns: Processed message or original message if a problem occurs.
 401         :rtype: str
 402         """
 403
 404         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 405         if not groups or groups.lastindex != 10:
 406             return u"Test Failed."
 407
 408         try:
 409             data = {
 410                 u"ndr_low": float(groups.group(1)),
 411                 u"ndr_low_b": float(groups.group(2)),
 412                 u"pdr_low": float(groups.group(3)),
 413                 u"pdr_low_b": float(groups.group(4)),
 414                 u"pdr_lat_90_1": groups.group(5),
 415                 u"pdr_lat_90_2": groups.group(6),
 416                 u"pdr_lat_50_1": groups.group(7),
 417                 u"pdr_lat_50_2": groups.group(8),
 418                 u"pdr_lat_10_1": groups.group(9),
 419                 u"pdr_lat_10_2": groups.group(10),
 420             }
 421         except (AttributeError, IndexError, ValueError, KeyError):
 422             return u"Test Failed."
 423
 424         def _process_lat(in_str_1, in_str_2):
 425             """Extract min, avg, max values from latency string.
 426
 427             :param in_str_1: Latency string for one direction produced by robot
 428                 framework.
 429             :param in_str_2: Latency string for second direction produced by
 430                 robot framework.
 431             :type in_str_1: str
 432             :type in_str_2: str
 433             :returns: Processed latency string or None if a problem occurs.
 434             :rtype: tuple
 435             """
 436             in_list_1 = in_str_1.split('/', 3)
 437             in_list_2 = in_str_2.split('/', 3)
 438
 439             if len(in_list_1) != 4 and len(in_list_2) != 4:
 440                 return None
 441
 442             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 443             try:
 444                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 445             except hdrh.codec.HdrLengthException:
 446                 return None
 447
 448             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 449             try:
 450                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 451             except hdrh.codec.HdrLengthException:
 452                 return None
 453
 454             if hdr_lat_1 and hdr_lat_2:
 455                 hdr_lat = (
 456                     hdr_lat_1.get_value_at_percentile(50.0),
 457                     hdr_lat_1.get_value_at_percentile(90.0),
 458                     hdr_lat_1.get_value_at_percentile(99.0),
 459                     hdr_lat_2.get_value_at_percentile(50.0),
 460                     hdr_lat_2.get_value_at_percentile(90.0),
 461                     hdr_lat_2.get_value_at_percentile(99.0)
 462                 )
 463
 464                 if all(hdr_lat):
 465                     return hdr_lat
 466
 467             return None
 468
 469         try:
 470             out_msg = (
 471                 f"1. {(data[u'ndr_low'] / 1e6):5.2f}      "
 472                 f"{data[u'ndr_low_b']:5.2f}"
 473                 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f}      "
 474                 f"{data[u'pdr_low_b']:5.2f}"
 475             )
 476             latency = (
 477                 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
 478                 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
 479                 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
 480             )
 481             if all(latency):
 482                 max_len = len(str(max((max(item) for item in latency))))
 483                 max_len = 4 if max_len < 4 else max_len
 484
 485                 for idx, lat in enumerate(latency):
 486                     if not idx:
 487                         out_msg += u"\n"
 488                     out_msg += (
 489                         f"\n{idx + 3}. "
 490                         f"{lat[0]:{max_len}d} "
 491                         f"{lat[1]:{max_len}d} "
 492                         f"{lat[2]:{max_len}d}      "
 493                         f"{lat[3]:{max_len}d} "
 494                         f"{lat[4]:{max_len}d} "
 495                         f"{lat[5]:{max_len}d} "
 496                     )
 497
 498             return out_msg
 499
 500         except (AttributeError, IndexError, ValueError, KeyError):
 501             return u"Test Failed."
 502
 503     def _get_testbed(self, msg):
 504         """Called when extraction of testbed IP is required.
 505         The testbed is identified by TG node IP address.
 506
 507         :param msg: Message to process.
 508         :type msg: Message
 509         :returns: Nothing.
 510         """
 511
 512         if msg.message.count(u"Setup of TG node") or \
 513                 msg.message.count(u"Setup of node TG host"):
 514             reg_tg_ip = re.compile(
 515                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 516             try:
 517                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 518             except (KeyError, ValueError, IndexError, AttributeError):
 519                 pass
 520             finally:
 521                 self._data[u"metadata"][u"testbed"] = self._testbed
 522                 self._msg_type = None
 523
 524     def _get_vpp_version(self, msg):
 525         """Called when extraction of VPP version is required.
 526
 527         :param msg: Message to process.
 528         :type msg: Message
 529         :returns: Nothing.
 530         """
 531
 532         if msg.message.count(u"return STDOUT Version:") or \
 533             msg.message.count(u"VPP Version:") or \
 534             msg.message.count(u"VPP version:"):
 535             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 536                                 group(2))
 537             self._data[u"metadata"][u"version"] = self._version
 538             self._msg_type = None
 539
 540     def _get_dpdk_version(self, msg):
 541         """Called when extraction of DPDK version is required.
 542
 543         :param msg: Message to process.
 544         :type msg: Message
 545         :returns: Nothing.
 546         """
 547
 548         if msg.message.count(u"DPDK Version:"):
 549             try:
 550                 self._version = str(re.search(
 551                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 552                 self._data[u"metadata"][u"version"] = self._version
 553             except IndexError:
 554                 pass
 555             finally:
 556                 self._msg_type = None
 557
 558     def _get_timestamp(self, msg):
 559         """Called when extraction of timestamp is required.
 560
 561         :param msg: Message to process.
 562         :type msg: Message
 563         :returns: Nothing.
 564         """
 565
 566         self._timestamp = msg.timestamp[:14]
 567         self._data[u"metadata"][u"generated"] = self._timestamp
 568         self._msg_type = None
 569
 570     def _get_vat_history(self, msg):
 571         """Called when extraction of VAT command history is required.
 572
 573         TODO: Remove when not needed.
 574
 575         :param msg: Message to process.
 576         :type msg: Message
 577         :returns: Nothing.
 578         """
 579         if msg.message.count(u"VAT command history:"):
 580             self._conf_history_lookup_nr += 1
 581             if self._conf_history_lookup_nr == 1:
 582                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 583             else:
 584                 self._msg_type = None
 585             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 586                           r"VAT command history:", u"",
 587                           msg.message, count=1).replace(u'\n', u' |br| ').\
 588                 replace(u'"', u"'")
 589
 590             self._data[u"tests"][self._test_id][u"conf-history"] += (
 591                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 592             )
 593
 594     def _get_papi_history(self, msg):
 595         """Called when extraction of PAPI command history is required.
 596
 597         :param msg: Message to process.
 598         :type msg: Message
 599         :returns: Nothing.
 600         """
 601         if msg.message.count(u"PAPI command history:"):
 602             self._conf_history_lookup_nr += 1
 603             if self._conf_history_lookup_nr == 1:
 604                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 605             else:
 606                 self._msg_type = None
 607             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 608                           r"PAPI command history:", u"",
 609                           msg.message, count=1).replace(u'\n', u' |br| ').\
 610                 replace(u'"', u"'")
 611             self._data[u"tests"][self._test_id][u"conf-history"] += (
 612                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 613             )
 614
 615     def _get_show_run(self, msg):
 616         """Called when extraction of VPP operational data (output of CLI command
 617         Show Runtime) is required.
 618
 619         :param msg: Message to process.
 620         :type msg: Message
 621         :returns: Nothing.
 622         """
 623
 624         if not msg.message.count(u"stats runtime"):
 625             return
 626
 627         # Temporary solution
 628         if self._sh_run_counter > 1:
 629             return
 630
 631         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 632             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 633
 634         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 635         if not groups:
 636             return
 637         try:
 638             host = groups.group(1)
 639         except (AttributeError, IndexError):
 640             host = u""
 641         try:
 642             sock = groups.group(2)
 643         except (AttributeError, IndexError):
 644             sock = u""
 645
 646         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 647                         replace(u"'", u'"').replace(u'b"', u'"').
 648                         replace(u'u"', u'"').split(u":", 1)[1])
 649
 650         try:
 651             threads_nr = len(runtime[0][u"clocks"])
 652         except (IndexError, KeyError):
 653             return
 654
 655         dut = u"DUT{nr}".format(
 656             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 657
 658         oper = {
 659             u"host": host,
 660             u"socket": sock,
 661             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 662         }
 663
 664         for item in runtime:
 665             for idx in range(threads_nr):
 666                 if item[u"vectors"][idx] > 0:
 667                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 668                 elif item[u"calls"][idx] > 0:
 669                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 670                 elif item[u"suspends"][idx] > 0:
 671                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 672                 else:
 673                     clocks = 0.0
 674
 675                 if item[u"calls"][idx] > 0:
 676                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 677                 else:
 678                     vectors_call = 0.0
 679
 680                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 681                         int(item[u"suspends"][idx]):
 682                     oper[u"threads"][idx].append([
 683                         item[u"name"],
 684                         item[u"calls"][idx],
 685                         item[u"vectors"][idx],
 686                         item[u"suspends"][idx],
 687                         clocks,
 688                         vectors_call
 689                     ])
 690
 691         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 692
 693     def _get_ndrpdr_throughput(self, msg):
 694         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 695         message.
 696
 697         :param msg: The test message to be parsed.
 698         :type msg: str
 699         :returns: Parsed data as a dict and the status (PASS/FAIL).
 700         :rtype: tuple(dict, str)
 701         """
 702
 703         throughput = {
 704             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 705             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 706         }
 707         status = u"FAIL"
 708         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 709
 710         if groups is not None:
 711             try:
 712                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 713                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 714                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 715                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 716                 status = u"PASS"
 717             except (IndexError, ValueError):
 718                 pass
 719
 720         return throughput, status
 721
 722     def _get_ndrpdr_throughput_gbps(self, msg):
 723         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
 724         test message.
 725
 726         :param msg: The test message to be parsed.
 727         :type msg: str
 728         :returns: Parsed data as a dict and the status (PASS/FAIL).
 729         :rtype: tuple(dict, str)
 730         """
 731
 732         gbps = {
 733             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 734             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 735         }
 736         status = u"FAIL"
 737         groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
 738
 739         if groups is not None:
 740             try:
 741                 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
 742                 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
 743                 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
 744                 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
 745                 status = u"PASS"
 746             except (IndexError, ValueError):
 747                 pass
 748
 749         return gbps, status
 750
 751     def _get_plr_throughput(self, msg):
 752         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 753         message.
 754
 755         :param msg: The test message to be parsed.
 756         :type msg: str
 757         :returns: Parsed data as a dict and the status (PASS/FAIL).
 758         :rtype: tuple(dict, str)
 759         """
 760
 761         throughput = {
 762             u"LOWER": -1.0,
 763             u"UPPER": -1.0
 764         }
 765         status = u"FAIL"
 766         groups = re.search(self.REGEX_PLR_RATE, msg)
 767
 768         if groups is not None:
 769             try:
 770                 throughput[u"LOWER"] = float(groups.group(1))
 771                 throughput[u"UPPER"] = float(groups.group(2))
 772                 status = u"PASS"
 773             except (IndexError, ValueError):
 774                 pass
 775
 776         return throughput, status
 777
 778     def _get_ndrpdr_latency(self, msg):
 779         """Get LATENCY from the test message.
 780
 781         :param msg: The test message to be parsed.
 782         :type msg: str
 783         :returns: Parsed data as a dict and the status (PASS/FAIL).
 784         :rtype: tuple(dict, str)
 785         """
 786         latency_default = {
 787             u"min": -1.0,
 788             u"avg": -1.0,
 789             u"max": -1.0,
 790             u"hdrh": u""
 791         }
 792         latency = {
 793             u"NDR": {
 794                 u"direction1": copy.copy(latency_default),
 795                 u"direction2": copy.copy(latency_default)
 796             },
 797             u"PDR": {
 798                 u"direction1": copy.copy(latency_default),
 799                 u"direction2": copy.copy(latency_default)
 800             },
 801             u"LAT0": {
 802                 u"direction1": copy.copy(latency_default),
 803                 u"direction2": copy.copy(latency_default)
 804             },
 805             u"PDR10": {
 806                 u"direction1": copy.copy(latency_default),
 807                 u"direction2": copy.copy(latency_default)
 808             },
 809             u"PDR50": {
 810                 u"direction1": copy.copy(latency_default),
 811                 u"direction2": copy.copy(latency_default)
 812             },
 813             u"PDR90": {
 814                 u"direction1": copy.copy(latency_default),
 815                 u"direction2": copy.copy(latency_default)
 816             },
 817         }
 818
 819         # TODO: Rewrite when long and base are not needed
 820         groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
 821         if groups is None:
 822             groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 823         if groups is None:
 824             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 825         if groups is None:
 826             return latency, u"FAIL"
 827
 828         def process_latency(in_str):
 829             """Return object with parsed latency values.
 830
 831             TODO: Define class for the return type.
 832
 833             :param in_str: Input string, min/avg/max/hdrh format.
 834             :type in_str: str
 835             :returns: Dict with corresponding keys, except hdrh float values.
 836             :rtype dict:
 837             :throws IndexError: If in_str does not have enough substrings.
 838             :throws ValueError: If a substring does not convert to float.
 839             """
 840             in_list = in_str.split('/', 3)
 841
 842             rval = {
 843                 u"min": float(in_list[0]),
 844                 u"avg": float(in_list[1]),
 845                 u"max": float(in_list[2]),
 846                 u"hdrh": u""
 847             }
 848
 849             if len(in_list) == 4:
 850                 rval[u"hdrh"] = str(in_list[3])
 851
 852             return rval
 853
 854         try:
 855             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 856             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 857             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 858             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 859             if groups.lastindex == 4:
 860                 return latency, u"PASS"
 861         except (IndexError, ValueError):
 862             pass
 863
 864         try:
 865             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 866             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 867             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 868             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 869             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 870             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 871             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 872             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 873             if groups.lastindex == 12:
 874                 return latency, u"PASS"
 875         except (IndexError, ValueError):
 876             pass
 877
 878         # TODO: Remove when not needed
 879         latency[u"NDR10"] = {
 880             u"direction1": copy.copy(latency_default),
 881             u"direction2": copy.copy(latency_default)
 882         }
 883         latency[u"NDR50"] = {
 884             u"direction1": copy.copy(latency_default),
 885             u"direction2": copy.copy(latency_default)
 886         }
 887         latency[u"NDR90"] = {
 888             u"direction1": copy.copy(latency_default),
 889             u"direction2": copy.copy(latency_default)
 890         }
 891         try:
 892             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
 893             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
 894             latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
 895             latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
 896             latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
 897             latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
 898             latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
 899             latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
 900             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
 901             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
 902             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
 903             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
 904             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
 905             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
 906             return latency, u"PASS"
 907         except (IndexError, ValueError):
 908             pass
 909
 910         return latency, u"FAIL"
 911
 912     @staticmethod
 913     def _get_hoststack_data(msg, tags):
 914         """Get data from the hoststack test message.
 915
 916         :param msg: The test message to be parsed.
 917         :param tags: Test tags.
 918         :type msg: str
 919         :type tags: list
 920         :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
 921         :rtype: tuple(dict, str)
 922         """
 923         result = dict()
 924         status = u"FAIL"
 925
 926         msg = msg.replace(u"'", u'"').replace(u" ", u"")
 927         if u"LDPRELOAD" in tags:
 928             try:
 929                 result = loads(msg)
 930                 status = u"PASS"
 931             except JSONDecodeError:
 932                 pass
 933         elif u"VPPECHO" in tags:
 934             try:
 935                 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
 936                 result = dict(
 937                     client=loads(msg_lst[0]),
 938                     server=loads(msg_lst[1])
 939                 )
 940                 status = u"PASS"
 941             except (JSONDecodeError, IndexError):
 942                 pass
 943
 944         return result, status
 945
 946     def visit_suite(self, suite):
 947         """Implements traversing through the suite and its direct children.
 948
 949         :param suite: Suite to process.
 950         :type suite: Suite
 951         :returns: Nothing.
 952         """
 953         if self.start_suite(suite) is not False:
 954             suite.suites.visit(self)
 955             suite.tests.visit(self)
 956             self.end_suite(suite)
 957
 958     def start_suite(self, suite):
 959         """Called when suite starts.
 960
 961         :param suite: Suite to process.
 962         :type suite: Suite
 963         :returns: Nothing.
 964         """
 965
 966         try:
 967             parent_name = suite.parent.name
 968         except AttributeError:
 969             return
 970
 971         doc_str = suite.doc.\
 972             replace(u'"', u"'").\
 973             replace(u'\n', u' ').\
 974             replace(u'\r', u'').\
 975             replace(u'*[', u' |br| *[').\
 976             replace(u"*", u"**").\
 977             replace(u' |br| *[', u'*[', 1)
 978
 979         self._data[u"suites"][suite.longname.lower().
 980                               replace(u'"', u"'").
 981                               replace(u" ", u"_")] = {
 982                                   u"name": suite.name.lower(),
 983                                   u"doc": doc_str,
 984                                   u"parent": parent_name,
 985                                   u"level": len(suite.longname.split(u"."))
 986                               }
 987
 988         suite.keywords.visit(self)
 989
 990     def end_suite(self, suite):
 991         """Called when suite ends.
 992
 993         :param suite: Suite to process.
 994         :type suite: Suite
 995         :returns: Nothing.
 996         """
 997
 998     def visit_test(self, test):
 999         """Implements traversing through the test.
1000
1001         :param test: Test to process.
1002         :type test: Test
1003         :returns: Nothing.
1004         """
1005         if self.start_test(test) is not False:
1006             test.keywords.visit(self)
1007             self.end_test(test)
1008
1009     def start_test(self, test):
1010         """Called when test starts.
1011
1012         :param test: Test to process.
1013         :type test: Test
1014         :returns: Nothing.
1015         """
1016
1017         self._sh_run_counter = 0
1018
1019         longname_orig = test.longname.lower()
1020
1021         # Check the ignore list
1022         if longname_orig in self._ignore:
1023             return
1024
1025         tags = [str(tag) for tag in test.tags]
1026         test_result = dict()
1027
1028         # Change the TC long name and name if defined in the mapping table
1029         longname = self._mapping.get(longname_orig, None)
1030         if longname is not None:
1031             name = longname.split(u'.')[-1]
1032             logging.debug(
1033                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1034                 f"{name}"
1035             )
1036         else:
1037             longname = longname_orig
1038             name = test.name.lower()
1039
1040         # Remove TC number from the TC long name (backward compatibility):
1041         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1042         # Remove TC number from the TC name (not needed):
1043         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1044
1045         test_result[u"parent"] = test.parent.name.lower()
1046         test_result[u"tags"] = tags
1047         test_result["doc"] = test.doc.\
1048             replace(u'"', u"'").\
1049             replace(u'\n', u' ').\
1050             replace(u'\r', u'').\
1051             replace(u'[', u' |br| [').\
1052             replace(u' |br| [', u'[', 1)
1053         test_result[u"type"] = u"FUNC"
1054         test_result[u"status"] = test.status
1055
1056         if test.status == u"PASS":
1057             if u"NDRPDR" in tags:
1058                 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1059                     test.message).replace(u'\n', u' |br| ').\
1060                     replace(u'\r', u'').replace(u'"', u"'")
1061             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1062                 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1063                     test.message).replace(u'\n', u' |br| ').\
1064                     replace(u'\r', u'').replace(u'"', u"'")
1065             else:
1066                 test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\
1067                     replace(u'\r', u'').replace(u'"', u"'")
1068         else:
1069             test_result[u"msg"] = u"Test Failed."
1070
1071         if u"PERFTEST" in tags:
1072             # Replace info about cores (e.g. -1c-) with the info about threads
1073             # and cores (e.g. -1t1c-) in the long test case names and in the
1074             # test case names if necessary.
1075             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1076             if not groups:
1077                 tag_count = 0
1078                 tag_tc = str()
1079                 for tag in test_result[u"tags"]:
1080                     groups = re.search(self.REGEX_TC_TAG, tag)
1081                     if groups:
1082                         tag_count += 1
1083                         tag_tc = tag
1084
1085                 if tag_count == 1:
1086                     self._test_id = re.sub(
1087                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1088                         self._test_id, count=1
1089                     )
1090                     test_result[u"name"] = re.sub(
1091                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1092                         test_result["name"], count=1
1093                     )
1094                 else:
1095                     test_result[u"status"] = u"FAIL"
1096                     self._data[u"tests"][self._test_id] = test_result
1097                     logging.debug(
1098                         f"The test {self._test_id} has no or more than one "
1099                         f"multi-threading tags.\n"
1100                         f"Tags: {test_result[u'tags']}"
1101                     )
1102                     return
1103
1104         if test.status == u"PASS":
1105             if u"NDRPDR" in tags:
1106                 test_result[u"type"] = u"NDRPDR"
1107                 test_result[u"throughput"], test_result[u"status"] = \
1108                     self._get_ndrpdr_throughput(test.message)
1109                 test_result[u"gbps"], test_result[u"status"] = \
1110                     self._get_ndrpdr_throughput_gbps(test.message)
1111                 test_result[u"latency"], test_result[u"status"] = \
1112                     self._get_ndrpdr_latency(test.message)
1113             elif u"SOAK" in tags:
1114                 test_result[u"type"] = u"SOAK"
1115                 test_result[u"throughput"], test_result[u"status"] = \
1116                     self._get_plr_throughput(test.message)
1117             elif u"HOSTSTACK" in tags:
1118                 test_result[u"type"] = u"HOSTSTACK"
1119                 test_result[u"result"], test_result[u"status"] = \
1120                     self._get_hoststack_data(test.message, tags)
1121             elif u"TCP" in tags:
1122                 test_result[u"type"] = u"TCP"
1123                 groups = re.search(self.REGEX_TCP, test.message)
1124                 test_result[u"result"] = int(groups.group(2))
1125             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1126                 if u"MRR" in tags:
1127                     test_result[u"type"] = u"MRR"
1128                 else:
1129                     test_result[u"type"] = u"BMRR"
1130
1131                 test_result[u"result"] = dict()
1132                 groups = re.search(self.REGEX_BMRR, test.message)
1133                 if groups is not None:
1134                     items_str = groups.group(1)
1135                     items_float = [
1136                         float(item.strip()) for item in items_str.split(",")
1137                     ]
1138                     # Use whole list in CSIT-1180.
1139                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1140                     test_result[u"result"][u"receive-rate"] = stats.avg
1141                     test_result[u"result"][u"receive-stdev"] = stats.stdev
1142                 else:
1143                     groups = re.search(self.REGEX_MRR, test.message)
1144                     test_result[u"result"][u"receive-rate"] = \
1145                         float(groups.group(3)) / float(groups.group(1))
1146             elif u"RECONF" in tags:
1147                 test_result[u"type"] = u"RECONF"
1148                 test_result[u"result"] = None
1149                 try:
1150                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1151                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1152                     test_result[u"result"] = {
1153                         u"loss": int(grps_loss.group(1)),
1154                         u"time": float(grps_time.group(1))
1155                     }
1156                 except (AttributeError, IndexError, ValueError, TypeError):
1157                     test_result[u"status"] = u"FAIL"
1158             elif u"DEVICETEST" in tags:
1159                 test_result[u"type"] = u"DEVICETEST"
1160             else:
1161                 test_result[u"status"] = u"FAIL"
1162                 self._data[u"tests"][self._test_id] = test_result
1163                 return
1164
1165         self._data[u"tests"][self._test_id] = test_result
1166
1167     def end_test(self, test):
1168         """Called when test ends.
1169
1170         :param test: Test to process.
1171         :type test: Test
1172         :returns: Nothing.
1173         """
1174
1175     def visit_keyword(self, keyword):
1176         """Implements traversing through the keyword and its child keywords.
1177
1178         :param keyword: Keyword to process.
1179         :type keyword: Keyword
1180         :returns: Nothing.
1181         """
1182         if self.start_keyword(keyword) is not False:
1183             self.end_keyword(keyword)
1184
1185     def start_keyword(self, keyword):
1186         """Called when keyword starts. Default implementation does nothing.
1187
1188         :param keyword: Keyword to process.
1189         :type keyword: Keyword
1190         :returns: Nothing.
1191         """
1192         try:
1193             if keyword.type == u"setup":
1194                 self.visit_setup_kw(keyword)
1195             elif keyword.type == u"teardown":
1196                 self.visit_teardown_kw(keyword)
1197             else:
1198                 self.visit_test_kw(keyword)
1199         except AttributeError:
1200             pass
1201
1202     def end_keyword(self, keyword):
1203         """Called when keyword ends. Default implementation does nothing.
1204
1205         :param keyword: Keyword to process.
1206         :type keyword: Keyword
1207         :returns: Nothing.
1208         """
1209
1210     def visit_test_kw(self, test_kw):
1211         """Implements traversing through the test keyword and its child
1212         keywords.
1213
1214         :param test_kw: Keyword to process.
1215         :type test_kw: Keyword
1216         :returns: Nothing.
1217         """
1218         for keyword in test_kw.keywords:
1219             if self.start_test_kw(keyword) is not False:
1220                 self.visit_test_kw(keyword)
1221                 self.end_test_kw(keyword)
1222
1223     def start_test_kw(self, test_kw):
1224         """Called when test keyword starts. Default implementation does
1225         nothing.
1226
1227         :param test_kw: Keyword to process.
1228         :type test_kw: Keyword
1229         :returns: Nothing.
1230         """
1231         if test_kw.name.count(u"Show Runtime On All Duts") or \
1232                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
1233             self._msg_type = u"test-show-runtime"
1234             self._sh_run_counter += 1
1235         elif test_kw.name.count(u"Install Dpdk Test On All Duts") and \
1236                 not self._version:
1237             self._msg_type = u"dpdk-version"
1238         else:
1239             return
1240         test_kw.messages.visit(self)
1241
1242     def end_test_kw(self, test_kw):
1243         """Called when keyword ends. Default implementation does nothing.
1244
1245         :param test_kw: Keyword to process.
1246         :type test_kw: Keyword
1247         :returns: Nothing.
1248         """
1249
1250     def visit_setup_kw(self, setup_kw):
1251         """Implements traversing through the teardown keyword and its child
1252         keywords.
1253
1254         :param setup_kw: Keyword to process.
1255         :type setup_kw: Keyword
1256         :returns: Nothing.
1257         """
1258         for keyword in setup_kw.keywords:
1259             if self.start_setup_kw(keyword) is not False:
1260                 self.visit_setup_kw(keyword)
1261                 self.end_setup_kw(keyword)
1262
1263     def start_setup_kw(self, setup_kw):
1264         """Called when teardown keyword starts. Default implementation does
1265         nothing.
1266
1267         :param setup_kw: Keyword to process.
1268         :type setup_kw: Keyword
1269         :returns: Nothing.
1270         """
1271         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1272                 and not self._version:
1273             self._msg_type = u"vpp-version"
1274         elif setup_kw.name.count(u"Set Global Variable") \
1275                 and not self._timestamp:
1276             self._msg_type = u"timestamp"
1277         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1278             self._msg_type = u"testbed"
1279         else:
1280             return
1281         setup_kw.messages.visit(self)
1282
1283     def end_setup_kw(self, setup_kw):
1284         """Called when keyword ends. Default implementation does nothing.
1285
1286         :param setup_kw: Keyword to process.
1287         :type setup_kw: Keyword
1288         :returns: Nothing.
1289         """
1290
1291     def visit_teardown_kw(self, teardown_kw):
1292         """Implements traversing through the teardown keyword and its child
1293         keywords.
1294
1295         :param teardown_kw: Keyword to process.
1296         :type teardown_kw: Keyword
1297         :returns: Nothing.
1298         """
1299         for keyword in teardown_kw.keywords:
1300             if self.start_teardown_kw(keyword) is not False:
1301                 self.visit_teardown_kw(keyword)
1302                 self.end_teardown_kw(keyword)
1303
1304     def start_teardown_kw(self, teardown_kw):
1305         """Called when teardown keyword starts
1306
1307         :param teardown_kw: Keyword to process.
1308         :type teardown_kw: Keyword
1309         :returns: Nothing.
1310         """
1311
1312         if teardown_kw.name.count(u"Show Vat History On All Duts"):
1313             # TODO: Remove when not needed:
1314             self._conf_history_lookup_nr = 0
1315             self._msg_type = u"teardown-vat-history"
1316             teardown_kw.messages.visit(self)
1317         elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1318             self._conf_history_lookup_nr = 0
1319             self._msg_type = u"teardown-papi-history"
1320             teardown_kw.messages.visit(self)
1321
1322     def end_teardown_kw(self, teardown_kw):
1323         """Called when keyword ends. Default implementation does nothing.
1324
1325         :param teardown_kw: Keyword to process.
1326         :type teardown_kw: Keyword
1327         :returns: Nothing.
1328         """
1329
1330     def visit_message(self, msg):
1331         """Implements visiting the message.
1332
1333         :param msg: Message to process.
1334         :type msg: Message
1335         :returns: Nothing.
1336         """
1337         if self.start_message(msg) is not False:
1338             self.end_message(msg)
1339
1340     def start_message(self, msg):
1341         """Called when message starts. Get required information from messages:
1342         - VPP version.
1343
1344         :param msg: Message to process.
1345         :type msg: Message
1346         :returns: Nothing.
1347         """
1348         if self._msg_type:
1349             self.parse_msg[self._msg_type](msg)
1350
1351     def end_message(self, msg):
1352         """Called when message ends. Default implementation does nothing.
1353
1354         :param msg: Message to process.
1355         :type msg: Message
1356         :returns: Nothing.
1357         """
1358
1359
1360 class InputData:
1361     """Input data
1362
1363     The data is extracted from output.xml files generated by Jenkins jobs and
1364     stored in pandas' DataFrames.
1365
1366     The data structure:
1367     - job name
1368       - build number
1369         - metadata
1370           (as described in ExecutionChecker documentation)
1371         - suites
1372           (as described in ExecutionChecker documentation)
1373         - tests
1374           (as described in ExecutionChecker documentation)
1375     """
1376
1377     def __init__(self, spec):
1378         """Initialization.
1379
1380         :param spec: Specification.
1381         :type spec: Specification
1382         """
1383
1384         # Specification:
1385         self._cfg = spec
1386
1387         # Data store:
1388         self._input_data = pd.Series()
1389
1390     @property
1391     def data(self):
1392         """Getter - Input data.
1393
1394         :returns: Input data
1395         :rtype: pandas.Series
1396         """
1397         return self._input_data
1398
1399     def metadata(self, job, build):
1400         """Getter - metadata
1401
1402         :param job: Job which metadata we want.
1403         :param build: Build which metadata we want.
1404         :type job: str
1405         :type build: str
1406         :returns: Metadata
1407         :rtype: pandas.Series
1408         """
1409         return self.data[job][build][u"metadata"]
1410
1411     def suites(self, job, build):
1412         """Getter - suites
1413
1414         :param job: Job which suites we want.
1415         :param build: Build which suites we want.
1416         :type job: str
1417         :type build: str
1418         :returns: Suites.
1419         :rtype: pandas.Series
1420         """
1421         return self.data[job][str(build)][u"suites"]
1422
1423     def tests(self, job, build):
1424         """Getter - tests
1425
1426         :param job: Job which tests we want.
1427         :param build: Build which tests we want.
1428         :type job: str
1429         :type build: str
1430         :returns: Tests.
1431         :rtype: pandas.Series
1432         """
1433         return self.data[job][build][u"tests"]
1434
1435     def _parse_tests(self, job, build):
1436         """Process data from robot output.xml file and return JSON structured
1437         data.
1438
1439         :param job: The name of job which build output data will be processed.
1440         :param build: The build which output data will be processed.
1441         :type job: str
1442         :type build: dict
1443         :returns: JSON data structure.
1444         :rtype: dict
1445         """
1446
1447         metadata = {
1448             u"job": job,
1449             u"build": build
1450         }
1451
1452         with open(build[u"file-name"], u'r') as data_file:
1453             try:
1454                 result = ExecutionResult(data_file)
1455             except errors.DataError as err:
1456                 logging.error(
1457                     f"Error occurred while parsing output.xml: {repr(err)}"
1458                 )
1459                 return None
1460         checker = ExecutionChecker(metadata, self._cfg.mapping,
1461                                    self._cfg.ignore)
1462         result.visit(checker)
1463
1464         return checker.data
1465
1466     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1467         """Download and parse the input data file.
1468
1469         :param pid: PID of the process executing this method.
1470         :param job: Name of the Jenkins job which generated the processed input
1471             file.
1472         :param build: Information about the Jenkins build which generated the
1473             processed input file.
1474         :param repeat: Repeat the download specified number of times if not
1475             successful.
1476         :type pid: int
1477         :type job: str
1478         :type build: dict
1479         :type repeat: int
1480         """
1481
1482         logging.info(f"  Processing the job/build: {job}: {build[u'build']}")
1483
1484         state = u"failed"
1485         success = False
1486         data = None
1487         do_repeat = repeat
1488         while do_repeat:
1489             success = download_and_unzip_data_file(self._cfg, job, build, pid)
1490             if success:
1491                 break
1492             do_repeat -= 1
1493         if not success:
1494             logging.error(
1495                 f"It is not possible to download the input data file from the "
1496                 f"job {job}, build {build[u'build']}, or it is damaged. "
1497                 f"Skipped."
1498             )
1499         if success:
1500             logging.info(f"    Processing data from build {build[u'build']}")
1501             data = self._parse_tests(job, build)
1502             if data is None:
1503                 logging.error(
1504                     f"Input data file from the job {job}, build "
1505                     f"{build[u'build']} is damaged. Skipped."
1506                 )
1507             else:
1508                 state = u"processed"
1509
1510             try:
1511                 remove(build[u"file-name"])
1512             except OSError as err:
1513                 logging.error(
1514                     f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1515                 )
1516
1517         # If the time-period is defined in the specification file, remove all
1518         # files which are outside the time period.
1519         is_last = False
1520         timeperiod = self._cfg.input.get(u"time-period", None)
1521         if timeperiod and data:
1522             now = dt.utcnow()
1523             timeperiod = timedelta(int(timeperiod))
1524             metadata = data.get(u"metadata", None)
1525             if metadata:
1526                 generated = metadata.get(u"generated", None)
1527                 if generated:
1528                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1529                     if (now - generated) > timeperiod:
1530                         # Remove the data and the file:
1531                         state = u"removed"
1532                         data = None
1533                         is_last = True
1534                         logging.info(
1535                             f"    The build {job}/{build[u'build']} is "
1536                             f"outdated, will be removed."
1537                         )
1538         logging.info(u"  Done.")
1539
1540         return {
1541             u"data": data,
1542             u"state": state,
1543             u"job": job,
1544             u"build": build,
1545             u"last": is_last
1546         }
1547
1548     def download_and_parse_data(self, repeat=1):
1549         """Download the input data files, parse input data from input files and
1550         store in pandas' Series.
1551
1552         :param repeat: Repeat the download specified number of times if not
1553             successful.
1554         :type repeat: int
1555         """
1556
1557         logging.info(u"Downloading and parsing input files ...")
1558
1559         for job, builds in self._cfg.builds.items():
1560             for build in builds:
1561
1562                 result = self._download_and_parse_build(job, build, repeat)
1563                 if result[u"last"]:
1564                     break
1565                 build_nr = result[u"build"][u"build"]
1566
1567                 if result[u"data"]:
1568                     data = result[u"data"]
1569                     build_data = pd.Series({
1570                         u"metadata": pd.Series(
1571                             list(data[u"metadata"].values()),
1572                             index=list(data[u"metadata"].keys())
1573                         ),
1574                         u"suites": pd.Series(
1575                             list(data[u"suites"].values()),
1576                             index=list(data[u"suites"].keys())
1577                         ),
1578                         u"tests": pd.Series(
1579                             list(data[u"tests"].values()),
1580                             index=list(data[u"tests"].keys())
1581                         )
1582                     })
1583
1584                     if self._input_data.get(job, None) is None:
1585                         self._input_data[job] = pd.Series()
1586                     self._input_data[job][str(build_nr)] = build_data
1587
1588                     self._cfg.set_input_file_name(
1589                         job, build_nr, result[u"build"][u"file-name"])
1590
1591                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1592
1593                 mem_alloc = \
1594                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1595                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1596
1597         logging.info(u"Done.")
1598
1599     def process_local_file(self, local_file, job=u"local", build_nr=1,
1600                            replace=True):
1601         """Process local XML file given as a command-line parameter.
1602
1603         :param local_file: The file to process.
1604         :param job: Job name.
1605         :param build_nr: Build number.
1606         :param replace: If True, the information about jobs and builds is
1607             replaced by the new one, otherwise the new jobs and builds are
1608             added.
1609         :type local_file: str
1610         :type job: str
1611         :type build_nr: int
1612         :type replace: bool
1613         :raises: PresentationError if an error occurs.
1614         """
1615         if not isfile(local_file):
1616             raise PresentationError(f"The file {local_file} does not exist.")
1617
1618         try:
1619             build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1620         except (IndexError, ValueError):
1621             pass
1622
1623         build = {
1624             u"build": build_nr,
1625             u"status": u"failed",
1626             u"file-name": local_file
1627         }
1628         if replace:
1629             self._cfg.builds = dict()
1630         self._cfg.add_build(job, build)
1631
1632         logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1633         data = self._parse_tests(job, build)
1634         if data is None:
1635             raise PresentationError(
1636                 f"Error occurred while parsing the file {local_file}"
1637             )
1638
1639         build_data = pd.Series({
1640             u"metadata": pd.Series(
1641                 list(data[u"metadata"].values()),
1642                 index=list(data[u"metadata"].keys())
1643             ),
1644             u"suites": pd.Series(
1645                 list(data[u"suites"].values()),
1646                 index=list(data[u"suites"].keys())
1647             ),
1648             u"tests": pd.Series(
1649                 list(data[u"tests"].values()),
1650                 index=list(data[u"tests"].keys())
1651             )
1652         })
1653
1654         if self._input_data.get(job, None) is None:
1655             self._input_data[job] = pd.Series()
1656         self._input_data[job][str(build_nr)] = build_data
1657
1658         self._cfg.set_input_state(job, build_nr, u"processed")
1659
1660     def process_local_directory(self, local_dir, replace=True):
1661         """Process local directory with XML file(s). The directory is processed
1662         as a 'job' and the XML files in it as builds.
1663         If the given directory contains only sub-directories, these
1664         sub-directories processed as jobs and corresponding XML files as builds
1665         of their job.
1666
1667         :param local_dir: Local directory to process.
1668         :param replace: If True, the information about jobs and builds is
1669             replaced by the new one, otherwise the new jobs and builds are
1670             added.
1671         :type local_dir: str
1672         :type replace: bool
1673         """
1674         if not isdir(local_dir):
1675             raise PresentationError(
1676                 f"The directory {local_dir} does not exist."
1677             )
1678
1679         # Check if the given directory includes only files, or only directories
1680         _, dirnames, filenames = next(walk(local_dir))
1681
1682         if filenames and not dirnames:
1683             filenames.sort()
1684             # local_builds:
1685             # key: dir (job) name, value: list of file names (builds)
1686             local_builds = {
1687                 local_dir: [join(local_dir, name) for name in filenames]
1688             }
1689
1690         elif dirnames and not filenames:
1691             dirnames.sort()
1692             # local_builds:
1693             # key: dir (job) name, value: list of file names (builds)
1694             local_builds = dict()
1695             for dirname in dirnames:
1696                 builds = [
1697                     join(local_dir, dirname, name)
1698                     for name in listdir(join(local_dir, dirname))
1699                     if isfile(join(local_dir, dirname, name))
1700                 ]
1701                 if builds:
1702                     local_builds[dirname] = sorted(builds)
1703
1704         elif not filenames and not dirnames:
1705             raise PresentationError(f"The directory {local_dir} is empty.")
1706         else:
1707             raise PresentationError(
1708                 f"The directory {local_dir} can include only files or only "
1709                 f"directories, not both.\nThe directory {local_dir} includes "
1710                 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1711             )
1712
1713         if replace:
1714             self._cfg.builds = dict()
1715
1716         for job, files in local_builds.items():
1717             for idx, local_file in enumerate(files):
1718                 self.process_local_file(local_file, job, idx + 1, replace=False)
1719
1720     @staticmethod
1721     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1722         """Return the index of character in the string which is the end of tag.
1723
1724         :param tag_filter: The string where the end of tag is being searched.
1725         :param start: The index where the searching is stated.
1726         :param closer: The character which is the tag closer.
1727         :type tag_filter: str
1728         :type start: int
1729         :type closer: str
1730         :returns: The index of the tag closer.
1731         :rtype: int
1732         """
1733         try:
1734             idx_opener = tag_filter.index(closer, start)
1735             return tag_filter.index(closer, idx_opener + 1)
1736         except ValueError:
1737             return None
1738
1739     @staticmethod
1740     def _condition(tag_filter):
1741         """Create a conditional statement from the given tag filter.
1742
1743         :param tag_filter: Filter based on tags from the element specification.
1744         :type tag_filter: str
1745         :returns: Conditional statement which can be evaluated.
1746         :rtype: str
1747         """
1748         index = 0
1749         while True:
1750             index = InputData._end_of_tag(tag_filter, index)
1751             if index is None:
1752                 return tag_filter
1753             index += 1
1754             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1755
1756     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1757                     continue_on_error=False):
1758         """Filter required data from the given jobs and builds.
1759
1760         The output data structure is:
1761         - job 1
1762           - build 1
1763             - test (or suite) 1 ID:
1764               - param 1
1765               - param 2
1766               ...
1767               - param n
1768             ...
1769             - test (or suite) n ID:
1770             ...
1771           ...
1772           - build n
1773         ...
1774         - job n
1775
1776         :param element: Element which will use the filtered data.
1777         :param params: Parameters which will be included in the output. If None,
1778             all parameters are included.
1779         :param data: If not None, this data is used instead of data specified
1780             in the element.
1781         :param data_set: The set of data to be filtered: tests, suites,
1782             metadata.
1783         :param continue_on_error: Continue if there is error while reading the
1784             data. The Item will be empty then
1785         :type element: pandas.Series
1786         :type params: list
1787         :type data: dict
1788         :type data_set: str
1789         :type continue_on_error: bool
1790         :returns: Filtered data.
1791         :rtype pandas.Series
1792         """
1793
1794         try:
1795             if data_set == "suites":
1796                 cond = u"True"
1797             elif element[u"filter"] in (u"all", u"template"):
1798                 cond = u"True"
1799             else:
1800                 cond = InputData._condition(element[u"filter"])
1801             logging.debug(f"   Filter: {cond}")
1802         except KeyError:
1803             logging.error(u"  No filter defined.")
1804             return None
1805
1806         if params is None:
1807             params = element.get(u"parameters", None)
1808             if params:
1809                 params.append(u"type")
1810
1811         data_to_filter = data if data else element[u"data"]
1812         data = pd.Series()
1813         try:
1814             for job, builds in data_to_filter.items():
1815                 data[job] = pd.Series()
1816                 for build in builds:
1817                     data[job][str(build)] = pd.Series()
1818                     try:
1819                         data_dict = dict(
1820                             self.data[job][str(build)][data_set].items())
1821                     except KeyError:
1822                         if continue_on_error:
1823                             continue
1824                         return None
1825
1826                     for test_id, test_data in data_dict.items():
1827                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1828                             data[job][str(build)][test_id] = pd.Series()
1829                             if params is None:
1830                                 for param, val in test_data.items():
1831                                     data[job][str(build)][test_id][param] = val
1832                             else:
1833                                 for param in params:
1834                                     try:
1835                                         data[job][str(build)][test_id][param] =\
1836                                             test_data[param]
1837                                     except KeyError:
1838                                         data[job][str(build)][test_id][param] =\
1839                                             u"No Data"
1840             return data
1841
1842         except (KeyError, IndexError, ValueError) as err:
1843             logging.error(
1844                 f"Missing mandatory parameter in the element specification: "
1845                 f"{repr(err)}"
1846             )
1847             return None
1848         except AttributeError as err:
1849             logging.error(repr(err))
1850             return None
1851         except SyntaxError as err:
1852             logging.error(
1853                 f"The filter {cond} is not correct. Check if all tags are "
1854                 f"enclosed by apostrophes.\n{repr(err)}"
1855             )
1856             return None
1857
1858     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1859                              continue_on_error=False):
1860         """Filter required data from the given jobs and builds.
1861
1862         The output data structure is:
1863         - job 1
1864           - build 1
1865             - test (or suite) 1 ID:
1866               - param 1
1867               - param 2
1868               ...
1869               - param n
1870             ...
1871             - test (or suite) n ID:
1872             ...
1873           ...
1874           - build n
1875         ...
1876         - job n
1877
1878         :param element: Element which will use the filtered data.
1879         :param params: Parameters which will be included in the output. If None,
1880         all parameters are included.
1881         :param data_set: The set of data to be filtered: tests, suites,
1882         metadata.
1883         :param continue_on_error: Continue if there is error while reading the
1884         data. The Item will be empty then
1885         :type element: pandas.Series
1886         :type params: list
1887         :type data_set: str
1888         :type continue_on_error: bool
1889         :returns: Filtered data.
1890         :rtype pandas.Series
1891         """
1892
1893         include = element.get(u"include", None)
1894         if not include:
1895             logging.warning(u"No tests to include, skipping the element.")
1896             return None
1897
1898         if params is None:
1899             params = element.get(u"parameters", None)
1900             if params:
1901                 params.append(u"type")
1902
1903         data = pd.Series()
1904         try:
1905             for job, builds in element[u"data"].items():
1906                 data[job] = pd.Series()
1907                 for build in builds:
1908                     data[job][str(build)] = pd.Series()
1909                     for test in include:
1910                         try:
1911                             reg_ex = re.compile(str(test).lower())
1912                             for test_id in self.data[job][
1913                                     str(build)][data_set].keys():
1914                                 if re.match(reg_ex, str(test_id).lower()):
1915                                     test_data = self.data[job][
1916                                         str(build)][data_set][test_id]
1917                                     data[job][str(build)][test_id] = pd.Series()
1918                                     if params is None:
1919                                         for param, val in test_data.items():
1920                                             data[job][str(build)][test_id]\
1921                                                 [param] = val
1922                                     else:
1923                                         for param in params:
1924                                             try:
1925                                                 data[job][str(build)][
1926                                                     test_id][param] = \
1927                                                     test_data[param]
1928                                             except KeyError:
1929                                                 data[job][str(build)][
1930                                                     test_id][param] = u"No Data"
1931                         except KeyError as err:
1932                             if continue_on_error:
1933                                 logging.debug(repr(err))
1934                                 continue
1935                             logging.error(repr(err))
1936                             return None
1937             return data
1938
1939         except (KeyError, IndexError, ValueError) as err:
1940             logging.error(
1941                 f"Missing mandatory parameter in the element "
1942                 f"specification: {repr(err)}"
1943             )
1944             return None
1945         except AttributeError as err:
1946             logging.error(repr(err))
1947             return None
1948
1949     @staticmethod
1950     def merge_data(data):
1951         """Merge data from more jobs and builds to a simple data structure.
1952
1953         The output data structure is:
1954
1955         - test (suite) 1 ID:
1956           - param 1
1957           - param 2
1958           ...
1959           - param n
1960         ...
1961         - test (suite) n ID:
1962         ...
1963
1964         :param data: Data to merge.
1965         :type data: pandas.Series
1966         :returns: Merged data.
1967         :rtype: pandas.Series
1968         """
1969
1970         logging.info(u"    Merging data ...")
1971
1972         merged_data = pd.Series()
1973         for builds in data.values:
1974             for item in builds.values:
1975                 for item_id, item_data in item.items():
1976                     merged_data[item_id] = item_data
1977         return merged_data
1978
1979     def print_all_oper_data(self):
1980         """Print all operational data to console.
1981         """
1982
1983         tbl_hdr = (
1984             u"Name",
1985             u"Nr of Vectors",
1986             u"Nr of Packets",
1987             u"Suspends",
1988             u"Cycles per Packet",
1989             u"Average Vector Size"
1990         )
1991
1992         for job in self._input_data.values:
1993             for build in job.values:
1994                 for test_id, test_data in build[u"tests"].items():
1995                     print(f"{test_id}")
1996                     if test_data.get(u"show-run", None) is None:
1997                         continue
1998                     for dut_name, data in test_data[u"show-run"].items():
1999                         if data.get(u"threads", None) is None:
2000                             continue
2001                         print(f"Host IP: {data.get(u'host', '')}, "
2002                               f"Socket: {data.get(u'socket', '')}")
2003                         for thread_nr, thread in data[u"threads"].items():
2004                             txt_table = prettytable.PrettyTable(tbl_hdr)
2005                             avg = 0.0
2006                             for row in thread:
2007                                 txt_table.add_row(row)
2008                                 avg += row[-1]
2009                             if len(thread) == 0:
2010                                 avg = u""
2011                             else:
2012                                 avg = f", Average Vector Size per Node: " \
2013                                       f"{(avg / len(thread)):.2f}"
2014                             th_name = u"main" if thread_nr == 0 \
2015                                 else f"worker_{thread_nr}"
2016                             print(f"{dut_name}, {th_name}{avg}")
2017                             txt_table.float_format = u".2"
2018                             txt_table.align = u"r"
2019                             txt_table.align[u"Name"] = u"l"
2020                             print(f"{txt_table.get_string()}\n")