resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove, walk, listdir
  29 from os.path import isfile, isdir, join
  30 from datetime import datetime as dt
  31 from datetime import timedelta
  32 from json import loads
  33 from json.decoder import JSONDecodeError
  34
  35 import hdrh.histogram
  36 import hdrh.codec
  37 import prettytable
  38 import pandas as pd
  39
  40 from robot.api import ExecutionResult, ResultVisitor
  41 from robot import errors
  42
  43 from resources.libraries.python import jumpavg
  44 from input_data_files import download_and_unzip_data_file
  45 from pal_errors import PresentationError
  46
  47
  48 # Separator used in file names
  49 SEPARATOR = u"__"
  50
  51
  52 class ExecutionChecker(ResultVisitor):
  53     """Class to traverse through the test suite structure.
  54
  55     The functionality implemented in this class generates a json structure:
  56
  57     Performance tests:
  58
  59     {
  60         "metadata": {
  61             "generated": "Timestamp",
  62             "version": "SUT version",
  63             "job": "Jenkins job name",
  64             "build": "Information about the build"
  65         },
  66         "suites": {
  67             "Suite long name 1": {
  68                 "name": Suite name,
  69                 "doc": "Suite 1 documentation",
  70                 "parent": "Suite 1 parent",
  71                 "level": "Level of the suite in the suite hierarchy"
  72             }
  73             "Suite long name N": {
  74                 "name": Suite name,
  75                 "doc": "Suite N documentation",
  76                 "parent": "Suite 2 parent",
  77                 "level": "Level of the suite in the suite hierarchy"
  78             }
  79         }
  80         "tests": {
  81             # NDRPDR tests:
  82             "ID": {
  83                 "name": "Test name",
  84                 "parent": "Name of the parent of the test",
  85                 "doc": "Test documentation",
  86                 "msg": "Test message",
  87                 "conf-history": "DUT1 and DUT2 VAT History",
  88                 "show-run": "Show Run",
  89                 "tags": ["tag 1", "tag 2", "tag n"],
  90                 "type": "NDRPDR",
  91                 "status": "PASS" | "FAIL",
  92                 "throughput": {
  93                     "NDR": {
  94                         "LOWER": float,
  95                         "UPPER": float
  96                     },
  97                     "PDR": {
  98                         "LOWER": float,
  99                         "UPPER": float
 100                     }
 101                 },
 102                 "latency": {
 103                     "NDR": {
 104                         "direction1": {
 105                             "min": float,
 106                             "avg": float,
 107                             "max": float,
 108                             "hdrh": str
 109                         },
 110                         "direction2": {
 111                             "min": float,
 112                             "avg": float,
 113                             "max": float,
 114                             "hdrh": str
 115                         }
 116                     },
 117                     "PDR": {
 118                         "direction1": {
 119                             "min": float,
 120                             "avg": float,
 121                             "max": float,
 122                             "hdrh": str
 123                         },
 124                         "direction2": {
 125                             "min": float,
 126                             "avg": float,
 127                             "max": float,
 128                             "hdrh": str
 129                         }
 130                     }
 131                 }
 132             }
 133
 134             # TCP tests:
 135             "ID": {
 136                 "name": "Test name",
 137                 "parent": "Name of the parent of the test",
 138                 "doc": "Test documentation",
 139                 "msg": "Test message",
 140                 "tags": ["tag 1", "tag 2", "tag n"],
 141                 "type": "TCP",
 142                 "status": "PASS" | "FAIL",
 143                 "result": int
 144             }
 145
 146             # MRR, BMRR tests:
 147             "ID": {
 148                 "name": "Test name",
 149                 "parent": "Name of the parent of the test",
 150                 "doc": "Test documentation",
 151                 "msg": "Test message",
 152                 "tags": ["tag 1", "tag 2", "tag n"],
 153                 "type": "MRR" | "BMRR",
 154                 "status": "PASS" | "FAIL",
 155                 "result": {
 156                     "receive-rate": float,
 157                     # Average of a list, computed using AvgStdevStats.
 158                     # In CSIT-1180, replace with List[float].
 159                 }
 160             }
 161
 162             "ID" {
 163                 # next test
 164             }
 165         }
 166     }
 167
 168
 169     Functional tests:
 170
 171     {
 172         "metadata": {  # Optional
 173             "version": "VPP version",
 174             "job": "Jenkins job name",
 175             "build": "Information about the build"
 176         },
 177         "suites": {
 178             "Suite name 1": {
 179                 "doc": "Suite 1 documentation",
 180                 "parent": "Suite 1 parent",
 181                 "level": "Level of the suite in the suite hierarchy"
 182             }
 183             "Suite name N": {
 184                 "doc": "Suite N documentation",
 185                 "parent": "Suite 2 parent",
 186                 "level": "Level of the suite in the suite hierarchy"
 187             }
 188         }
 189         "tests": {
 190             "ID": {
 191                 "name": "Test name",
 192                 "parent": "Name of the parent of the test",
 193                 "doc": "Test documentation"
 194                 "msg": "Test message"
 195                 "tags": ["tag 1", "tag 2", "tag n"],
 196                 "conf-history": "DUT1 and DUT2 VAT History"
 197                 "show-run": "Show Run"
 198                 "status": "PASS" | "FAIL"
 199             },
 200             "ID" {
 201                 # next test
 202             }
 203         }
 204     }
 205
 206     .. note:: ID is the lowercase full path to the test.
 207     """
 208
 209     REGEX_PLR_RATE = re.compile(
 210         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 211         r'PLRsearch upper bound::?\s(\d+.\d+)'
 212     )
 213     REGEX_NDRPDR_RATE = re.compile(
 214         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 215         r'NDR_UPPER:\s(\d+.\d+).*\n'
 216         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 217         r'PDR_UPPER:\s(\d+.\d+)'
 218     )
 219     REGEX_NDRPDR_GBPS = re.compile(
 220         r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
 221         r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
 222         r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
 223         r'PDR_UPPER:.*,\s(\d+.\d+)'
 224     )
 225     REGEX_PERF_MSG_INFO = re.compile(
 226         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 227         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 228         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 229         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 230         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 231     )
 232     REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
 233
 234     # TODO: Remove when not needed
 235     REGEX_NDRPDR_LAT_BASE = re.compile(
 236         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 237         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 238     )
 239     REGEX_NDRPDR_LAT = re.compile(
 240         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 241         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 242         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 243         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 244         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 245         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 246     )
 247     # TODO: Remove when not needed
 248     REGEX_NDRPDR_LAT_LONG = re.compile(
 249         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 250         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 251         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 252         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 253         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 254         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 255         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 256         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 257         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 258     )
 259     REGEX_VERSION_VPP = re.compile(
 260         r"(return STDOUT Version:\s*|"
 261         r"VPP Version:\s*|VPP version:\s*)(.*)"
 262     )
 263     REGEX_VERSION_DPDK = re.compile(
 264         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 265     )
 266     REGEX_TCP = re.compile(
 267         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 268     )
 269     REGEX_MRR = re.compile(
 270         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 271         r'tx\s(\d*),\srx\s(\d*)'
 272     )
 273     REGEX_BMRR = re.compile(
 274         r'Maximum Receive Rate trial results'
 275         r' in packets per second: \[(.*)\]'
 276     )
 277     REGEX_RECONF_LOSS = re.compile(
 278         r'Packets lost due to reconfig: (\d*)'
 279     )
 280     REGEX_RECONF_TIME = re.compile(
 281         r'Implied time lost: (\d*.[\de-]*)'
 282     )
 283     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 284
 285     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 286
 287     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 288
 289     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 290
 291     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 292
 293     def __init__(self, metadata, mapping, ignore):
 294         """Initialisation.
 295
 296         :param metadata: Key-value pairs to be included in "metadata" part of
 297             JSON structure.
 298         :param mapping: Mapping of the old names of test cases to the new
 299             (actual) one.
 300         :param ignore: List of TCs to be ignored.
 301         :type metadata: dict
 302         :type mapping: dict
 303         :type ignore: list
 304         """
 305
 306         # Type of message to parse out from the test messages
 307         self._msg_type = None
 308
 309         # VPP version
 310         self._version = None
 311
 312         # Timestamp
 313         self._timestamp = None
 314
 315         # Testbed. The testbed is identified by TG node IP address.
 316         self._testbed = None
 317
 318         # Mapping of TCs long names
 319         self._mapping = mapping
 320
 321         # Ignore list
 322         self._ignore = ignore
 323
 324         # Number of PAPI History messages found:
 325         # 0 - no message
 326         # 1 - PAPI History of DUT1
 327         # 2 - PAPI History of DUT2
 328         self._conf_history_lookup_nr = 0
 329
 330         self._sh_run_counter = 0
 331
 332         # Test ID of currently processed test- the lowercase full path to the
 333         # test
 334         self._test_id = None
 335
 336         # The main data structure
 337         self._data = {
 338             u"metadata": OrderedDict(),
 339             u"suites": OrderedDict(),
 340             u"tests": OrderedDict()
 341         }
 342
 343         # Save the provided metadata
 344         for key, val in metadata.items():
 345             self._data[u"metadata"][key] = val
 346
 347         # Dictionary defining the methods used to parse different types of
 348         # messages
 349         self.parse_msg = {
 350             u"timestamp": self._get_timestamp,
 351             u"vpp-version": self._get_vpp_version,
 352             u"dpdk-version": self._get_dpdk_version,
 353             # TODO: Remove when not needed:
 354             u"teardown-vat-history": self._get_vat_history,
 355             u"teardown-papi-history": self._get_papi_history,
 356             u"test-show-runtime": self._get_show_run,
 357             u"testbed": self._get_testbed
 358         }
 359
 360     @property
 361     def data(self):
 362         """Getter - Data parsed from the XML file.
 363
 364         :returns: Data parsed from the XML file.
 365         :rtype: dict
 366         """
 367         return self._data
 368
 369     def _get_data_from_mrr_test_msg(self, msg):
 370         """Get info from message of MRR performance tests.
 371
 372         :param msg: Message to be processed.
 373         :type msg: str
 374         :returns: Processed message or original message if a problem occurs.
 375         :rtype: str
 376         """
 377
 378         groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
 379         if not groups or groups.lastindex != 1:
 380             return u"Test Failed."
 381
 382         try:
 383             data = groups.group(1).split(u", ")
 384         except (AttributeError, IndexError, ValueError, KeyError):
 385             return u"Test Failed."
 386
 387         out_str = u"["
 388         try:
 389             for item in data:
 390                 out_str += f"{(float(item) / 1e6):.2f}, "
 391             return out_str[:-2] + u"]"
 392         except (AttributeError, IndexError, ValueError, KeyError):
 393             return u"Test Failed."
 394
 395     def _get_data_from_perf_test_msg(self, msg):
 396         """Get info from message of NDRPDR performance tests.
 397
 398         :param msg: Message to be processed.
 399         :type msg: str
 400         :returns: Processed message or original message if a problem occurs.
 401         :rtype: str
 402         """
 403
 404         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 405         if not groups or groups.lastindex != 10:
 406             return u"Test Failed."
 407
 408         try:
 409             data = {
 410                 u"ndr_low": float(groups.group(1)),
 411                 u"ndr_low_b": float(groups.group(2)),
 412                 u"pdr_low": float(groups.group(3)),
 413                 u"pdr_low_b": float(groups.group(4)),
 414                 u"pdr_lat_90_1": groups.group(5),
 415                 u"pdr_lat_90_2": groups.group(6),
 416                 u"pdr_lat_50_1": groups.group(7),
 417                 u"pdr_lat_50_2": groups.group(8),
 418                 u"pdr_lat_10_1": groups.group(9),
 419                 u"pdr_lat_10_2": groups.group(10),
 420             }
 421         except (AttributeError, IndexError, ValueError, KeyError):
 422             return u"Test Failed."
 423
 424         def _process_lat(in_str_1, in_str_2):
 425             """Extract min, avg, max values from latency string.
 426
 427             :param in_str_1: Latency string for one direction produced by robot
 428                 framework.
 429             :param in_str_2: Latency string for second direction produced by
 430                 robot framework.
 431             :type in_str_1: str
 432             :type in_str_2: str
 433             :returns: Processed latency string or None if a problem occurs.
 434             :rtype: tuple
 435             """
 436             in_list_1 = in_str_1.split('/', 3)
 437             in_list_2 = in_str_2.split('/', 3)
 438
 439             if len(in_list_1) != 4 and len(in_list_2) != 4:
 440                 return None
 441
 442             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 443             try:
 444                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 445             except hdrh.codec.HdrLengthException:
 446                 return None
 447
 448             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 449             try:
 450                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 451             except hdrh.codec.HdrLengthException:
 452                 return None
 453
 454             if hdr_lat_1 and hdr_lat_2:
 455                 hdr_lat = (
 456                     hdr_lat_1.get_value_at_percentile(50.0),
 457                     hdr_lat_1.get_value_at_percentile(90.0),
 458                     hdr_lat_1.get_value_at_percentile(99.0),
 459                     hdr_lat_2.get_value_at_percentile(50.0),
 460                     hdr_lat_2.get_value_at_percentile(90.0),
 461                     hdr_lat_2.get_value_at_percentile(99.0)
 462                 )
 463
 464                 if all(hdr_lat):
 465                     return hdr_lat
 466
 467             return None
 468
 469         try:
 470             out_msg = (
 471                 f"1. {(data[u'ndr_low'] / 1e6):5.2f}      "
 472                 f"{data[u'ndr_low_b']:5.2f}"
 473                 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f}      "
 474                 f"{data[u'pdr_low_b']:5.2f}"
 475             )
 476             latency = (
 477                 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
 478                 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
 479                 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
 480             )
 481             if all(latency):
 482                 max_len = len(str(max((max(item) for item in latency))))
 483                 max_len = 4 if max_len < 4 else max_len
 484
 485                 for idx, lat in enumerate(latency):
 486                     if not idx:
 487                         out_msg += u"\n"
 488                     out_msg += (
 489                         f"\n{idx + 3}. "
 490                         f"{lat[0]:{max_len}d} "
 491                         f"{lat[1]:{max_len}d} "
 492                         f"{lat[2]:{max_len}d}      "
 493                         f"{lat[3]:{max_len}d} "
 494                         f"{lat[4]:{max_len}d} "
 495                         f"{lat[5]:{max_len}d} "
 496                     )
 497
 498             return out_msg
 499
 500         except (AttributeError, IndexError, ValueError, KeyError):
 501             return u"Test Failed."
 502
 503     def _get_testbed(self, msg):
 504         """Called when extraction of testbed IP is required.
 505         The testbed is identified by TG node IP address.
 506
 507         :param msg: Message to process.
 508         :type msg: Message
 509         :returns: Nothing.
 510         """
 511
 512         if msg.message.count(u"Setup of TG node") or \
 513                 msg.message.count(u"Setup of node TG host"):
 514             reg_tg_ip = re.compile(
 515                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 516             try:
 517                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 518             except (KeyError, ValueError, IndexError, AttributeError):
 519                 pass
 520             finally:
 521                 self._data[u"metadata"][u"testbed"] = self._testbed
 522                 self._msg_type = None
 523
 524     def _get_vpp_version(self, msg):
 525         """Called when extraction of VPP version is required.
 526
 527         :param msg: Message to process.
 528         :type msg: Message
 529         :returns: Nothing.
 530         """
 531
 532         if msg.message.count(u"return STDOUT Version:") or \
 533                 msg.message.count(u"VPP Version:") or \
 534                 msg.message.count(u"VPP version:"):
 535             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 536                                 group(2))
 537             self._data[u"metadata"][u"version"] = self._version
 538             self._msg_type = None
 539
 540     def _get_dpdk_version(self, msg):
 541         """Called when extraction of DPDK version is required.
 542
 543         :param msg: Message to process.
 544         :type msg: Message
 545         :returns: Nothing.
 546         """
 547
 548         if msg.message.count(u"DPDK Version:"):
 549             try:
 550                 self._version = str(re.search(
 551                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 552                 self._data[u"metadata"][u"version"] = self._version
 553             except IndexError:
 554                 pass
 555             finally:
 556                 self._msg_type = None
 557
 558     def _get_timestamp(self, msg):
 559         """Called when extraction of timestamp is required.
 560
 561         :param msg: Message to process.
 562         :type msg: Message
 563         :returns: Nothing.
 564         """
 565
 566         self._timestamp = msg.timestamp[:14]
 567         self._data[u"metadata"][u"generated"] = self._timestamp
 568         self._msg_type = None
 569
 570     def _get_vat_history(self, msg):
 571         """Called when extraction of VAT command history is required.
 572
 573         TODO: Remove when not needed.
 574
 575         :param msg: Message to process.
 576         :type msg: Message
 577         :returns: Nothing.
 578         """
 579         if msg.message.count(u"VAT command history:"):
 580             self._conf_history_lookup_nr += 1
 581             if self._conf_history_lookup_nr == 1:
 582                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 583             else:
 584                 self._msg_type = None
 585             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 586                           r"VAT command history:", u"",
 587                           msg.message, count=1).replace(u'\n', u' |br| ').\
 588                 replace(u'"', u"'")
 589
 590             self._data[u"tests"][self._test_id][u"conf-history"] += (
 591                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 592             )
 593
 594     def _get_papi_history(self, msg):
 595         """Called when extraction of PAPI command history is required.
 596
 597         :param msg: Message to process.
 598         :type msg: Message
 599         :returns: Nothing.
 600         """
 601         if msg.message.count(u"PAPI command history:"):
 602             self._conf_history_lookup_nr += 1
 603             if self._conf_history_lookup_nr == 1:
 604                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 605             else:
 606                 self._msg_type = None
 607             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 608                           r"PAPI command history:", u"",
 609                           msg.message, count=1).replace(u'\n', u' |br| ').\
 610                 replace(u'"', u"'")
 611             self._data[u"tests"][self._test_id][u"conf-history"] += (
 612                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 613             )
 614
 615     def _get_show_run(self, msg):
 616         """Called when extraction of VPP operational data (output of CLI command
 617         Show Runtime) is required.
 618
 619         :param msg: Message to process.
 620         :type msg: Message
 621         :returns: Nothing.
 622         """
 623
 624         if not msg.message.count(u"stats runtime"):
 625             return
 626
 627         # Temporary solution
 628         if self._sh_run_counter > 1:
 629             return
 630
 631         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 632             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 633
 634         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 635         if not groups:
 636             return
 637         try:
 638             host = groups.group(1)
 639         except (AttributeError, IndexError):
 640             host = u""
 641         try:
 642             sock = groups.group(2)
 643         except (AttributeError, IndexError):
 644             sock = u""
 645
 646         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 647                         replace(u"'", u'"').replace(u'b"', u'"').
 648                         replace(u'u"', u'"').split(u":", 1)[1])
 649
 650         try:
 651             threads_nr = len(runtime[0][u"clocks"])
 652         except (IndexError, KeyError):
 653             return
 654
 655         dut = u"DUT{nr}".format(
 656             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 657
 658         oper = {
 659             u"host": host,
 660             u"socket": sock,
 661             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 662         }
 663
 664         for item in runtime:
 665             for idx in range(threads_nr):
 666                 if item[u"vectors"][idx] > 0:
 667                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 668                 elif item[u"calls"][idx] > 0:
 669                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 670                 elif item[u"suspends"][idx] > 0:
 671                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 672                 else:
 673                     clocks = 0.0
 674
 675                 if item[u"calls"][idx] > 0:
 676                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 677                 else:
 678                     vectors_call = 0.0
 679
 680                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 681                         int(item[u"suspends"][idx]):
 682                     oper[u"threads"][idx].append([
 683                         item[u"name"],
 684                         item[u"calls"][idx],
 685                         item[u"vectors"][idx],
 686                         item[u"suspends"][idx],
 687                         clocks,
 688                         vectors_call
 689                     ])
 690
 691         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 692
 693     def _get_ndrpdr_throughput(self, msg):
 694         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 695         message.
 696
 697         :param msg: The test message to be parsed.
 698         :type msg: str
 699         :returns: Parsed data as a dict and the status (PASS/FAIL).
 700         :rtype: tuple(dict, str)
 701         """
 702
 703         throughput = {
 704             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 705             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 706         }
 707         status = u"FAIL"
 708         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 709
 710         if groups is not None:
 711             try:
 712                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 713                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 714                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 715                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 716                 status = u"PASS"
 717             except (IndexError, ValueError):
 718                 pass
 719
 720         return throughput, status
 721
 722     def _get_ndrpdr_throughput_gbps(self, msg):
 723         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
 724         test message.
 725
 726         :param msg: The test message to be parsed.
 727         :type msg: str
 728         :returns: Parsed data as a dict and the status (PASS/FAIL).
 729         :rtype: tuple(dict, str)
 730         """
 731
 732         gbps = {
 733             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 734             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 735         }
 736         status = u"FAIL"
 737         groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
 738
 739         if groups is not None:
 740             try:
 741                 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
 742                 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
 743                 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
 744                 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
 745                 status = u"PASS"
 746             except (IndexError, ValueError):
 747                 pass
 748
 749         return gbps, status
 750
 751     def _get_plr_throughput(self, msg):
 752         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 753         message.
 754
 755         :param msg: The test message to be parsed.
 756         :type msg: str
 757         :returns: Parsed data as a dict and the status (PASS/FAIL).
 758         :rtype: tuple(dict, str)
 759         """
 760
 761         throughput = {
 762             u"LOWER": -1.0,
 763             u"UPPER": -1.0
 764         }
 765         status = u"FAIL"
 766         groups = re.search(self.REGEX_PLR_RATE, msg)
 767
 768         if groups is not None:
 769             try:
 770                 throughput[u"LOWER"] = float(groups.group(1))
 771                 throughput[u"UPPER"] = float(groups.group(2))
 772                 status = u"PASS"
 773             except (IndexError, ValueError):
 774                 pass
 775
 776         return throughput, status
 777
 778     def _get_ndrpdr_latency(self, msg):
 779         """Get LATENCY from the test message.
 780
 781         :param msg: The test message to be parsed.
 782         :type msg: str
 783         :returns: Parsed data as a dict and the status (PASS/FAIL).
 784         :rtype: tuple(dict, str)
 785         """
 786         latency_default = {
 787             u"min": -1.0,
 788             u"avg": -1.0,
 789             u"max": -1.0,
 790             u"hdrh": u""
 791         }
 792         latency = {
 793             u"NDR": {
 794                 u"direction1": copy.copy(latency_default),
 795                 u"direction2": copy.copy(latency_default)
 796             },
 797             u"PDR": {
 798                 u"direction1": copy.copy(latency_default),
 799                 u"direction2": copy.copy(latency_default)
 800             },
 801             u"LAT0": {
 802                 u"direction1": copy.copy(latency_default),
 803                 u"direction2": copy.copy(latency_default)
 804             },
 805             u"PDR10": {
 806                 u"direction1": copy.copy(latency_default),
 807                 u"direction2": copy.copy(latency_default)
 808             },
 809             u"PDR50": {
 810                 u"direction1": copy.copy(latency_default),
 811                 u"direction2": copy.copy(latency_default)
 812             },
 813             u"PDR90": {
 814                 u"direction1": copy.copy(latency_default),
 815                 u"direction2": copy.copy(latency_default)
 816             },
 817         }
 818
 819         # TODO: Rewrite when long and base are not needed
 820         groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
 821         if groups is None:
 822             groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 823         if groups is None:
 824             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 825         if groups is None:
 826             return latency, u"FAIL"
 827
 828         def process_latency(in_str):
 829             """Return object with parsed latency values.
 830
 831             TODO: Define class for the return type.
 832
 833             :param in_str: Input string, min/avg/max/hdrh format.
 834             :type in_str: str
 835             :returns: Dict with corresponding keys, except hdrh float values.
 836             :rtype dict:
 837             :throws IndexError: If in_str does not have enough substrings.
 838             :throws ValueError: If a substring does not convert to float.
 839             """
 840             in_list = in_str.split('/', 3)
 841
 842             rval = {
 843                 u"min": float(in_list[0]),
 844                 u"avg": float(in_list[1]),
 845                 u"max": float(in_list[2]),
 846                 u"hdrh": u""
 847             }
 848
 849             if len(in_list) == 4:
 850                 rval[u"hdrh"] = str(in_list[3])
 851
 852             return rval
 853
 854         try:
 855             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 856             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 857             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 858             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 859             if groups.lastindex == 4:
 860                 return latency, u"PASS"
 861         except (IndexError, ValueError):
 862             pass
 863
 864         try:
 865             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 866             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 867             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 868             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 869             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 870             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 871             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 872             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 873             if groups.lastindex == 12:
 874                 return latency, u"PASS"
 875         except (IndexError, ValueError):
 876             pass
 877
 878         # TODO: Remove when not needed
 879         latency[u"NDR10"] = {
 880             u"direction1": copy.copy(latency_default),
 881             u"direction2": copy.copy(latency_default)
 882         }
 883         latency[u"NDR50"] = {
 884             u"direction1": copy.copy(latency_default),
 885             u"direction2": copy.copy(latency_default)
 886         }
 887         latency[u"NDR90"] = {
 888             u"direction1": copy.copy(latency_default),
 889             u"direction2": copy.copy(latency_default)
 890         }
 891         try:
 892             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
 893             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
 894             latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
 895             latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
 896             latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
 897             latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
 898             latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
 899             latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
 900             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
 901             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
 902             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
 903             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
 904             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
 905             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
 906             return latency, u"PASS"
 907         except (IndexError, ValueError):
 908             pass
 909
 910         return latency, u"FAIL"
 911
 912     @staticmethod
 913     def _get_hoststack_data(msg, tags):
 914         """Get data from the hoststack test message.
 915
 916         :param msg: The test message to be parsed.
 917         :param tags: Test tags.
 918         :type msg: str
 919         :type tags: list
 920         :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
 921         :rtype: tuple(dict, str)
 922         """
 923         result = dict()
 924         status = u"FAIL"
 925
 926         msg = msg.replace(u"'", u'"').replace(u" ", u"")
 927         if u"LDPRELOAD" in tags:
 928             try:
 929                 result = loads(msg)
 930                 status = u"PASS"
 931             except JSONDecodeError:
 932                 pass
 933         elif u"VPPECHO" in tags:
 934             try:
 935                 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
 936                 result = dict(
 937                     client=loads(msg_lst[0]),
 938                     server=loads(msg_lst[1])
 939                 )
 940                 status = u"PASS"
 941             except (JSONDecodeError, IndexError):
 942                 pass
 943
 944         return result, status
 945
 946     def visit_suite(self, suite):
 947         """Implements traversing through the suite and its direct children.
 948
 949         :param suite: Suite to process.
 950         :type suite: Suite
 951         :returns: Nothing.
 952         """
 953         if self.start_suite(suite) is not False:
 954             suite.suites.visit(self)
 955             suite.tests.visit(self)
 956             self.end_suite(suite)
 957
 958     def start_suite(self, suite):
 959         """Called when suite starts.
 960
 961         :param suite: Suite to process.
 962         :type suite: Suite
 963         :returns: Nothing.
 964         """
 965
 966         try:
 967             parent_name = suite.parent.name
 968         except AttributeError:
 969             return
 970
 971         doc_str = suite.doc.\
 972             replace(u'"', u"'").\
 973             replace(u'\n', u' ').\
 974             replace(u'\r', u'').\
 975             replace(u'*[', u' |br| *[').\
 976             replace(u"*", u"**").\
 977             replace(u' |br| *[', u'*[', 1)
 978
 979         self._data[u"suites"][suite.longname.lower().
 980                               replace(u'"', u"'").
 981                               replace(u" ", u"_")] = {
 982                                   u"name": suite.name.lower(),
 983                                   u"doc": doc_str,
 984                                   u"parent": parent_name,
 985                                   u"level": len(suite.longname.split(u"."))
 986                               }
 987
 988         suite.keywords.visit(self)
 989
 990     def end_suite(self, suite):
 991         """Called when suite ends.
 992
 993         :param suite: Suite to process.
 994         :type suite: Suite
 995         :returns: Nothing.
 996         """
 997
 998     def visit_test(self, test):
 999         """Implements traversing through the test.
1000
1001         :param test: Test to process.
1002         :type test: Test
1003         :returns: Nothing.
1004         """
1005         if self.start_test(test) is not False:
1006             test.keywords.visit(self)
1007             self.end_test(test)
1008
1009     def start_test(self, test):
1010         """Called when test starts.
1011
1012         :param test: Test to process.
1013         :type test: Test
1014         :returns: Nothing.
1015         """
1016
1017         self._sh_run_counter = 0
1018
1019         longname_orig = test.longname.lower()
1020
1021         # Check the ignore list
1022         if longname_orig in self._ignore:
1023             return
1024
1025         tags = [str(tag) for tag in test.tags]
1026         test_result = dict()
1027
1028         # Change the TC long name and name if defined in the mapping table
1029         longname = self._mapping.get(longname_orig, None)
1030         if longname is not None:
1031             name = longname.split(u'.')[-1]
1032             logging.debug(
1033                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1034                 f"{name}"
1035             )
1036         else:
1037             longname = longname_orig
1038             name = test.name.lower()
1039
1040         # Remove TC number from the TC long name (backward compatibility):
1041         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1042         # Remove TC number from the TC name (not needed):
1043         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1044
1045         test_result[u"parent"] = test.parent.name.lower()
1046         test_result[u"tags"] = tags
1047         test_result["doc"] = test.doc.\
1048             replace(u'"', u"'").\
1049             replace(u'\n', u' ').\
1050             replace(u'\r', u'').\
1051             replace(u'[', u' |br| [').\
1052             replace(u' |br| [', u'[', 1)
1053         test_result[u"type"] = u"FUNC"
1054         test_result[u"status"] = test.status
1055
1056         if test.status == u"PASS":
1057             if u"NDRPDR" in tags:
1058                 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1059                     test.message).replace(u'\n', u' |br| ').\
1060                     replace(u'\r', u'').replace(u'"', u"'")
1061             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1062                 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1063                     test.message).replace(u'\n', u' |br| ').\
1064                     replace(u'\r', u'').replace(u'"', u"'")
1065             else:
1066                 test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\
1067                     replace(u'\r', u'').replace(u'"', u"'")
1068         else:
1069             test_result[u"msg"] = u"Test Failed."
1070
1071         if u"PERFTEST" in tags:
1072             # Replace info about cores (e.g. -1c-) with the info about threads
1073             # and cores (e.g. -1t1c-) in the long test case names and in the
1074             # test case names if necessary.
1075             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1076             if not groups:
1077                 tag_count = 0
1078                 tag_tc = str()
1079                 for tag in test_result[u"tags"]:
1080                     groups = re.search(self.REGEX_TC_TAG, tag)
1081                     if groups:
1082                         tag_count += 1
1083                         tag_tc = tag
1084
1085                 if tag_count == 1:
1086                     self._test_id = re.sub(
1087                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1088                         self._test_id, count=1
1089                     )
1090                     test_result[u"name"] = re.sub(
1091                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1092                         test_result["name"], count=1
1093                     )
1094                 else:
1095                     test_result[u"status"] = u"FAIL"
1096                     self._data[u"tests"][self._test_id] = test_result
1097                     logging.debug(
1098                         f"The test {self._test_id} has no or more than one "
1099                         f"multi-threading tags.\n"
1100                         f"Tags: {test_result[u'tags']}"
1101                     )
1102                     return
1103
1104         if test.status == u"PASS":
1105             if u"NDRPDR" in tags:
1106                 test_result[u"type"] = u"NDRPDR"
1107                 test_result[u"throughput"], test_result[u"status"] = \
1108                     self._get_ndrpdr_throughput(test.message)
1109                 test_result[u"gbps"], test_result[u"status"] = \
1110                     self._get_ndrpdr_throughput_gbps(test.message)
1111                 test_result[u"latency"], test_result[u"status"] = \
1112                     self._get_ndrpdr_latency(test.message)
1113             elif u"SOAK" in tags:
1114                 test_result[u"type"] = u"SOAK"
1115                 test_result[u"throughput"], test_result[u"status"] = \
1116                     self._get_plr_throughput(test.message)
1117             elif u"HOSTSTACK" in tags:
1118                 test_result[u"type"] = u"HOSTSTACK"
1119                 test_result[u"result"], test_result[u"status"] = \
1120                     self._get_hoststack_data(test.message, tags)
1121             elif u"TCP" in tags:
1122                 test_result[u"type"] = u"TCP"
1123                 groups = re.search(self.REGEX_TCP, test.message)
1124                 test_result[u"result"] = int(groups.group(2))
1125             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1126                 if u"MRR" in tags:
1127                     test_result[u"type"] = u"MRR"
1128                 else:
1129                     test_result[u"type"] = u"BMRR"
1130
1131                 test_result[u"result"] = dict()
1132                 groups = re.search(self.REGEX_BMRR, test.message)
1133                 if groups is not None:
1134                     items_str = groups.group(1)
1135                     items_float = [
1136                         float(item.strip()) for item in items_str.split(",")
1137                     ]
1138                     # Use whole list in CSIT-1180.
1139                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1140                     test_result[u"result"][u"receive-rate"] = stats.avg
1141                     test_result[u"result"][u"receive-stdev"] = stats.stdev
1142                 else:
1143                     groups = re.search(self.REGEX_MRR, test.message)
1144                     test_result[u"result"][u"receive-rate"] = \
1145                         float(groups.group(3)) / float(groups.group(1))
1146             elif u"RECONF" in tags:
1147                 test_result[u"type"] = u"RECONF"
1148                 test_result[u"result"] = None
1149                 try:
1150                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1151                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1152                     test_result[u"result"] = {
1153                         u"loss": int(grps_loss.group(1)),
1154                         u"time": float(grps_time.group(1))
1155                     }
1156                 except (AttributeError, IndexError, ValueError, TypeError):
1157                     test_result[u"status"] = u"FAIL"
1158             elif u"DEVICETEST" in tags:
1159                 test_result[u"type"] = u"DEVICETEST"
1160             else:
1161                 test_result[u"status"] = u"FAIL"
1162                 self._data[u"tests"][self._test_id] = test_result
1163                 return
1164
1165         self._data[u"tests"][self._test_id] = test_result
1166
1167     def end_test(self, test):
1168         """Called when test ends.
1169
1170         :param test: Test to process.
1171         :type test: Test
1172         :returns: Nothing.
1173         """
1174
1175     def visit_keyword(self, keyword):
1176         """Implements traversing through the keyword and its child keywords.
1177
1178         :param keyword: Keyword to process.
1179         :type keyword: Keyword
1180         :returns: Nothing.
1181         """
1182         if self.start_keyword(keyword) is not False:
1183             self.end_keyword(keyword)
1184
1185     def start_keyword(self, keyword):
1186         """Called when keyword starts. Default implementation does nothing.
1187
1188         :param keyword: Keyword to process.
1189         :type keyword: Keyword
1190         :returns: Nothing.
1191         """
1192         try:
1193             if keyword.type == u"setup":
1194                 self.visit_setup_kw(keyword)
1195             elif keyword.type == u"teardown":
1196                 self.visit_teardown_kw(keyword)
1197             else:
1198                 self.visit_test_kw(keyword)
1199         except AttributeError:
1200             pass
1201
1202     def end_keyword(self, keyword):
1203         """Called when keyword ends. Default implementation does nothing.
1204
1205         :param keyword: Keyword to process.
1206         :type keyword: Keyword
1207         :returns: Nothing.
1208         """
1209
1210     def visit_test_kw(self, test_kw):
1211         """Implements traversing through the test keyword and its child
1212         keywords.
1213
1214         :param test_kw: Keyword to process.
1215         :type test_kw: Keyword
1216         :returns: Nothing.
1217         """
1218         for keyword in test_kw.keywords:
1219             if self.start_test_kw(keyword) is not False:
1220                 self.visit_test_kw(keyword)
1221                 self.end_test_kw(keyword)
1222
1223     def start_test_kw(self, test_kw):
1224         """Called when test keyword starts. Default implementation does
1225         nothing.
1226
1227         :param test_kw: Keyword to process.
1228         :type test_kw: Keyword
1229         :returns: Nothing.
1230         """
1231         if test_kw.name.count(u"Show Runtime On All Duts") or \
1232                 test_kw.name.count(u"Show Runtime Counters On All Duts") or \
1233                 test_kw.name.count(u"Vpp Show Runtime On All Duts"):
1234             self._msg_type = u"test-show-runtime"
1235             self._sh_run_counter += 1
1236         else:
1237             return
1238         test_kw.messages.visit(self)
1239
1240     def end_test_kw(self, test_kw):
1241         """Called when keyword ends. Default implementation does nothing.
1242
1243         :param test_kw: Keyword to process.
1244         :type test_kw: Keyword
1245         :returns: Nothing.
1246         """
1247
1248     def visit_setup_kw(self, setup_kw):
1249         """Implements traversing through the teardown keyword and its child
1250         keywords.
1251
1252         :param setup_kw: Keyword to process.
1253         :type setup_kw: Keyword
1254         :returns: Nothing.
1255         """
1256         for keyword in setup_kw.keywords:
1257             if self.start_setup_kw(keyword) is not False:
1258                 self.visit_setup_kw(keyword)
1259                 self.end_setup_kw(keyword)
1260
1261     def start_setup_kw(self, setup_kw):
1262         """Called when teardown keyword starts. Default implementation does
1263         nothing.
1264
1265         :param setup_kw: Keyword to process.
1266         :type setup_kw: Keyword
1267         :returns: Nothing.
1268         """
1269         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1270                 and not self._version:
1271             self._msg_type = u"vpp-version"
1272         elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1273                 not self._version:
1274             self._msg_type = u"dpdk-version"
1275         elif setup_kw.name.count(u"Set Global Variable") \
1276                 and not self._timestamp:
1277             self._msg_type = u"timestamp"
1278         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1279             self._msg_type = u"testbed"
1280         else:
1281             return
1282         setup_kw.messages.visit(self)
1283
1284     def end_setup_kw(self, setup_kw):
1285         """Called when keyword ends. Default implementation does nothing.
1286
1287         :param setup_kw: Keyword to process.
1288         :type setup_kw: Keyword
1289         :returns: Nothing.
1290         """
1291
1292     def visit_teardown_kw(self, teardown_kw):
1293         """Implements traversing through the teardown keyword and its child
1294         keywords.
1295
1296         :param teardown_kw: Keyword to process.
1297         :type teardown_kw: Keyword
1298         :returns: Nothing.
1299         """
1300         for keyword in teardown_kw.keywords:
1301             if self.start_teardown_kw(keyword) is not False:
1302                 self.visit_teardown_kw(keyword)
1303                 self.end_teardown_kw(keyword)
1304
1305     def start_teardown_kw(self, teardown_kw):
1306         """Called when teardown keyword starts
1307
1308         :param teardown_kw: Keyword to process.
1309         :type teardown_kw: Keyword
1310         :returns: Nothing.
1311         """
1312
1313         if teardown_kw.name.count(u"Show Vat History On All Duts"):
1314             # TODO: Remove when not needed:
1315             self._conf_history_lookup_nr = 0
1316             self._msg_type = u"teardown-vat-history"
1317             teardown_kw.messages.visit(self)
1318         elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1319             self._conf_history_lookup_nr = 0
1320             self._msg_type = u"teardown-papi-history"
1321             teardown_kw.messages.visit(self)
1322
1323     def end_teardown_kw(self, teardown_kw):
1324         """Called when keyword ends. Default implementation does nothing.
1325
1326         :param teardown_kw: Keyword to process.
1327         :type teardown_kw: Keyword
1328         :returns: Nothing.
1329         """
1330
1331     def visit_message(self, msg):
1332         """Implements visiting the message.
1333
1334         :param msg: Message to process.
1335         :type msg: Message
1336         :returns: Nothing.
1337         """
1338         if self.start_message(msg) is not False:
1339             self.end_message(msg)
1340
1341     def start_message(self, msg):
1342         """Called when message starts. Get required information from messages:
1343         - VPP version.
1344
1345         :param msg: Message to process.
1346         :type msg: Message
1347         :returns: Nothing.
1348         """
1349         if self._msg_type:
1350             self.parse_msg[self._msg_type](msg)
1351
1352     def end_message(self, msg):
1353         """Called when message ends. Default implementation does nothing.
1354
1355         :param msg: Message to process.
1356         :type msg: Message
1357         :returns: Nothing.
1358         """
1359
1360
1361 class InputData:
1362     """Input data
1363
1364     The data is extracted from output.xml files generated by Jenkins jobs and
1365     stored in pandas' DataFrames.
1366
1367     The data structure:
1368     - job name
1369       - build number
1370         - metadata
1371           (as described in ExecutionChecker documentation)
1372         - suites
1373           (as described in ExecutionChecker documentation)
1374         - tests
1375           (as described in ExecutionChecker documentation)
1376     """
1377
1378     def __init__(self, spec):
1379         """Initialization.
1380
1381         :param spec: Specification.
1382         :type spec: Specification
1383         """
1384
1385         # Specification:
1386         self._cfg = spec
1387
1388         # Data store:
1389         self._input_data = pd.Series()
1390
1391     @property
1392     def data(self):
1393         """Getter - Input data.
1394
1395         :returns: Input data
1396         :rtype: pandas.Series
1397         """
1398         return self._input_data
1399
1400     def metadata(self, job, build):
1401         """Getter - metadata
1402
1403         :param job: Job which metadata we want.
1404         :param build: Build which metadata we want.
1405         :type job: str
1406         :type build: str
1407         :returns: Metadata
1408         :rtype: pandas.Series
1409         """
1410         return self.data[job][build][u"metadata"]
1411
1412     def suites(self, job, build):
1413         """Getter - suites
1414
1415         :param job: Job which suites we want.
1416         :param build: Build which suites we want.
1417         :type job: str
1418         :type build: str
1419         :returns: Suites.
1420         :rtype: pandas.Series
1421         """
1422         return self.data[job][str(build)][u"suites"]
1423
1424     def tests(self, job, build):
1425         """Getter - tests
1426
1427         :param job: Job which tests we want.
1428         :param build: Build which tests we want.
1429         :type job: str
1430         :type build: str
1431         :returns: Tests.
1432         :rtype: pandas.Series
1433         """
1434         return self.data[job][build][u"tests"]
1435
1436     def _parse_tests(self, job, build):
1437         """Process data from robot output.xml file and return JSON structured
1438         data.
1439
1440         :param job: The name of job which build output data will be processed.
1441         :param build: The build which output data will be processed.
1442         :type job: str
1443         :type build: dict
1444         :returns: JSON data structure.
1445         :rtype: dict
1446         """
1447
1448         metadata = {
1449             u"job": job,
1450             u"build": build
1451         }
1452
1453         with open(build[u"file-name"], u'r') as data_file:
1454             try:
1455                 result = ExecutionResult(data_file)
1456             except errors.DataError as err:
1457                 logging.error(
1458                     f"Error occurred while parsing output.xml: {repr(err)}"
1459                 )
1460                 return None
1461         checker = ExecutionChecker(metadata, self._cfg.mapping,
1462                                    self._cfg.ignore)
1463         result.visit(checker)
1464
1465         return checker.data
1466
1467     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1468         """Download and parse the input data file.
1469
1470         :param pid: PID of the process executing this method.
1471         :param job: Name of the Jenkins job which generated the processed input
1472             file.
1473         :param build: Information about the Jenkins build which generated the
1474             processed input file.
1475         :param repeat: Repeat the download specified number of times if not
1476             successful.
1477         :type pid: int
1478         :type job: str
1479         :type build: dict
1480         :type repeat: int
1481         """
1482
1483         logging.info(f"  Processing the job/build: {job}: {build[u'build']}")
1484
1485         state = u"failed"
1486         success = False
1487         data = None
1488         do_repeat = repeat
1489         while do_repeat:
1490             success = download_and_unzip_data_file(self._cfg, job, build, pid)
1491             if success:
1492                 break
1493             do_repeat -= 1
1494         if not success:
1495             logging.error(
1496                 f"It is not possible to download the input data file from the "
1497                 f"job {job}, build {build[u'build']}, or it is damaged. "
1498                 f"Skipped."
1499             )
1500         if success:
1501             logging.info(f"    Processing data from build {build[u'build']}")
1502             data = self._parse_tests(job, build)
1503             if data is None:
1504                 logging.error(
1505                     f"Input data file from the job {job}, build "
1506                     f"{build[u'build']} is damaged. Skipped."
1507                 )
1508             else:
1509                 state = u"processed"
1510
1511             try:
1512                 remove(build[u"file-name"])
1513             except OSError as err:
1514                 logging.error(
1515                     f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1516                 )
1517
1518         # If the time-period is defined in the specification file, remove all
1519         # files which are outside the time period.
1520         is_last = False
1521         timeperiod = self._cfg.input.get(u"time-period", None)
1522         if timeperiod and data:
1523             now = dt.utcnow()
1524             timeperiod = timedelta(int(timeperiod))
1525             metadata = data.get(u"metadata", None)
1526             if metadata:
1527                 generated = metadata.get(u"generated", None)
1528                 if generated:
1529                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1530                     if (now - generated) > timeperiod:
1531                         # Remove the data and the file:
1532                         state = u"removed"
1533                         data = None
1534                         is_last = True
1535                         logging.info(
1536                             f"    The build {job}/{build[u'build']} is "
1537                             f"outdated, will be removed."
1538                         )
1539         logging.info(u"  Done.")
1540
1541         return {
1542             u"data": data,
1543             u"state": state,
1544             u"job": job,
1545             u"build": build,
1546             u"last": is_last
1547         }
1548
1549     def download_and_parse_data(self, repeat=1):
1550         """Download the input data files, parse input data from input files and
1551         store in pandas' Series.
1552
1553         :param repeat: Repeat the download specified number of times if not
1554             successful.
1555         :type repeat: int
1556         """
1557
1558         logging.info(u"Downloading and parsing input files ...")
1559
1560         for job, builds in self._cfg.builds.items():
1561             for build in builds:
1562
1563                 result = self._download_and_parse_build(job, build, repeat)
1564                 if result[u"last"]:
1565                     break
1566                 build_nr = result[u"build"][u"build"]
1567
1568                 if result[u"data"]:
1569                     data = result[u"data"]
1570                     build_data = pd.Series({
1571                         u"metadata": pd.Series(
1572                             list(data[u"metadata"].values()),
1573                             index=list(data[u"metadata"].keys())
1574                         ),
1575                         u"suites": pd.Series(
1576                             list(data[u"suites"].values()),
1577                             index=list(data[u"suites"].keys())
1578                         ),
1579                         u"tests": pd.Series(
1580                             list(data[u"tests"].values()),
1581                             index=list(data[u"tests"].keys())
1582                         )
1583                     })
1584
1585                     if self._input_data.get(job, None) is None:
1586                         self._input_data[job] = pd.Series()
1587                     self._input_data[job][str(build_nr)] = build_data
1588
1589                     self._cfg.set_input_file_name(
1590                         job, build_nr, result[u"build"][u"file-name"])
1591
1592                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1593
1594                 mem_alloc = \
1595                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1596                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1597
1598         logging.info(u"Done.")
1599
1600     def process_local_file(self, local_file, job=u"local", build_nr=1,
1601                            replace=True):
1602         """Process local XML file given as a command-line parameter.
1603
1604         :param local_file: The file to process.
1605         :param job: Job name.
1606         :param build_nr: Build number.
1607         :param replace: If True, the information about jobs and builds is
1608             replaced by the new one, otherwise the new jobs and builds are
1609             added.
1610         :type local_file: str
1611         :type job: str
1612         :type build_nr: int
1613         :type replace: bool
1614         :raises: PresentationError if an error occurs.
1615         """
1616         if not isfile(local_file):
1617             raise PresentationError(f"The file {local_file} does not exist.")
1618
1619         try:
1620             build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1621         except (IndexError, ValueError):
1622             pass
1623
1624         build = {
1625             u"build": build_nr,
1626             u"status": u"failed",
1627             u"file-name": local_file
1628         }
1629         if replace:
1630             self._cfg.builds = dict()
1631         self._cfg.add_build(job, build)
1632
1633         logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1634         data = self._parse_tests(job, build)
1635         if data is None:
1636             raise PresentationError(
1637                 f"Error occurred while parsing the file {local_file}"
1638             )
1639
1640         build_data = pd.Series({
1641             u"metadata": pd.Series(
1642                 list(data[u"metadata"].values()),
1643                 index=list(data[u"metadata"].keys())
1644             ),
1645             u"suites": pd.Series(
1646                 list(data[u"suites"].values()),
1647                 index=list(data[u"suites"].keys())
1648             ),
1649             u"tests": pd.Series(
1650                 list(data[u"tests"].values()),
1651                 index=list(data[u"tests"].keys())
1652             )
1653         })
1654
1655         if self._input_data.get(job, None) is None:
1656             self._input_data[job] = pd.Series()
1657         self._input_data[job][str(build_nr)] = build_data
1658
1659         self._cfg.set_input_state(job, build_nr, u"processed")
1660
1661     def process_local_directory(self, local_dir, replace=True):
1662         """Process local directory with XML file(s). The directory is processed
1663         as a 'job' and the XML files in it as builds.
1664         If the given directory contains only sub-directories, these
1665         sub-directories processed as jobs and corresponding XML files as builds
1666         of their job.
1667
1668         :param local_dir: Local directory to process.
1669         :param replace: If True, the information about jobs and builds is
1670             replaced by the new one, otherwise the new jobs and builds are
1671             added.
1672         :type local_dir: str
1673         :type replace: bool
1674         """
1675         if not isdir(local_dir):
1676             raise PresentationError(
1677                 f"The directory {local_dir} does not exist."
1678             )
1679
1680         # Check if the given directory includes only files, or only directories
1681         _, dirnames, filenames = next(walk(local_dir))
1682
1683         if filenames and not dirnames:
1684             filenames.sort()
1685             # local_builds:
1686             # key: dir (job) name, value: list of file names (builds)
1687             local_builds = {
1688                 local_dir: [join(local_dir, name) for name in filenames]
1689             }
1690
1691         elif dirnames and not filenames:
1692             dirnames.sort()
1693             # local_builds:
1694             # key: dir (job) name, value: list of file names (builds)
1695             local_builds = dict()
1696             for dirname in dirnames:
1697                 builds = [
1698                     join(local_dir, dirname, name)
1699                     for name in listdir(join(local_dir, dirname))
1700                     if isfile(join(local_dir, dirname, name))
1701                 ]
1702                 if builds:
1703                     local_builds[dirname] = sorted(builds)
1704
1705         elif not filenames and not dirnames:
1706             raise PresentationError(f"The directory {local_dir} is empty.")
1707         else:
1708             raise PresentationError(
1709                 f"The directory {local_dir} can include only files or only "
1710                 f"directories, not both.\nThe directory {local_dir} includes "
1711                 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1712             )
1713
1714         if replace:
1715             self._cfg.builds = dict()
1716
1717         for job, files in local_builds.items():
1718             for idx, local_file in enumerate(files):
1719                 self.process_local_file(local_file, job, idx + 1, replace=False)
1720
1721     @staticmethod
1722     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1723         """Return the index of character in the string which is the end of tag.
1724
1725         :param tag_filter: The string where the end of tag is being searched.
1726         :param start: The index where the searching is stated.
1727         :param closer: The character which is the tag closer.
1728         :type tag_filter: str
1729         :type start: int
1730         :type closer: str
1731         :returns: The index of the tag closer.
1732         :rtype: int
1733         """
1734         try:
1735             idx_opener = tag_filter.index(closer, start)
1736             return tag_filter.index(closer, idx_opener + 1)
1737         except ValueError:
1738             return None
1739
1740     @staticmethod
1741     def _condition(tag_filter):
1742         """Create a conditional statement from the given tag filter.
1743
1744         :param tag_filter: Filter based on tags from the element specification.
1745         :type tag_filter: str
1746         :returns: Conditional statement which can be evaluated.
1747         :rtype: str
1748         """
1749         index = 0
1750         while True:
1751             index = InputData._end_of_tag(tag_filter, index)
1752             if index is None:
1753                 return tag_filter
1754             index += 1
1755             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1756
1757     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1758                     continue_on_error=False):
1759         """Filter required data from the given jobs and builds.
1760
1761         The output data structure is:
1762         - job 1
1763           - build 1
1764             - test (or suite) 1 ID:
1765               - param 1
1766               - param 2
1767               ...
1768               - param n
1769             ...
1770             - test (or suite) n ID:
1771             ...
1772           ...
1773           - build n
1774         ...
1775         - job n
1776
1777         :param element: Element which will use the filtered data.
1778         :param params: Parameters which will be included in the output. If None,
1779             all parameters are included.
1780         :param data: If not None, this data is used instead of data specified
1781             in the element.
1782         :param data_set: The set of data to be filtered: tests, suites,
1783             metadata.
1784         :param continue_on_error: Continue if there is error while reading the
1785             data. The Item will be empty then
1786         :type element: pandas.Series
1787         :type params: list
1788         :type data: dict
1789         :type data_set: str
1790         :type continue_on_error: bool
1791         :returns: Filtered data.
1792         :rtype pandas.Series
1793         """
1794
1795         try:
1796             if data_set == "suites":
1797                 cond = u"True"
1798             elif element[u"filter"] in (u"all", u"template"):
1799                 cond = u"True"
1800             else:
1801                 cond = InputData._condition(element[u"filter"])
1802             logging.debug(f"   Filter: {cond}")
1803         except KeyError:
1804             logging.error(u"  No filter defined.")
1805             return None
1806
1807         if params is None:
1808             params = element.get(u"parameters", None)
1809             if params:
1810                 params.append(u"type")
1811
1812         data_to_filter = data if data else element[u"data"]
1813         data = pd.Series()
1814         try:
1815             for job, builds in data_to_filter.items():
1816                 data[job] = pd.Series()
1817                 for build in builds:
1818                     data[job][str(build)] = pd.Series()
1819                     try:
1820                         data_dict = dict(
1821                             self.data[job][str(build)][data_set].items())
1822                     except KeyError:
1823                         if continue_on_error:
1824                             continue
1825                         return None
1826
1827                     for test_id, test_data in data_dict.items():
1828                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1829                             data[job][str(build)][test_id] = pd.Series()
1830                             if params is None:
1831                                 for param, val in test_data.items():
1832                                     data[job][str(build)][test_id][param] = val
1833                             else:
1834                                 for param in params:
1835                                     try:
1836                                         data[job][str(build)][test_id][param] =\
1837                                             test_data[param]
1838                                     except KeyError:
1839                                         data[job][str(build)][test_id][param] =\
1840                                             u"No Data"
1841             return data
1842
1843         except (KeyError, IndexError, ValueError) as err:
1844             logging.error(
1845                 f"Missing mandatory parameter in the element specification: "
1846                 f"{repr(err)}"
1847             )
1848             return None
1849         except AttributeError as err:
1850             logging.error(repr(err))
1851             return None
1852         except SyntaxError as err:
1853             logging.error(
1854                 f"The filter {cond} is not correct. Check if all tags are "
1855                 f"enclosed by apostrophes.\n{repr(err)}"
1856             )
1857             return None
1858
1859     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1860                              continue_on_error=False):
1861         """Filter required data from the given jobs and builds.
1862
1863         The output data structure is:
1864         - job 1
1865           - build 1
1866             - test (or suite) 1 ID:
1867               - param 1
1868               - param 2
1869               ...
1870               - param n
1871             ...
1872             - test (or suite) n ID:
1873             ...
1874           ...
1875           - build n
1876         ...
1877         - job n
1878
1879         :param element: Element which will use the filtered data.
1880         :param params: Parameters which will be included in the output. If None,
1881         all parameters are included.
1882         :param data_set: The set of data to be filtered: tests, suites,
1883         metadata.
1884         :param continue_on_error: Continue if there is error while reading the
1885         data. The Item will be empty then
1886         :type element: pandas.Series
1887         :type params: list
1888         :type data_set: str
1889         :type continue_on_error: bool
1890         :returns: Filtered data.
1891         :rtype pandas.Series
1892         """
1893
1894         include = element.get(u"include", None)
1895         if not include:
1896             logging.warning(u"No tests to include, skipping the element.")
1897             return None
1898
1899         if params is None:
1900             params = element.get(u"parameters", None)
1901             if params:
1902                 params.append(u"type")
1903
1904         data = pd.Series()
1905         try:
1906             for job, builds in element[u"data"].items():
1907                 data[job] = pd.Series()
1908                 for build in builds:
1909                     data[job][str(build)] = pd.Series()
1910                     for test in include:
1911                         try:
1912                             reg_ex = re.compile(str(test).lower())
1913                             for test_id in self.data[job][
1914                                     str(build)][data_set].keys():
1915                                 if re.match(reg_ex, str(test_id).lower()):
1916                                     test_data = self.data[job][
1917                                         str(build)][data_set][test_id]
1918                                     data[job][str(build)][test_id] = pd.Series()
1919                                     if params is None:
1920                                         for param, val in test_data.items():
1921                                             data[job][str(build)][test_id]\
1922                                                 [param] = val
1923                                     else:
1924                                         for param in params:
1925                                             try:
1926                                                 data[job][str(build)][
1927                                                     test_id][param] = \
1928                                                     test_data[param]
1929                                             except KeyError:
1930                                                 data[job][str(build)][
1931                                                     test_id][param] = u"No Data"
1932                         except KeyError as err:
1933                             if continue_on_error:
1934                                 logging.debug(repr(err))
1935                                 continue
1936                             logging.error(repr(err))
1937                             return None
1938             return data
1939
1940         except (KeyError, IndexError, ValueError) as err:
1941             logging.error(
1942                 f"Missing mandatory parameter in the element "
1943                 f"specification: {repr(err)}"
1944             )
1945             return None
1946         except AttributeError as err:
1947             logging.error(repr(err))
1948             return None
1949
1950     @staticmethod
1951     def merge_data(data):
1952         """Merge data from more jobs and builds to a simple data structure.
1953
1954         The output data structure is:
1955
1956         - test (suite) 1 ID:
1957           - param 1
1958           - param 2
1959           ...
1960           - param n
1961         ...
1962         - test (suite) n ID:
1963         ...
1964
1965         :param data: Data to merge.
1966         :type data: pandas.Series
1967         :returns: Merged data.
1968         :rtype: pandas.Series
1969         """
1970
1971         logging.info(u"    Merging data ...")
1972
1973         merged_data = pd.Series()
1974         for builds in data.values:
1975             for item in builds.values:
1976                 for item_id, item_data in item.items():
1977                     merged_data[item_id] = item_data
1978         return merged_data
1979
1980     def print_all_oper_data(self):
1981         """Print all operational data to console.
1982         """
1983
1984         tbl_hdr = (
1985             u"Name",
1986             u"Nr of Vectors",
1987             u"Nr of Packets",
1988             u"Suspends",
1989             u"Cycles per Packet",
1990             u"Average Vector Size"
1991         )
1992
1993         for job in self._input_data.values:
1994             for build in job.values:
1995                 for test_id, test_data in build[u"tests"].items():
1996                     print(f"{test_id}")
1997                     if test_data.get(u"show-run", None) is None:
1998                         continue
1999                     for dut_name, data in test_data[u"show-run"].items():
2000                         if data.get(u"threads", None) is None:
2001                             continue
2002                         print(f"Host IP: {data.get(u'host', '')}, "
2003                               f"Socket: {data.get(u'socket', '')}")
2004                         for thread_nr, thread in data[u"threads"].items():
2005                             txt_table = prettytable.PrettyTable(tbl_hdr)
2006                             avg = 0.0
2007                             for row in thread:
2008                                 txt_table.add_row(row)
2009                                 avg += row[-1]
2010                             if len(thread) == 0:
2011                                 avg = u""
2012                             else:
2013                                 avg = f", Average Vector Size per Node: " \
2014                                       f"{(avg / len(thread)):.2f}"
2015                             th_name = u"main" if thread_nr == 0 \
2016                                 else f"worker_{thread_nr}"
2017                             print(f"{dut_name}, {th_name}{avg}")
2018                             txt_table.float_format = u".2"
2019                             txt_table.align = u"r"
2020                             txt_table.align[u"Name"] = u"l"
2021                             print(f"{txt_table.get_string()}\n")