resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove, walk, listdir
  29 from os.path import isfile, isdir, join
  30 from datetime import datetime as dt
  31 from datetime import timedelta
  32 from json import loads
  33 from json.decoder import JSONDecodeError
  34
  35 import hdrh.histogram
  36 import hdrh.codec
  37 import prettytable
  38 import pandas as pd
  39
  40 from robot.api import ExecutionResult, ResultVisitor
  41 from robot import errors
  42
  43 from resources.libraries.python import jumpavg
  44 from input_data_files import download_and_unzip_data_file
  45 from pal_errors import PresentationError
  46
  47
  48 # Separator used in file names
  49 SEPARATOR = u"__"
  50
  51
  52 class ExecutionChecker(ResultVisitor):
  53     """Class to traverse through the test suite structure.
  54
  55     The functionality implemented in this class generates a json structure:
  56
  57     Performance tests:
  58
  59     {
  60         "metadata": {
  61             "generated": "Timestamp",
  62             "version": "SUT version",
  63             "job": "Jenkins job name",
  64             "build": "Information about the build"
  65         },
  66         "suites": {
  67             "Suite long name 1": {
  68                 "name": Suite name,
  69                 "doc": "Suite 1 documentation",
  70                 "parent": "Suite 1 parent",
  71                 "level": "Level of the suite in the suite hierarchy"
  72             }
  73             "Suite long name N": {
  74                 "name": Suite name,
  75                 "doc": "Suite N documentation",
  76                 "parent": "Suite 2 parent",
  77                 "level": "Level of the suite in the suite hierarchy"
  78             }
  79         }
  80         "tests": {
  81             # NDRPDR tests:
  82             "ID": {
  83                 "name": "Test name",
  84                 "parent": "Name of the parent of the test",
  85                 "doc": "Test documentation",
  86                 "msg": "Test message",
  87                 "conf-history": "DUT1 and DUT2 VAT History",
  88                 "show-run": "Show Run",
  89                 "tags": ["tag 1", "tag 2", "tag n"],
  90                 "type": "NDRPDR",
  91                 "status": "PASS" | "FAIL",
  92                 "throughput": {
  93                     "NDR": {
  94                         "LOWER": float,
  95                         "UPPER": float
  96                     },
  97                     "PDR": {
  98                         "LOWER": float,
  99                         "UPPER": float
 100                     }
 101                 },
 102                 "latency": {
 103                     "NDR": {
 104                         "direction1": {
 105                             "min": float,
 106                             "avg": float,
 107                             "max": float,
 108                             "hdrh": str
 109                         },
 110                         "direction2": {
 111                             "min": float,
 112                             "avg": float,
 113                             "max": float,
 114                             "hdrh": str
 115                         }
 116                     },
 117                     "PDR": {
 118                         "direction1": {
 119                             "min": float,
 120                             "avg": float,
 121                             "max": float,
 122                             "hdrh": str
 123                         },
 124                         "direction2": {
 125                             "min": float,
 126                             "avg": float,
 127                             "max": float,
 128                             "hdrh": str
 129                         }
 130                     }
 131                 }
 132             }
 133
 134             # TCP tests:
 135             "ID": {
 136                 "name": "Test name",
 137                 "parent": "Name of the parent of the test",
 138                 "doc": "Test documentation",
 139                 "msg": "Test message",
 140                 "tags": ["tag 1", "tag 2", "tag n"],
 141                 "type": "TCP",
 142                 "status": "PASS" | "FAIL",
 143                 "result": int
 144             }
 145
 146             # MRR, BMRR tests:
 147             "ID": {
 148                 "name": "Test name",
 149                 "parent": "Name of the parent of the test",
 150                 "doc": "Test documentation",
 151                 "msg": "Test message",
 152                 "tags": ["tag 1", "tag 2", "tag n"],
 153                 "type": "MRR" | "BMRR",
 154                 "status": "PASS" | "FAIL",
 155                 "result": {
 156                     "receive-rate": float,
 157                     # Average of a list, computed using AvgStdevStats.
 158                     # In CSIT-1180, replace with List[float].
 159                 }
 160             }
 161
 162             "ID" {
 163                 # next test
 164             }
 165         }
 166     }
 167
 168
 169     Functional tests:
 170
 171     {
 172         "metadata": {  # Optional
 173             "version": "VPP version",
 174             "job": "Jenkins job name",
 175             "build": "Information about the build"
 176         },
 177         "suites": {
 178             "Suite name 1": {
 179                 "doc": "Suite 1 documentation",
 180                 "parent": "Suite 1 parent",
 181                 "level": "Level of the suite in the suite hierarchy"
 182             }
 183             "Suite name N": {
 184                 "doc": "Suite N documentation",
 185                 "parent": "Suite 2 parent",
 186                 "level": "Level of the suite in the suite hierarchy"
 187             }
 188         }
 189         "tests": {
 190             "ID": {
 191                 "name": "Test name",
 192                 "parent": "Name of the parent of the test",
 193                 "doc": "Test documentation"
 194                 "msg": "Test message"
 195                 "tags": ["tag 1", "tag 2", "tag n"],
 196                 "conf-history": "DUT1 and DUT2 VAT History"
 197                 "show-run": "Show Run"
 198                 "status": "PASS" | "FAIL"
 199             },
 200             "ID" {
 201                 # next test
 202             }
 203         }
 204     }
 205
 206     .. note:: ID is the lowercase full path to the test.
 207     """
 208
 209     REGEX_PLR_RATE = re.compile(
 210         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 211         r'PLRsearch upper bound::?\s(\d+.\d+)'
 212     )
 213     REGEX_NDRPDR_RATE = re.compile(
 214         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 215         r'NDR_UPPER:\s(\d+.\d+).*\n'
 216         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 217         r'PDR_UPPER:\s(\d+.\d+)'
 218     )
 219     REGEX_NDRPDR_GBPS = re.compile(
 220         r'NDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
 221         r'NDR_UPPER:.*,\s(\d+.\d+).*\n'
 222         r'PDR_LOWER:.*,\s(\d+.\d+).*\n.*\n'
 223         r'PDR_UPPER:.*,\s(\d+.\d+)'
 224     )
 225     REGEX_PERF_MSG_INFO = re.compile(
 226         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 227         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 228         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 229         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 230         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 231     )
 232     REGEX_CPS_MSG_INFO = re.compile(
 233         r'NDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*\n'
 234         r'PDR_LOWER:\s(\d+.\d+)\s.*\s.*\n.*\n.*'
 235     )
 236     REGEX_PPS_MSG_INFO = re.compile(
 237         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 238         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*'
 239     )
 240     REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
 241
 242     # Needed for CPS and PPS tests
 243     REGEX_NDRPDR_LAT_BASE = re.compile(
 244         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 245         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 246     )
 247     REGEX_NDRPDR_LAT = re.compile(
 248         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 249         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 250         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 251         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 252         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 253         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 254     )
 255
 256     REGEX_VERSION_VPP = re.compile(
 257         r"(return STDOUT Version:\s*|"
 258         r"VPP Version:\s*|VPP version:\s*)(.*)"
 259     )
 260     REGEX_VERSION_DPDK = re.compile(
 261         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 262     )
 263     REGEX_TCP = re.compile(
 264         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 265     )
 266     REGEX_MRR = re.compile(
 267         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 268         r'tx\s(\d*),\srx\s(\d*)'
 269     )
 270     REGEX_BMRR = re.compile(
 271         r'.*trial results.*: \[(.*)\]'
 272     )
 273     REGEX_RECONF_LOSS = re.compile(
 274         r'Packets lost due to reconfig: (\d*)'
 275     )
 276     REGEX_RECONF_TIME = re.compile(
 277         r'Implied time lost: (\d*.[\de-]*)'
 278     )
 279     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 280
 281     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 282
 283     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 284
 285     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 286
 287     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 288
 289     def __init__(self, metadata, mapping, ignore):
 290         """Initialisation.
 291
 292         :param metadata: Key-value pairs to be included in "metadata" part of
 293             JSON structure.
 294         :param mapping: Mapping of the old names of test cases to the new
 295             (actual) one.
 296         :param ignore: List of TCs to be ignored.
 297         :type metadata: dict
 298         :type mapping: dict
 299         :type ignore: list
 300         """
 301
 302         # Type of message to parse out from the test messages
 303         self._msg_type = None
 304
 305         # VPP version
 306         self._version = None
 307
 308         # Timestamp
 309         self._timestamp = None
 310
 311         # Testbed. The testbed is identified by TG node IP address.
 312         self._testbed = None
 313
 314         # Mapping of TCs long names
 315         self._mapping = mapping
 316
 317         # Ignore list
 318         self._ignore = ignore
 319
 320         # Number of PAPI History messages found:
 321         # 0 - no message
 322         # 1 - PAPI History of DUT1
 323         # 2 - PAPI History of DUT2
 324         self._conf_history_lookup_nr = 0
 325
 326         self._sh_run_counter = 0
 327
 328         # Test ID of currently processed test- the lowercase full path to the
 329         # test
 330         self._test_id = None
 331
 332         # The main data structure
 333         self._data = {
 334             u"metadata": OrderedDict(),
 335             u"suites": OrderedDict(),
 336             u"tests": OrderedDict()
 337         }
 338
 339         # Save the provided metadata
 340         for key, val in metadata.items():
 341             self._data[u"metadata"][key] = val
 342
 343         # Dictionary defining the methods used to parse different types of
 344         # messages
 345         self.parse_msg = {
 346             u"timestamp": self._get_timestamp,
 347             u"vpp-version": self._get_vpp_version,
 348             u"dpdk-version": self._get_dpdk_version,
 349             u"teardown-papi-history": self._get_papi_history,
 350             u"test-show-runtime": self._get_show_run,
 351             u"testbed": self._get_testbed
 352         }
 353
 354     @property
 355     def data(self):
 356         """Getter - Data parsed from the XML file.
 357
 358         :returns: Data parsed from the XML file.
 359         :rtype: dict
 360         """
 361         return self._data
 362
 363     def _get_data_from_mrr_test_msg(self, msg):
 364         """Get info from message of MRR performance tests.
 365
 366         :param msg: Message to be processed.
 367         :type msg: str
 368         :returns: Processed message or original message if a problem occurs.
 369         :rtype: str
 370         """
 371
 372         groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
 373         if not groups or groups.lastindex != 1:
 374             return u"Test Failed."
 375
 376         try:
 377             data = groups.group(1).split(u", ")
 378         except (AttributeError, IndexError, ValueError, KeyError):
 379             return u"Test Failed."
 380
 381         out_str = u"["
 382         try:
 383             for item in data:
 384                 out_str += f"{(float(item) / 1e6):.2f}, "
 385             return out_str[:-2] + u"]"
 386         except (AttributeError, IndexError, ValueError, KeyError):
 387             return u"Test Failed."
 388
 389     def _get_data_from_cps_test_msg(self, msg):
 390         """Get info from message of NDRPDR CPS tests.
 391
 392         :param msg: Message to be processed.
 393         :type msg: str
 394         :returns: Processed message or "Test Failed." if a problem occurs.
 395         :rtype: str
 396         """
 397
 398         groups = re.search(self.REGEX_CPS_MSG_INFO, msg)
 399         if not groups or groups.lastindex != 2:
 400             return u"Test Failed."
 401
 402         try:
 403             return (
 404                 f"1. {(float(groups.group(1)) / 1e6):5.2f}\n"
 405                 f"2. {(float(groups.group(2)) / 1e6):5.2f}"
 406             )
 407         except (AttributeError, IndexError, ValueError, KeyError):
 408             return u"Test Failed."
 409
 410     def _get_data_from_pps_test_msg(self, msg):
 411         """Get info from message of NDRPDR PPS tests.
 412
 413         :param msg: Message to be processed.
 414         :type msg: str
 415         :returns: Processed message or "Test Failed." if a problem occurs.
 416         :rtype: str
 417         """
 418
 419         groups = re.search(self.REGEX_PPS_MSG_INFO, msg)
 420         if not groups or groups.lastindex != 4:
 421             return u"Test Failed."
 422
 423         try:
 424             return (
 425                 f"1. {(float(groups.group(1)) / 1e6):5.2f}      "
 426                 f"{float(groups.group(2)):5.2f}\n"
 427                 f"2. {(float(groups.group(3)) / 1e6):5.2f}      "
 428                 f"{float(groups.group(4)):5.2f}"
 429             )
 430         except (AttributeError, IndexError, ValueError, KeyError):
 431             return u"Test Failed."
 432
 433     def _get_data_from_perf_test_msg(self, msg):
 434         """Get info from message of NDRPDR performance tests.
 435
 436         :param msg: Message to be processed.
 437         :type msg: str
 438         :returns: Processed message or "Test Failed." if a problem occurs.
 439         :rtype: str
 440         """
 441
 442         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 443         if not groups or groups.lastindex != 10:
 444             return u"Test Failed."
 445
 446         try:
 447             data = {
 448                 u"ndr_low": float(groups.group(1)),
 449                 u"ndr_low_b": float(groups.group(2)),
 450                 u"pdr_low": float(groups.group(3)),
 451                 u"pdr_low_b": float(groups.group(4)),
 452                 u"pdr_lat_90_1": groups.group(5),
 453                 u"pdr_lat_90_2": groups.group(6),
 454                 u"pdr_lat_50_1": groups.group(7),
 455                 u"pdr_lat_50_2": groups.group(8),
 456                 u"pdr_lat_10_1": groups.group(9),
 457                 u"pdr_lat_10_2": groups.group(10),
 458             }
 459         except (AttributeError, IndexError, ValueError, KeyError):
 460             return u"Test Failed."
 461
 462         def _process_lat(in_str_1, in_str_2):
 463             """Extract min, avg, max values from latency string.
 464
 465             :param in_str_1: Latency string for one direction produced by robot
 466                 framework.
 467             :param in_str_2: Latency string for second direction produced by
 468                 robot framework.
 469             :type in_str_1: str
 470             :type in_str_2: str
 471             :returns: Processed latency string or None if a problem occurs.
 472             :rtype: tuple
 473             """
 474             in_list_1 = in_str_1.split('/', 3)
 475             in_list_2 = in_str_2.split('/', 3)
 476
 477             if len(in_list_1) != 4 and len(in_list_2) != 4:
 478                 return None
 479
 480             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 481             try:
 482                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 483             except hdrh.codec.HdrLengthException:
 484                 return None
 485
 486             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 487             try:
 488                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 489             except hdrh.codec.HdrLengthException:
 490                 return None
 491
 492             if hdr_lat_1 and hdr_lat_2:
 493                 hdr_lat = (
 494                     hdr_lat_1.get_value_at_percentile(50.0),
 495                     hdr_lat_1.get_value_at_percentile(90.0),
 496                     hdr_lat_1.get_value_at_percentile(99.0),
 497                     hdr_lat_2.get_value_at_percentile(50.0),
 498                     hdr_lat_2.get_value_at_percentile(90.0),
 499                     hdr_lat_2.get_value_at_percentile(99.0)
 500                 )
 501
 502                 if all(hdr_lat):
 503                     return hdr_lat
 504
 505             return None
 506
 507         try:
 508             out_msg = (
 509                 f"1. {(data[u'ndr_low'] / 1e6):5.2f}      "
 510                 f"{data[u'ndr_low_b']:5.2f}"
 511                 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f}      "
 512                 f"{data[u'pdr_low_b']:5.2f}"
 513             )
 514             latency = (
 515                 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
 516                 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
 517                 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
 518             )
 519             if all(latency):
 520                 max_len = len(str(max((max(item) for item in latency))))
 521                 max_len = 4 if max_len < 4 else max_len
 522
 523                 for idx, lat in enumerate(latency):
 524                     if not idx:
 525                         out_msg += u"\n"
 526                     out_msg += (
 527                         f"\n{idx + 3}. "
 528                         f"{lat[0]:{max_len}d} "
 529                         f"{lat[1]:{max_len}d} "
 530                         f"{lat[2]:{max_len}d}      "
 531                         f"{lat[3]:{max_len}d} "
 532                         f"{lat[4]:{max_len}d} "
 533                         f"{lat[5]:{max_len}d} "
 534                     )
 535
 536             return out_msg
 537
 538         except (AttributeError, IndexError, ValueError, KeyError):
 539             return u"Test Failed."
 540
 541     def _get_testbed(self, msg):
 542         """Called when extraction of testbed IP is required.
 543         The testbed is identified by TG node IP address.
 544
 545         :param msg: Message to process.
 546         :type msg: Message
 547         :returns: Nothing.
 548         """
 549
 550         if msg.message.count(u"Setup of TG node") or \
 551                 msg.message.count(u"Setup of node TG host"):
 552             reg_tg_ip = re.compile(
 553                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 554             try:
 555                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 556             except (KeyError, ValueError, IndexError, AttributeError):
 557                 pass
 558             finally:
 559                 self._data[u"metadata"][u"testbed"] = self._testbed
 560                 self._msg_type = None
 561
 562     def _get_vpp_version(self, msg):
 563         """Called when extraction of VPP version is required.
 564
 565         :param msg: Message to process.
 566         :type msg: Message
 567         :returns: Nothing.
 568         """
 569
 570         if msg.message.count(u"return STDOUT Version:") or \
 571                 msg.message.count(u"VPP Version:") or \
 572                 msg.message.count(u"VPP version:"):
 573             self._version = str(
 574                 re.search(self.REGEX_VERSION_VPP, msg.message).group(2)
 575             )
 576             self._data[u"metadata"][u"version"] = self._version
 577             self._msg_type = None
 578
 579     def _get_dpdk_version(self, msg):
 580         """Called when extraction of DPDK version is required.
 581
 582         :param msg: Message to process.
 583         :type msg: Message
 584         :returns: Nothing.
 585         """
 586
 587         if msg.message.count(u"DPDK Version:"):
 588             try:
 589                 self._version = str(re.search(
 590                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 591                 self._data[u"metadata"][u"version"] = self._version
 592             except IndexError:
 593                 pass
 594             finally:
 595                 self._msg_type = None
 596
 597     def _get_timestamp(self, msg):
 598         """Called when extraction of timestamp is required.
 599
 600         :param msg: Message to process.
 601         :type msg: Message
 602         :returns: Nothing.
 603         """
 604
 605         self._timestamp = msg.timestamp[:14]
 606         self._data[u"metadata"][u"generated"] = self._timestamp
 607         self._msg_type = None
 608
 609     def _get_papi_history(self, msg):
 610         """Called when extraction of PAPI command history is required.
 611
 612         :param msg: Message to process.
 613         :type msg: Message
 614         :returns: Nothing.
 615         """
 616         if msg.message.count(u"PAPI command history:"):
 617             self._conf_history_lookup_nr += 1
 618             if self._conf_history_lookup_nr == 1:
 619                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 620             else:
 621                 self._msg_type = None
 622             text = re.sub(
 623                 r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} PAPI command history:",
 624                 u"",
 625                 msg.message,
 626                 count=1
 627             ).replace(u'"', u"'")
 628             self._data[u"tests"][self._test_id][u"conf-history"] += (
 629                 f"**DUT{str(self._conf_history_lookup_nr)}:** {text}"
 630             )
 631
 632     def _get_show_run(self, msg):
 633         """Called when extraction of VPP operational data (output of CLI command
 634         Show Runtime) is required.
 635
 636         :param msg: Message to process.
 637         :type msg: Message
 638         :returns: Nothing.
 639         """
 640
 641         if not msg.message.count(u"stats runtime"):
 642             return
 643
 644         # Temporary solution
 645         if self._sh_run_counter > 1:
 646             return
 647
 648         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 649             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 650
 651         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 652         if not groups:
 653             return
 654         try:
 655             host = groups.group(1)
 656         except (AttributeError, IndexError):
 657             host = u""
 658         try:
 659             sock = groups.group(2)
 660         except (AttributeError, IndexError):
 661             sock = u""
 662
 663         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 664                         replace(u"'", u'"').replace(u'b"', u'"').
 665                         replace(u'u"', u'"').split(u":", 1)[1])
 666
 667         try:
 668             threads_nr = len(runtime[0][u"clocks"])
 669         except (IndexError, KeyError):
 670             return
 671
 672         dut = u"dut{nr}".format(
 673             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 674
 675         oper = {
 676             u"host": host,
 677             u"socket": sock,
 678             u"runtime": runtime,
 679             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 680         }
 681
 682         for item in runtime:
 683             for idx in range(threads_nr):
 684                 if item[u"vectors"][idx] > 0:
 685                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 686                 elif item[u"calls"][idx] > 0:
 687                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 688                 elif item[u"suspends"][idx] > 0:
 689                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 690                 else:
 691                     clocks = 0.0
 692
 693                 if item[u"calls"][idx] > 0:
 694                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 695                 else:
 696                     vectors_call = 0.0
 697
 698                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 699                         int(item[u"suspends"][idx]):
 700                     oper[u"threads"][idx].append([
 701                         item[u"name"],
 702                         item[u"calls"][idx],
 703                         item[u"vectors"][idx],
 704                         item[u"suspends"][idx],
 705                         clocks,
 706                         vectors_call
 707                     ])
 708
 709         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 710
 711     def _get_ndrpdr_throughput(self, msg):
 712         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 713         message.
 714
 715         :param msg: The test message to be parsed.
 716         :type msg: str
 717         :returns: Parsed data as a dict and the status (PASS/FAIL).
 718         :rtype: tuple(dict, str)
 719         """
 720
 721         throughput = {
 722             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 723             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 724         }
 725         status = u"FAIL"
 726         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 727
 728         if groups is not None:
 729             try:
 730                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 731                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 732                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 733                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 734                 status = u"PASS"
 735             except (IndexError, ValueError):
 736                 pass
 737
 738         return throughput, status
 739
 740     def _get_ndrpdr_throughput_gbps(self, msg):
 741         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER in Gbps from the
 742         test message.
 743
 744         :param msg: The test message to be parsed.
 745         :type msg: str
 746         :returns: Parsed data as a dict and the status (PASS/FAIL).
 747         :rtype: tuple(dict, str)
 748         """
 749
 750         gbps = {
 751             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 752             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 753         }
 754         status = u"FAIL"
 755         groups = re.search(self.REGEX_NDRPDR_GBPS, msg)
 756
 757         if groups is not None:
 758             try:
 759                 gbps[u"NDR"][u"LOWER"] = float(groups.group(1))
 760                 gbps[u"NDR"][u"UPPER"] = float(groups.group(2))
 761                 gbps[u"PDR"][u"LOWER"] = float(groups.group(3))
 762                 gbps[u"PDR"][u"UPPER"] = float(groups.group(4))
 763                 status = u"PASS"
 764             except (IndexError, ValueError):
 765                 pass
 766
 767         return gbps, status
 768
 769     def _get_plr_throughput(self, msg):
 770         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 771         message.
 772
 773         :param msg: The test message to be parsed.
 774         :type msg: str
 775         :returns: Parsed data as a dict and the status (PASS/FAIL).
 776         :rtype: tuple(dict, str)
 777         """
 778
 779         throughput = {
 780             u"LOWER": -1.0,
 781             u"UPPER": -1.0
 782         }
 783         status = u"FAIL"
 784         groups = re.search(self.REGEX_PLR_RATE, msg)
 785
 786         if groups is not None:
 787             try:
 788                 throughput[u"LOWER"] = float(groups.group(1))
 789                 throughput[u"UPPER"] = float(groups.group(2))
 790                 status = u"PASS"
 791             except (IndexError, ValueError):
 792                 pass
 793
 794         return throughput, status
 795
 796     def _get_ndrpdr_latency(self, msg):
 797         """Get LATENCY from the test message.
 798
 799         :param msg: The test message to be parsed.
 800         :type msg: str
 801         :returns: Parsed data as a dict and the status (PASS/FAIL).
 802         :rtype: tuple(dict, str)
 803         """
 804         latency_default = {
 805             u"min": -1.0,
 806             u"avg": -1.0,
 807             u"max": -1.0,
 808             u"hdrh": u""
 809         }
 810         latency = {
 811             u"NDR": {
 812                 u"direction1": copy.copy(latency_default),
 813                 u"direction2": copy.copy(latency_default)
 814             },
 815             u"PDR": {
 816                 u"direction1": copy.copy(latency_default),
 817                 u"direction2": copy.copy(latency_default)
 818             },
 819             u"LAT0": {
 820                 u"direction1": copy.copy(latency_default),
 821                 u"direction2": copy.copy(latency_default)
 822             },
 823             u"PDR10": {
 824                 u"direction1": copy.copy(latency_default),
 825                 u"direction2": copy.copy(latency_default)
 826             },
 827             u"PDR50": {
 828                 u"direction1": copy.copy(latency_default),
 829                 u"direction2": copy.copy(latency_default)
 830             },
 831             u"PDR90": {
 832                 u"direction1": copy.copy(latency_default),
 833                 u"direction2": copy.copy(latency_default)
 834             },
 835         }
 836
 837         groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 838         if groups is None:
 839             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 840         if groups is None:
 841             return latency, u"FAIL"
 842
 843         def process_latency(in_str):
 844             """Return object with parsed latency values.
 845
 846             TODO: Define class for the return type.
 847
 848             :param in_str: Input string, min/avg/max/hdrh format.
 849             :type in_str: str
 850             :returns: Dict with corresponding keys, except hdrh float values.
 851             :rtype dict:
 852             :throws IndexError: If in_str does not have enough substrings.
 853             :throws ValueError: If a substring does not convert to float.
 854             """
 855             in_list = in_str.split('/', 3)
 856
 857             rval = {
 858                 u"min": float(in_list[0]),
 859                 u"avg": float(in_list[1]),
 860                 u"max": float(in_list[2]),
 861                 u"hdrh": u""
 862             }
 863
 864             if len(in_list) == 4:
 865                 rval[u"hdrh"] = str(in_list[3])
 866
 867             return rval
 868
 869         try:
 870             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 871             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 872             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 873             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 874             if groups.lastindex == 4:
 875                 return latency, u"PASS"
 876         except (IndexError, ValueError):
 877             pass
 878
 879         try:
 880             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 881             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 882             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 883             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 884             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 885             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 886             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 887             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 888             if groups.lastindex == 12:
 889                 return latency, u"PASS"
 890         except (IndexError, ValueError):
 891             pass
 892
 893         return latency, u"FAIL"
 894
 895     @staticmethod
 896     def _get_hoststack_data(msg, tags):
 897         """Get data from the hoststack test message.
 898
 899         :param msg: The test message to be parsed.
 900         :param tags: Test tags.
 901         :type msg: str
 902         :type tags: list
 903         :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
 904         :rtype: tuple(dict, str)
 905         """
 906         result = dict()
 907         status = u"FAIL"
 908
 909         msg = msg.replace(u"'", u'"').replace(u" ", u"")
 910         if u"LDPRELOAD" in tags:
 911             try:
 912                 result = loads(msg)
 913                 status = u"PASS"
 914             except JSONDecodeError:
 915                 pass
 916         elif u"VPPECHO" in tags:
 917             try:
 918                 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
 919                 result = dict(
 920                     client=loads(msg_lst[0]),
 921                     server=loads(msg_lst[1])
 922                 )
 923                 status = u"PASS"
 924             except (JSONDecodeError, IndexError):
 925                 pass
 926
 927         return result, status
 928
 929     def visit_suite(self, suite):
 930         """Implements traversing through the suite and its direct children.
 931
 932         :param suite: Suite to process.
 933         :type suite: Suite
 934         :returns: Nothing.
 935         """
 936         if self.start_suite(suite) is not False:
 937             suite.suites.visit(self)
 938             suite.tests.visit(self)
 939             self.end_suite(suite)
 940
 941     def start_suite(self, suite):
 942         """Called when suite starts.
 943
 944         :param suite: Suite to process.
 945         :type suite: Suite
 946         :returns: Nothing.
 947         """
 948
 949         try:
 950             parent_name = suite.parent.name
 951         except AttributeError:
 952             return
 953
 954         self._data[u"suites"][suite.longname.lower().
 955                               replace(u'"', u"'").
 956                               replace(u" ", u"_")] = {
 957                                   u"name": suite.name.lower(),
 958                                   u"doc": suite.doc,
 959                                   u"parent": parent_name,
 960                                   u"level": len(suite.longname.split(u"."))
 961                               }
 962
 963         suite.keywords.visit(self)
 964
 965     def end_suite(self, suite):
 966         """Called when suite ends.
 967
 968         :param suite: Suite to process.
 969         :type suite: Suite
 970         :returns: Nothing.
 971         """
 972
 973     def visit_test(self, test):
 974         """Implements traversing through the test.
 975
 976         :param test: Test to process.
 977         :type test: Test
 978         :returns: Nothing.
 979         """
 980         if self.start_test(test) is not False:
 981             test.keywords.visit(self)
 982             self.end_test(test)
 983
 984     def start_test(self, test):
 985         """Called when test starts.
 986
 987         :param test: Test to process.
 988         :type test: Test
 989         :returns: Nothing.
 990         """
 991
 992         self._sh_run_counter = 0
 993
 994         longname_orig = test.longname.lower()
 995
 996         # Check the ignore list
 997         if longname_orig in self._ignore:
 998             return
 999
1000         tags = [str(tag) for tag in test.tags]
1001         test_result = dict()
1002
1003         # Change the TC long name and name if defined in the mapping table
1004         longname = self._mapping.get(longname_orig, None)
1005         if longname is not None:
1006             name = longname.split(u'.')[-1]
1007             logging.debug(
1008                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
1009                 f"{name}"
1010             )
1011         else:
1012             longname = longname_orig
1013             name = test.name.lower()
1014
1015         # Remove TC number from the TC long name (backward compatibility):
1016         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1017         # Remove TC number from the TC name (not needed):
1018         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1019
1020         test_result[u"parent"] = test.parent.name.lower()
1021         test_result[u"tags"] = tags
1022         test_result["doc"] = test.doc
1023         test_result[u"type"] = u""
1024         test_result[u"status"] = test.status
1025         test_result[u"starttime"] = test.starttime
1026         test_result[u"endtime"] = test.endtime
1027
1028         if test.status == u"PASS":
1029             if u"NDRPDR" in tags:
1030                 if u"TCP_PPS" in tags or u"UDP_PPS" in tags:
1031                     test_result[u"msg"] = self._get_data_from_pps_test_msg(
1032                         test.message)
1033                 elif u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1034                     test_result[u"msg"] = self._get_data_from_cps_test_msg(
1035                         test.message)
1036                 else:
1037                     test_result[u"msg"] = self._get_data_from_perf_test_msg(
1038                         test.message)
1039             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1040                 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1041                     test.message)
1042             else:
1043                 test_result[u"msg"] = test.message
1044         else:
1045             test_result[u"msg"] = test.message
1046
1047         if u"PERFTEST" in tags:
1048             # Replace info about cores (e.g. -1c-) with the info about threads
1049             # and cores (e.g. -1t1c-) in the long test case names and in the
1050             # test case names if necessary.
1051             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1052             if not groups:
1053                 tag_count = 0
1054                 tag_tc = str()
1055                 for tag in test_result[u"tags"]:
1056                     groups = re.search(self.REGEX_TC_TAG, tag)
1057                     if groups:
1058                         tag_count += 1
1059                         tag_tc = tag
1060
1061                 if tag_count == 1:
1062                     self._test_id = re.sub(
1063                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1064                         self._test_id, count=1
1065                     )
1066                     test_result[u"name"] = re.sub(
1067                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1068                         test_result["name"], count=1
1069                     )
1070                 else:
1071                     test_result[u"status"] = u"FAIL"
1072                     self._data[u"tests"][self._test_id] = test_result
1073                     logging.debug(
1074                         f"The test {self._test_id} has no or more than one "
1075                         f"multi-threading tags.\n"
1076                         f"Tags: {test_result[u'tags']}"
1077                     )
1078                     return
1079
1080         if u"DEVICETEST" in tags:
1081             test_result[u"type"] = u"DEVICETEST"
1082         elif u"NDRPDR" in tags:
1083             if u"TCP_CPS" in tags or u"UDP_CPS" in tags:
1084                 test_result[u"type"] = u"CPS"
1085             else:
1086                 test_result[u"type"] = u"NDRPDR"
1087             if test.status == u"PASS":
1088                 test_result[u"throughput"], test_result[u"status"] = \
1089                     self._get_ndrpdr_throughput(test.message)
1090                 test_result[u"gbps"], test_result[u"status"] = \
1091                     self._get_ndrpdr_throughput_gbps(test.message)
1092                 test_result[u"latency"], test_result[u"status"] = \
1093                     self._get_ndrpdr_latency(test.message)
1094         elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1095             if u"MRR" in tags:
1096                 test_result[u"type"] = u"MRR"
1097             else:
1098                 test_result[u"type"] = u"BMRR"
1099             if test.status == u"PASS":
1100                 test_result[u"result"] = dict()
1101                 groups = re.search(self.REGEX_BMRR, test.message)
1102                 if groups is not None:
1103                     items_str = groups.group(1)
1104                     items_float = [
1105                         float(item.strip().replace(u"'", u""))
1106                         for item in items_str.split(",")
1107                     ]
1108                     # Use whole list in CSIT-1180.
1109                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1110                     test_result[u"result"][u"samples"] = items_float
1111                     test_result[u"result"][u"receive-rate"] = stats.avg
1112                     test_result[u"result"][u"receive-stdev"] = stats.stdev
1113                 else:
1114                     groups = re.search(self.REGEX_MRR, test.message)
1115                     test_result[u"result"][u"receive-rate"] = \
1116                         float(groups.group(3)) / float(groups.group(1))
1117         elif u"SOAK" in tags:
1118             test_result[u"type"] = u"SOAK"
1119             if test.status == u"PASS":
1120                 test_result[u"throughput"], test_result[u"status"] = \
1121                     self._get_plr_throughput(test.message)
1122         elif u"HOSTSTACK" in tags:
1123             test_result[u"type"] = u"HOSTSTACK"
1124             if test.status == u"PASS":
1125                 test_result[u"result"], test_result[u"status"] = \
1126                     self._get_hoststack_data(test.message, tags)
1127         # elif u"TCP" in tags:  # This might be not used
1128         #     test_result[u"type"] = u"TCP"
1129         #     if test.status == u"PASS":
1130         #         groups = re.search(self.REGEX_TCP, test.message)
1131         #         test_result[u"result"] = int(groups.group(2))
1132         elif u"RECONF" in tags:
1133             test_result[u"type"] = u"RECONF"
1134             if test.status == u"PASS":
1135                 test_result[u"result"] = None
1136                 try:
1137                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1138                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1139                     test_result[u"result"] = {
1140                         u"loss": int(grps_loss.group(1)),
1141                         u"time": float(grps_time.group(1))
1142                     }
1143                 except (AttributeError, IndexError, ValueError, TypeError):
1144                     test_result[u"status"] = u"FAIL"
1145         else:
1146             test_result[u"status"] = u"FAIL"
1147
1148         self._data[u"tests"][self._test_id] = test_result
1149
1150     def end_test(self, test):
1151         """Called when test ends.
1152
1153         :param test: Test to process.
1154         :type test: Test
1155         :returns: Nothing.
1156         """
1157
1158     def visit_keyword(self, keyword):
1159         """Implements traversing through the keyword and its child keywords.
1160
1161         :param keyword: Keyword to process.
1162         :type keyword: Keyword
1163         :returns: Nothing.
1164         """
1165         if self.start_keyword(keyword) is not False:
1166             self.end_keyword(keyword)
1167
1168     def start_keyword(self, keyword):
1169         """Called when keyword starts. Default implementation does nothing.
1170
1171         :param keyword: Keyword to process.
1172         :type keyword: Keyword
1173         :returns: Nothing.
1174         """
1175         try:
1176             if keyword.type == u"setup":
1177                 self.visit_setup_kw(keyword)
1178             elif keyword.type == u"teardown":
1179                 self.visit_teardown_kw(keyword)
1180             else:
1181                 self.visit_test_kw(keyword)
1182         except AttributeError:
1183             pass
1184
1185     def end_keyword(self, keyword):
1186         """Called when keyword ends. Default implementation does nothing.
1187
1188         :param keyword: Keyword to process.
1189         :type keyword: Keyword
1190         :returns: Nothing.
1191         """
1192
1193     def visit_test_kw(self, test_kw):
1194         """Implements traversing through the test keyword and its child
1195         keywords.
1196
1197         :param test_kw: Keyword to process.
1198         :type test_kw: Keyword
1199         :returns: Nothing.
1200         """
1201         for keyword in test_kw.keywords:
1202             if self.start_test_kw(keyword) is not False:
1203                 self.visit_test_kw(keyword)
1204                 self.end_test_kw(keyword)
1205
1206     def start_test_kw(self, test_kw):
1207         """Called when test keyword starts. Default implementation does
1208         nothing.
1209
1210         :param test_kw: Keyword to process.
1211         :type test_kw: Keyword
1212         :returns: Nothing.
1213         """
1214         if test_kw.name.count(u"Show Runtime On All Duts") or \
1215                 test_kw.name.count(u"Show Runtime Counters On All Duts") or \
1216                 test_kw.name.count(u"Vpp Show Runtime On All Duts"):
1217             self._msg_type = u"test-show-runtime"
1218             self._sh_run_counter += 1
1219         else:
1220             return
1221         test_kw.messages.visit(self)
1222
1223     def end_test_kw(self, test_kw):
1224         """Called when keyword ends. Default implementation does nothing.
1225
1226         :param test_kw: Keyword to process.
1227         :type test_kw: Keyword
1228         :returns: Nothing.
1229         """
1230
1231     def visit_setup_kw(self, setup_kw):
1232         """Implements traversing through the teardown keyword and its child
1233         keywords.
1234
1235         :param setup_kw: Keyword to process.
1236         :type setup_kw: Keyword
1237         :returns: Nothing.
1238         """
1239         for keyword in setup_kw.keywords:
1240             if self.start_setup_kw(keyword) is not False:
1241                 self.visit_setup_kw(keyword)
1242                 self.end_setup_kw(keyword)
1243
1244     def start_setup_kw(self, setup_kw):
1245         """Called when teardown keyword starts. Default implementation does
1246         nothing.
1247
1248         :param setup_kw: Keyword to process.
1249         :type setup_kw: Keyword
1250         :returns: Nothing.
1251         """
1252         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1253                 and not self._version:
1254             self._msg_type = u"vpp-version"
1255         elif setup_kw.name.count(u"Install Dpdk Framework On All Duts") and \
1256                 not self._version:
1257             self._msg_type = u"dpdk-version"
1258         elif setup_kw.name.count(u"Set Global Variable") \
1259                 and not self._timestamp:
1260             self._msg_type = u"timestamp"
1261         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1262             self._msg_type = u"testbed"
1263         else:
1264             return
1265         setup_kw.messages.visit(self)
1266
1267     def end_setup_kw(self, setup_kw):
1268         """Called when keyword ends. Default implementation does nothing.
1269
1270         :param setup_kw: Keyword to process.
1271         :type setup_kw: Keyword
1272         :returns: Nothing.
1273         """
1274
1275     def visit_teardown_kw(self, teardown_kw):
1276         """Implements traversing through the teardown keyword and its child
1277         keywords.
1278
1279         :param teardown_kw: Keyword to process.
1280         :type teardown_kw: Keyword
1281         :returns: Nothing.
1282         """
1283         for keyword in teardown_kw.keywords:
1284             if self.start_teardown_kw(keyword) is not False:
1285                 self.visit_teardown_kw(keyword)
1286                 self.end_teardown_kw(keyword)
1287
1288     def start_teardown_kw(self, teardown_kw):
1289         """Called when teardown keyword starts
1290
1291         :param teardown_kw: Keyword to process.
1292         :type teardown_kw: Keyword
1293         :returns: Nothing.
1294         """
1295         if teardown_kw.name.count(u"Show Papi History On All Duts"):
1296             self._conf_history_lookup_nr = 0
1297             self._msg_type = u"teardown-papi-history"
1298             teardown_kw.messages.visit(self)
1299
1300     def end_teardown_kw(self, teardown_kw):
1301         """Called when keyword ends. Default implementation does nothing.
1302
1303         :param teardown_kw: Keyword to process.
1304         :type teardown_kw: Keyword
1305         :returns: Nothing.
1306         """
1307
1308     def visit_message(self, msg):
1309         """Implements visiting the message.
1310
1311         :param msg: Message to process.
1312         :type msg: Message
1313         :returns: Nothing.
1314         """
1315         if self.start_message(msg) is not False:
1316             self.end_message(msg)
1317
1318     def start_message(self, msg):
1319         """Called when message starts. Get required information from messages:
1320         - VPP version.
1321
1322         :param msg: Message to process.
1323         :type msg: Message
1324         :returns: Nothing.
1325         """
1326         if self._msg_type:
1327             self.parse_msg[self._msg_type](msg)
1328
1329     def end_message(self, msg):
1330         """Called when message ends. Default implementation does nothing.
1331
1332         :param msg: Message to process.
1333         :type msg: Message
1334         :returns: Nothing.
1335         """
1336
1337
1338 class InputData:
1339     """Input data
1340
1341     The data is extracted from output.xml files generated by Jenkins jobs and
1342     stored in pandas' DataFrames.
1343
1344     The data structure:
1345     - job name
1346       - build number
1347         - metadata
1348           (as described in ExecutionChecker documentation)
1349         - suites
1350           (as described in ExecutionChecker documentation)
1351         - tests
1352           (as described in ExecutionChecker documentation)
1353     """
1354
1355     def __init__(self, spec):
1356         """Initialization.
1357
1358         :param spec: Specification.
1359         :type spec: Specification
1360         """
1361
1362         # Specification:
1363         self._cfg = spec
1364
1365         # Data store:
1366         self._input_data = pd.Series()
1367
1368     @property
1369     def data(self):
1370         """Getter - Input data.
1371
1372         :returns: Input data
1373         :rtype: pandas.Series
1374         """
1375         return self._input_data
1376
1377     def metadata(self, job, build):
1378         """Getter - metadata
1379
1380         :param job: Job which metadata we want.
1381         :param build: Build which metadata we want.
1382         :type job: str
1383         :type build: str
1384         :returns: Metadata
1385         :rtype: pandas.Series
1386         """
1387         return self.data[job][build][u"metadata"]
1388
1389     def suites(self, job, build):
1390         """Getter - suites
1391
1392         :param job: Job which suites we want.
1393         :param build: Build which suites we want.
1394         :type job: str
1395         :type build: str
1396         :returns: Suites.
1397         :rtype: pandas.Series
1398         """
1399         return self.data[job][str(build)][u"suites"]
1400
1401     def tests(self, job, build):
1402         """Getter - tests
1403
1404         :param job: Job which tests we want.
1405         :param build: Build which tests we want.
1406         :type job: str
1407         :type build: str
1408         :returns: Tests.
1409         :rtype: pandas.Series
1410         """
1411         return self.data[job][build][u"tests"]
1412
1413     def _parse_tests(self, job, build):
1414         """Process data from robot output.xml file and return JSON structured
1415         data.
1416
1417         :param job: The name of job which build output data will be processed.
1418         :param build: The build which output data will be processed.
1419         :type job: str
1420         :type build: dict
1421         :returns: JSON data structure.
1422         :rtype: dict
1423         """
1424
1425         metadata = {
1426             u"job": job,
1427             u"build": build
1428         }
1429
1430         with open(build[u"file-name"], u'r') as data_file:
1431             try:
1432                 result = ExecutionResult(data_file)
1433             except errors.DataError as err:
1434                 logging.error(
1435                     f"Error occurred while parsing output.xml: {repr(err)}"
1436                 )
1437                 return None
1438         checker = ExecutionChecker(
1439             metadata, self._cfg.mapping, self._cfg.ignore
1440         )
1441         result.visit(checker)
1442
1443         return checker.data
1444
1445     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1446         """Download and parse the input data file.
1447
1448         :param pid: PID of the process executing this method.
1449         :param job: Name of the Jenkins job which generated the processed input
1450             file.
1451         :param build: Information about the Jenkins build which generated the
1452             processed input file.
1453         :param repeat: Repeat the download specified number of times if not
1454             successful.
1455         :type pid: int
1456         :type job: str
1457         :type build: dict
1458         :type repeat: int
1459         """
1460
1461         logging.info(f"Processing the job/build: {job}: {build[u'build']}")
1462
1463         state = u"failed"
1464         success = False
1465         data = None
1466         do_repeat = repeat
1467         while do_repeat:
1468             success = download_and_unzip_data_file(self._cfg, job, build, pid)
1469             if success:
1470                 break
1471             do_repeat -= 1
1472         if not success:
1473             logging.error(
1474                 f"It is not possible to download the input data file from the "
1475                 f"job {job}, build {build[u'build']}, or it is damaged. "
1476                 f"Skipped."
1477             )
1478         if success:
1479             logging.info(f"  Processing data from build {build[u'build']}")
1480             data = self._parse_tests(job, build)
1481             if data is None:
1482                 logging.error(
1483                     f"Input data file from the job {job}, build "
1484                     f"{build[u'build']} is damaged. Skipped."
1485                 )
1486             else:
1487                 state = u"processed"
1488
1489             try:
1490                 remove(build[u"file-name"])
1491             except OSError as err:
1492                 logging.error(
1493                     f"Cannot remove the file {build[u'file-name']}: {repr(err)}"
1494                 )
1495
1496         # If the time-period is defined in the specification file, remove all
1497         # files which are outside the time period.
1498         is_last = False
1499         timeperiod = self._cfg.environment.get(u"time-period", None)
1500         if timeperiod and data:
1501             now = dt.utcnow()
1502             timeperiod = timedelta(int(timeperiod))
1503             metadata = data.get(u"metadata", None)
1504             if metadata:
1505                 generated = metadata.get(u"generated", None)
1506                 if generated:
1507                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1508                     if (now - generated) > timeperiod:
1509                         # Remove the data and the file:
1510                         state = u"removed"
1511                         data = None
1512                         is_last = True
1513                         logging.info(
1514                             f"  The build {job}/{build[u'build']} is "
1515                             f"outdated, will be removed."
1516                         )
1517         return {
1518             u"data": data,
1519             u"state": state,
1520             u"job": job,
1521             u"build": build,
1522             u"last": is_last
1523         }
1524
1525     def download_and_parse_data(self, repeat=1):
1526         """Download the input data files, parse input data from input files and
1527         store in pandas' Series.
1528
1529         :param repeat: Repeat the download specified number of times if not
1530             successful.
1531         :type repeat: int
1532         """
1533
1534         logging.info(u"Downloading and parsing input files ...")
1535
1536         for job, builds in self._cfg.input.items():
1537             for build in builds:
1538
1539                 result = self._download_and_parse_build(job, build, repeat)
1540                 if result[u"last"]:
1541                     break
1542                 build_nr = result[u"build"][u"build"]
1543
1544                 if result[u"data"]:
1545                     data = result[u"data"]
1546                     build_data = pd.Series({
1547                         u"metadata": pd.Series(
1548                             list(data[u"metadata"].values()),
1549                             index=list(data[u"metadata"].keys())
1550                         ),
1551                         u"suites": pd.Series(
1552                             list(data[u"suites"].values()),
1553                             index=list(data[u"suites"].keys())
1554                         ),
1555                         u"tests": pd.Series(
1556                             list(data[u"tests"].values()),
1557                             index=list(data[u"tests"].keys())
1558                         )
1559                     })
1560
1561                     if self._input_data.get(job, None) is None:
1562                         self._input_data[job] = pd.Series()
1563                     self._input_data[job][str(build_nr)] = build_data
1564                     self._cfg.set_input_file_name(
1565                         job, build_nr, result[u"build"][u"file-name"]
1566                     )
1567                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1568
1569                 mem_alloc = \
1570                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1571                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1572
1573         logging.info(u"Done.")
1574
1575         msg = f"Successful downloads from the sources:\n"
1576         for source in self._cfg.environment[u"data-sources"]:
1577             if source[u"successful-downloads"]:
1578                 msg += (
1579                     f"{source[u'url']}/{source[u'path']}/"
1580                     f"{source[u'file-name']}: "
1581                     f"{source[u'successful-downloads']}\n"
1582                 )
1583         logging.info(msg)
1584
1585     def process_local_file(self, local_file, job=u"local", build_nr=1,
1586                            replace=True):
1587         """Process local XML file given as a command-line parameter.
1588
1589         :param local_file: The file to process.
1590         :param job: Job name.
1591         :param build_nr: Build number.
1592         :param replace: If True, the information about jobs and builds is
1593             replaced by the new one, otherwise the new jobs and builds are
1594             added.
1595         :type local_file: str
1596         :type job: str
1597         :type build_nr: int
1598         :type replace: bool
1599         :raises: PresentationError if an error occurs.
1600         """
1601         if not isfile(local_file):
1602             raise PresentationError(f"The file {local_file} does not exist.")
1603
1604         try:
1605             build_nr = int(local_file.split(u"/")[-1].split(u".")[0])
1606         except (IndexError, ValueError):
1607             pass
1608
1609         build = {
1610             u"build": build_nr,
1611             u"status": u"failed",
1612             u"file-name": local_file
1613         }
1614         if replace:
1615             self._cfg.input = dict()
1616         self._cfg.add_build(job, build)
1617
1618         logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1619         data = self._parse_tests(job, build)
1620         if data is None:
1621             raise PresentationError(
1622                 f"Error occurred while parsing the file {local_file}"
1623             )
1624
1625         build_data = pd.Series({
1626             u"metadata": pd.Series(
1627                 list(data[u"metadata"].values()),
1628                 index=list(data[u"metadata"].keys())
1629             ),
1630             u"suites": pd.Series(
1631                 list(data[u"suites"].values()),
1632                 index=list(data[u"suites"].keys())
1633             ),
1634             u"tests": pd.Series(
1635                 list(data[u"tests"].values()),
1636                 index=list(data[u"tests"].keys())
1637             )
1638         })
1639
1640         if self._input_data.get(job, None) is None:
1641             self._input_data[job] = pd.Series()
1642         self._input_data[job][str(build_nr)] = build_data
1643
1644         self._cfg.set_input_state(job, build_nr, u"processed")
1645
1646     def process_local_directory(self, local_dir, replace=True):
1647         """Process local directory with XML file(s). The directory is processed
1648         as a 'job' and the XML files in it as builds.
1649         If the given directory contains only sub-directories, these
1650         sub-directories processed as jobs and corresponding XML files as builds
1651         of their job.
1652
1653         :param local_dir: Local directory to process.
1654         :param replace: If True, the information about jobs and builds is
1655             replaced by the new one, otherwise the new jobs and builds are
1656             added.
1657         :type local_dir: str
1658         :type replace: bool
1659         """
1660         if not isdir(local_dir):
1661             raise PresentationError(
1662                 f"The directory {local_dir} does not exist."
1663             )
1664
1665         # Check if the given directory includes only files, or only directories
1666         _, dirnames, filenames = next(walk(local_dir))
1667
1668         if filenames and not dirnames:
1669             filenames.sort()
1670             # local_builds:
1671             # key: dir (job) name, value: list of file names (builds)
1672             local_builds = {
1673                 local_dir: [join(local_dir, name) for name in filenames]
1674             }
1675
1676         elif dirnames and not filenames:
1677             dirnames.sort()
1678             # local_builds:
1679             # key: dir (job) name, value: list of file names (builds)
1680             local_builds = dict()
1681             for dirname in dirnames:
1682                 builds = [
1683                     join(local_dir, dirname, name)
1684                     for name in listdir(join(local_dir, dirname))
1685                     if isfile(join(local_dir, dirname, name))
1686                 ]
1687                 if builds:
1688                     local_builds[dirname] = sorted(builds)
1689
1690         elif not filenames and not dirnames:
1691             raise PresentationError(f"The directory {local_dir} is empty.")
1692         else:
1693             raise PresentationError(
1694                 f"The directory {local_dir} can include only files or only "
1695                 f"directories, not both.\nThe directory {local_dir} includes "
1696                 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1697             )
1698
1699         if replace:
1700             self._cfg.input = dict()
1701
1702         for job, files in local_builds.items():
1703             for idx, local_file in enumerate(files):
1704                 self.process_local_file(local_file, job, idx + 1, replace=False)
1705
1706     @staticmethod
1707     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1708         """Return the index of character in the string which is the end of tag.
1709
1710         :param tag_filter: The string where the end of tag is being searched.
1711         :param start: The index where the searching is stated.
1712         :param closer: The character which is the tag closer.
1713         :type tag_filter: str
1714         :type start: int
1715         :type closer: str
1716         :returns: The index of the tag closer.
1717         :rtype: int
1718         """
1719         try:
1720             idx_opener = tag_filter.index(closer, start)
1721             return tag_filter.index(closer, idx_opener + 1)
1722         except ValueError:
1723             return None
1724
1725     @staticmethod
1726     def _condition(tag_filter):
1727         """Create a conditional statement from the given tag filter.
1728
1729         :param tag_filter: Filter based on tags from the element specification.
1730         :type tag_filter: str
1731         :returns: Conditional statement which can be evaluated.
1732         :rtype: str
1733         """
1734         index = 0
1735         while True:
1736             index = InputData._end_of_tag(tag_filter, index)
1737             if index is None:
1738                 return tag_filter
1739             index += 1
1740             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1741
1742     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1743                     continue_on_error=False):
1744         """Filter required data from the given jobs and builds.
1745
1746         The output data structure is:
1747         - job 1
1748           - build 1
1749             - test (or suite) 1 ID:
1750               - param 1
1751               - param 2
1752               ...
1753               - param n
1754             ...
1755             - test (or suite) n ID:
1756             ...
1757           ...
1758           - build n
1759         ...
1760         - job n
1761
1762         :param element: Element which will use the filtered data.
1763         :param params: Parameters which will be included in the output. If None,
1764             all parameters are included.
1765         :param data: If not None, this data is used instead of data specified
1766             in the element.
1767         :param data_set: The set of data to be filtered: tests, suites,
1768             metadata.
1769         :param continue_on_error: Continue if there is error while reading the
1770             data. The Item will be empty then
1771         :type element: pandas.Series
1772         :type params: list
1773         :type data: dict
1774         :type data_set: str
1775         :type continue_on_error: bool
1776         :returns: Filtered data.
1777         :rtype pandas.Series
1778         """
1779
1780         try:
1781             if data_set == "suites":
1782                 cond = u"True"
1783             elif element[u"filter"] in (u"all", u"template"):
1784                 cond = u"True"
1785             else:
1786                 cond = InputData._condition(element[u"filter"])
1787             logging.debug(f"   Filter: {cond}")
1788         except KeyError:
1789             logging.error(u"  No filter defined.")
1790             return None
1791
1792         if params is None:
1793             params = element.get(u"parameters", None)
1794             if params:
1795                 params.extend((u"type", u"status"))
1796
1797         data_to_filter = data if data else element[u"data"]
1798         data = pd.Series()
1799         try:
1800             for job, builds in data_to_filter.items():
1801                 data[job] = pd.Series()
1802                 for build in builds:
1803                     data[job][str(build)] = pd.Series()
1804                     try:
1805                         data_dict = dict(
1806                             self.data[job][str(build)][data_set].items())
1807                     except KeyError:
1808                         if continue_on_error:
1809                             continue
1810                         return None
1811
1812                     for test_id, test_data in data_dict.items():
1813                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1814                             data[job][str(build)][test_id] = pd.Series()
1815                             if params is None:
1816                                 for param, val in test_data.items():
1817                                     data[job][str(build)][test_id][param] = val
1818                             else:
1819                                 for param in params:
1820                                     try:
1821                                         data[job][str(build)][test_id][param] =\
1822                                             test_data[param]
1823                                     except KeyError:
1824                                         data[job][str(build)][test_id][param] =\
1825                                             u"No Data"
1826             return data
1827
1828         except (KeyError, IndexError, ValueError) as err:
1829             logging.error(
1830                 f"Missing mandatory parameter in the element specification: "
1831                 f"{repr(err)}"
1832             )
1833             return None
1834         except AttributeError as err:
1835             logging.error(repr(err))
1836             return None
1837         except SyntaxError as err:
1838             logging.error(
1839                 f"The filter {cond} is not correct. Check if all tags are "
1840                 f"enclosed by apostrophes.\n{repr(err)}"
1841             )
1842             return None
1843
1844     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1845                              continue_on_error=False):
1846         """Filter required data from the given jobs and builds.
1847
1848         The output data structure is:
1849         - job 1
1850           - build 1
1851             - test (or suite) 1 ID:
1852               - param 1
1853               - param 2
1854               ...
1855               - param n
1856             ...
1857             - test (or suite) n ID:
1858             ...
1859           ...
1860           - build n
1861         ...
1862         - job n
1863
1864         :param element: Element which will use the filtered data.
1865         :param params: Parameters which will be included in the output. If None,
1866         all parameters are included.
1867         :param data_set: The set of data to be filtered: tests, suites,
1868         metadata.
1869         :param continue_on_error: Continue if there is error while reading the
1870         data. The Item will be empty then
1871         :type element: pandas.Series
1872         :type params: list
1873         :type data_set: str
1874         :type continue_on_error: bool
1875         :returns: Filtered data.
1876         :rtype pandas.Series
1877         """
1878
1879         include = element.get(u"include", None)
1880         if not include:
1881             logging.warning(u"No tests to include, skipping the element.")
1882             return None
1883
1884         if params is None:
1885             params = element.get(u"parameters", None)
1886             if params and u"type" not in params:
1887                 params.append(u"type")
1888
1889         cores = element.get(u"core", None)
1890         if cores:
1891             tests = list()
1892             for core in cores:
1893                 for test in include:
1894                     tests.append(test.format(core=core))
1895         else:
1896             tests = include
1897
1898         data = pd.Series()
1899         try:
1900             for job, builds in element[u"data"].items():
1901                 data[job] = pd.Series()
1902                 for build in builds:
1903                     data[job][str(build)] = pd.Series()
1904                     for test in tests:
1905                         try:
1906                             reg_ex = re.compile(str(test).lower())
1907                             for test_id in self.data[job][
1908                                     str(build)][data_set].keys():
1909                                 if re.match(reg_ex, str(test_id).lower()):
1910                                     test_data = self.data[job][
1911                                         str(build)][data_set][test_id]
1912                                     data[job][str(build)][test_id] = pd.Series()
1913                                     if params is None:
1914                                         for param, val in test_data.items():
1915                                             data[job][str(build)][test_id]\
1916                                                 [param] = val
1917                                     else:
1918                                         for param in params:
1919                                             try:
1920                                                 data[job][str(build)][
1921                                                     test_id][param] = \
1922                                                     test_data[param]
1923                                             except KeyError:
1924                                                 data[job][str(build)][
1925                                                     test_id][param] = u"No Data"
1926                         except KeyError as err:
1927                             if continue_on_error:
1928                                 logging.debug(repr(err))
1929                                 continue
1930                             logging.error(repr(err))
1931                             return None
1932             return data
1933
1934         except (KeyError, IndexError, ValueError) as err:
1935             logging.error(
1936                 f"Missing mandatory parameter in the element "
1937                 f"specification: {repr(err)}"
1938             )
1939             return None
1940         except AttributeError as err:
1941             logging.error(repr(err))
1942             return None
1943
1944     @staticmethod
1945     def merge_data(data):
1946         """Merge data from more jobs and builds to a simple data structure.
1947
1948         The output data structure is:
1949
1950         - test (suite) 1 ID:
1951           - param 1
1952           - param 2
1953           ...
1954           - param n
1955         ...
1956         - test (suite) n ID:
1957         ...
1958
1959         :param data: Data to merge.
1960         :type data: pandas.Series
1961         :returns: Merged data.
1962         :rtype: pandas.Series
1963         """
1964
1965         logging.info(u"    Merging data ...")
1966
1967         merged_data = pd.Series()
1968         for builds in data.values:
1969             for item in builds.values:
1970                 for item_id, item_data in item.items():
1971                     merged_data[item_id] = item_data
1972         return merged_data
1973
1974     def print_all_oper_data(self):
1975         """Print all operational data to console.
1976         """
1977
1978         tbl_hdr = (
1979             u"Name",
1980             u"Nr of Vectors",
1981             u"Nr of Packets",
1982             u"Suspends",
1983             u"Cycles per Packet",
1984             u"Average Vector Size"
1985         )
1986
1987         for job in self._input_data.values:
1988             for build in job.values:
1989                 for test_id, test_data in build[u"tests"].items():
1990                     print(f"{test_id}")
1991                     if test_data.get(u"show-run", None) is None:
1992                         continue
1993                     for dut_name, data in test_data[u"show-run"].items():
1994                         if data.get(u"threads", None) is None:
1995                             continue
1996                         print(f"Host IP: {data.get(u'host', '')}, "
1997                               f"Socket: {data.get(u'socket', '')}")
1998                         for thread_nr, thread in data[u"threads"].items():
1999                             txt_table = prettytable.PrettyTable(tbl_hdr)
2000                             avg = 0.0
2001                             for row in thread:
2002                                 txt_table.add_row(row)
2003                                 avg += row[-1]
2004                             if len(thread) == 0:
2005                                 avg = u""
2006                             else:
2007                                 avg = f", Average Vector Size per Node: " \
2008                                       f"{(avg / len(thread)):.2f}"
2009                             th_name = u"main" if thread_nr == 0 \
2010                                 else f"worker_{thread_nr}"
2011                             print(f"{dut_name}, {th_name}{avg}")
2012                             txt_table.float_format = u".2"
2013                             txt_table.align = u"r"
2014                             txt_table.align[u"Name"] = u"l"
2015                             print(f"{txt_table.get_string()}\n")