resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove, walk, listdir
  29 from os.path import isfile, isdir, join
  30 from datetime import datetime as dt
  31 from datetime import timedelta
  32 from json import loads
  33 from json.decoder import JSONDecodeError
  34
  35 import hdrh.histogram
  36 import hdrh.codec
  37 import prettytable
  38 import pandas as pd
  39
  40 from robot.api import ExecutionResult, ResultVisitor
  41 from robot import errors
  42
  43 from resources.libraries.python import jumpavg
  44 from input_data_files import download_and_unzip_data_file
  45 from pal_errors import PresentationError
  46
  47
  48 # Separator used in file names
  49 SEPARATOR = u"__"
  50
  51
  52 class ExecutionChecker(ResultVisitor):
  53     """Class to traverse through the test suite structure.
  54
  55     The functionality implemented in this class generates a json structure:
  56
  57     Performance tests:
  58
  59     {
  60         "metadata": {
  61             "generated": "Timestamp",
  62             "version": "SUT version",
  63             "job": "Jenkins job name",
  64             "build": "Information about the build"
  65         },
  66         "suites": {
  67             "Suite long name 1": {
  68                 "name": Suite name,
  69                 "doc": "Suite 1 documentation",
  70                 "parent": "Suite 1 parent",
  71                 "level": "Level of the suite in the suite hierarchy"
  72             }
  73             "Suite long name N": {
  74                 "name": Suite name,
  75                 "doc": "Suite N documentation",
  76                 "parent": "Suite 2 parent",
  77                 "level": "Level of the suite in the suite hierarchy"
  78             }
  79         }
  80         "tests": {
  81             # NDRPDR tests:
  82             "ID": {
  83                 "name": "Test name",
  84                 "parent": "Name of the parent of the test",
  85                 "doc": "Test documentation",
  86                 "msg": "Test message",
  87                 "conf-history": "DUT1 and DUT2 VAT History",
  88                 "show-run": "Show Run",
  89                 "tags": ["tag 1", "tag 2", "tag n"],
  90                 "type": "NDRPDR",
  91                 "status": "PASS" | "FAIL",
  92                 "throughput": {
  93                     "NDR": {
  94                         "LOWER": float,
  95                         "UPPER": float
  96                     },
  97                     "PDR": {
  98                         "LOWER": float,
  99                         "UPPER": float
 100                     }
 101                 },
 102                 "latency": {
 103                     "NDR": {
 104                         "direction1": {
 105                             "min": float,
 106                             "avg": float,
 107                             "max": float,
 108                             "hdrh": str
 109                         },
 110                         "direction2": {
 111                             "min": float,
 112                             "avg": float,
 113                             "max": float,
 114                             "hdrh": str
 115                         }
 116                     },
 117                     "PDR": {
 118                         "direction1": {
 119                             "min": float,
 120                             "avg": float,
 121                             "max": float,
 122                             "hdrh": str
 123                         },
 124                         "direction2": {
 125                             "min": float,
 126                             "avg": float,
 127                             "max": float,
 128                             "hdrh": str
 129                         }
 130                     }
 131                 }
 132             }
 133
 134             # TCP tests:
 135             "ID": {
 136                 "name": "Test name",
 137                 "parent": "Name of the parent of the test",
 138                 "doc": "Test documentation",
 139                 "msg": "Test message",
 140                 "tags": ["tag 1", "tag 2", "tag n"],
 141                 "type": "TCP",
 142                 "status": "PASS" | "FAIL",
 143                 "result": int
 144             }
 145
 146             # MRR, BMRR tests:
 147             "ID": {
 148                 "name": "Test name",
 149                 "parent": "Name of the parent of the test",
 150                 "doc": "Test documentation",
 151                 "msg": "Test message",
 152                 "tags": ["tag 1", "tag 2", "tag n"],
 153                 "type": "MRR" | "BMRR",
 154                 "status": "PASS" | "FAIL",
 155                 "result": {
 156                     "receive-rate": float,
 157                     # Average of a list, computed using AvgStdevStats.
 158                     # In CSIT-1180, replace with List[float].
 159                 }
 160             }
 161
 162             "ID" {
 163                 # next test
 164             }
 165         }
 166     }
 167
 168
 169     Functional tests:
 170
 171     {
 172         "metadata": {  # Optional
 173             "version": "VPP version",
 174             "job": "Jenkins job name",
 175             "build": "Information about the build"
 176         },
 177         "suites": {
 178             "Suite name 1": {
 179                 "doc": "Suite 1 documentation",
 180                 "parent": "Suite 1 parent",
 181                 "level": "Level of the suite in the suite hierarchy"
 182             }
 183             "Suite name N": {
 184                 "doc": "Suite N documentation",
 185                 "parent": "Suite 2 parent",
 186                 "level": "Level of the suite in the suite hierarchy"
 187             }
 188         }
 189         "tests": {
 190             "ID": {
 191                 "name": "Test name",
 192                 "parent": "Name of the parent of the test",
 193                 "doc": "Test documentation"
 194                 "msg": "Test message"
 195                 "tags": ["tag 1", "tag 2", "tag n"],
 196                 "conf-history": "DUT1 and DUT2 VAT History"
 197                 "show-run": "Show Run"
 198                 "status": "PASS" | "FAIL"
 199             },
 200             "ID" {
 201                 # next test
 202             }
 203         }
 204     }
 205
 206     .. note:: ID is the lowercase full path to the test.
 207     """
 208
 209     REGEX_PLR_RATE = re.compile(
 210         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 211         r'PLRsearch upper bound::?\s(\d+.\d+)'
 212     )
 213     REGEX_NDRPDR_RATE = re.compile(
 214         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 215         r'NDR_UPPER:\s(\d+.\d+).*\n'
 216         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 217         r'PDR_UPPER:\s(\d+.\d+)'
 218     )
 219     REGEX_PERF_MSG_INFO = re.compile(
 220         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 221         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 222         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 223         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 224         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 225     )
 226     REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
 227
 228     # TODO: Remove when not needed
 229     REGEX_NDRPDR_LAT_BASE = re.compile(
 230         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 231         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 232     )
 233     REGEX_NDRPDR_LAT = re.compile(
 234         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 235         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 236         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 237         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 238         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 239         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 240     )
 241     # TODO: Remove when not needed
 242     REGEX_NDRPDR_LAT_LONG = re.compile(
 243         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 244         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 245         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 246         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 247         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 248         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 249         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 250         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 251         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 252     )
 253     REGEX_VERSION_VPP = re.compile(
 254         r"(return STDOUT Version:\s*|"
 255         r"VPP Version:\s*|VPP version:\s*)(.*)"
 256     )
 257     REGEX_VERSION_DPDK = re.compile(
 258         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 259     )
 260     REGEX_TCP = re.compile(
 261         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 262     )
 263     REGEX_MRR = re.compile(
 264         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 265         r'tx\s(\d*),\srx\s(\d*)'
 266     )
 267     REGEX_BMRR = re.compile(
 268         r'Maximum Receive Rate trial results'
 269         r' in packets per second: \[(.*)\]'
 270     )
 271     REGEX_RECONF_LOSS = re.compile(
 272         r'Packets lost due to reconfig: (\d*)'
 273     )
 274     REGEX_RECONF_TIME = re.compile(
 275         r'Implied time lost: (\d*.[\de-]*)'
 276     )
 277     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 278
 279     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 280
 281     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 282
 283     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 284
 285     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 286
 287     def __init__(self, metadata, mapping, ignore):
 288         """Initialisation.
 289
 290         :param metadata: Key-value pairs to be included in "metadata" part of
 291             JSON structure.
 292         :param mapping: Mapping of the old names of test cases to the new
 293             (actual) one.
 294         :param ignore: List of TCs to be ignored.
 295         :type metadata: dict
 296         :type mapping: dict
 297         :type ignore: list
 298         """
 299
 300         # Type of message to parse out from the test messages
 301         self._msg_type = None
 302
 303         # VPP version
 304         self._version = None
 305
 306         # Timestamp
 307         self._timestamp = None
 308
 309         # Testbed. The testbed is identified by TG node IP address.
 310         self._testbed = None
 311
 312         # Mapping of TCs long names
 313         self._mapping = mapping
 314
 315         # Ignore list
 316         self._ignore = ignore
 317
 318         # Number of PAPI History messages found:
 319         # 0 - no message
 320         # 1 - PAPI History of DUT1
 321         # 2 - PAPI History of DUT2
 322         self._conf_history_lookup_nr = 0
 323
 324         self._sh_run_counter = 0
 325
 326         # Test ID of currently processed test- the lowercase full path to the
 327         # test
 328         self._test_id = None
 329
 330         # The main data structure
 331         self._data = {
 332             u"metadata": OrderedDict(),
 333             u"suites": OrderedDict(),
 334             u"tests": OrderedDict()
 335         }
 336
 337         # Save the provided metadata
 338         for key, val in metadata.items():
 339             self._data[u"metadata"][key] = val
 340
 341         # Dictionary defining the methods used to parse different types of
 342         # messages
 343         self.parse_msg = {
 344             u"timestamp": self._get_timestamp,
 345             u"vpp-version": self._get_vpp_version,
 346             u"dpdk-version": self._get_dpdk_version,
 347             # TODO: Remove when not needed:
 348             u"teardown-vat-history": self._get_vat_history,
 349             u"teardown-papi-history": self._get_papi_history,
 350             u"test-show-runtime": self._get_show_run,
 351             u"testbed": self._get_testbed
 352         }
 353
 354     @property
 355     def data(self):
 356         """Getter - Data parsed from the XML file.
 357
 358         :returns: Data parsed from the XML file.
 359         :rtype: dict
 360         """
 361         return self._data
 362
 363     def _get_data_from_mrr_test_msg(self, msg):
 364         """Get info from message of MRR performance tests.
 365
 366         :param msg: Message to be processed.
 367         :type msg: str
 368         :returns: Processed message or original message if a problem occurs.
 369         :rtype: str
 370         """
 371
 372         groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
 373         if not groups or groups.lastindex != 1:
 374             return u"Test Failed."
 375
 376         try:
 377             data = groups.group(1).split(u", ")
 378         except (AttributeError, IndexError, ValueError, KeyError):
 379             return u"Test Failed."
 380
 381         out_str = u"["
 382         try:
 383             for item in data:
 384                 out_str += f"{(float(item) / 1e6):.2f}, "
 385             return out_str[:-2] + u"]"
 386         except (AttributeError, IndexError, ValueError, KeyError):
 387             return u"Test Failed."
 388
 389     def _get_data_from_perf_test_msg(self, msg):
 390         """Get info from message of NDRPDR performance tests.
 391
 392         :param msg: Message to be processed.
 393         :type msg: str
 394         :returns: Processed message or original message if a problem occurs.
 395         :rtype: str
 396         """
 397
 398         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 399         if not groups or groups.lastindex != 10:
 400             return u"Test Failed."
 401
 402         try:
 403             data = {
 404                 u"ndr_low": float(groups.group(1)),
 405                 u"ndr_low_b": float(groups.group(2)),
 406                 u"pdr_low": float(groups.group(3)),
 407                 u"pdr_low_b": float(groups.group(4)),
 408                 u"pdr_lat_90_1": groups.group(5),
 409                 u"pdr_lat_90_2": groups.group(6),
 410                 u"pdr_lat_50_1": groups.group(7),
 411                 u"pdr_lat_50_2": groups.group(8),
 412                 u"pdr_lat_10_1": groups.group(9),
 413                 u"pdr_lat_10_2": groups.group(10),
 414             }
 415         except (AttributeError, IndexError, ValueError, KeyError):
 416             return u"Test Failed."
 417
 418         def _process_lat(in_str_1, in_str_2):
 419             """Extract min, avg, max values from latency string.
 420
 421             :param in_str_1: Latency string for one direction produced by robot
 422                 framework.
 423             :param in_str_2: Latency string for second direction produced by
 424                 robot framework.
 425             :type in_str_1: str
 426             :type in_str_2: str
 427             :returns: Processed latency string or None if a problem occurs.
 428             :rtype: tuple
 429             """
 430             in_list_1 = in_str_1.split('/', 3)
 431             in_list_2 = in_str_2.split('/', 3)
 432
 433             if len(in_list_1) != 4 and len(in_list_2) != 4:
 434                 return None
 435
 436             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 437             try:
 438                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 439             except hdrh.codec.HdrLengthException:
 440                 return None
 441
 442             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 443             try:
 444                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 445             except hdrh.codec.HdrLengthException:
 446                 return None
 447
 448             if hdr_lat_1 and hdr_lat_2:
 449                 hdr_lat = (
 450                     hdr_lat_1.get_value_at_percentile(50.0),
 451                     hdr_lat_1.get_value_at_percentile(90.0),
 452                     hdr_lat_1.get_value_at_percentile(99.0),
 453                     hdr_lat_2.get_value_at_percentile(50.0),
 454                     hdr_lat_2.get_value_at_percentile(90.0),
 455                     hdr_lat_2.get_value_at_percentile(99.0)
 456                 )
 457
 458                 if all(hdr_lat):
 459                     return hdr_lat
 460
 461             return None
 462
 463         try:
 464             out_msg = (
 465                 f"1. {(data[u'ndr_low'] / 1e6):5.2f}      "
 466                 f"{data[u'ndr_low_b']:5.2f}"
 467                 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f}      "
 468                 f"{data[u'pdr_low_b']:5.2f}"
 469             )
 470             latency = (
 471                 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
 472                 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
 473                 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
 474             )
 475             if all(latency):
 476                 max_len = len(str(max((max(item) for item in latency))))
 477                 max_len = 4 if max_len < 4 else max_len
 478
 479                 for idx, lat in enumerate(latency):
 480                     if not idx:
 481                         out_msg += u"\n"
 482                     out_msg += (
 483                         f"\n{idx + 3}. "
 484                         f"{lat[0]:{max_len}d} "
 485                         f"{lat[1]:{max_len}d} "
 486                         f"{lat[2]:{max_len}d}      "
 487                         f"{lat[3]:{max_len}d} "
 488                         f"{lat[4]:{max_len}d} "
 489                         f"{lat[5]:{max_len}d} "
 490                     )
 491
 492             return out_msg
 493
 494         except (AttributeError, IndexError, ValueError, KeyError):
 495             return u"Test Failed."
 496
 497     def _get_testbed(self, msg):
 498         """Called when extraction of testbed IP is required.
 499         The testbed is identified by TG node IP address.
 500
 501         :param msg: Message to process.
 502         :type msg: Message
 503         :returns: Nothing.
 504         """
 505
 506         if msg.message.count(u"Setup of TG node") or \
 507                 msg.message.count(u"Setup of node TG host"):
 508             reg_tg_ip = re.compile(
 509                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 510             try:
 511                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 512             except (KeyError, ValueError, IndexError, AttributeError):
 513                 pass
 514             finally:
 515                 self._data[u"metadata"][u"testbed"] = self._testbed
 516                 self._msg_type = None
 517
 518     def _get_vpp_version(self, msg):
 519         """Called when extraction of VPP version is required.
 520
 521         :param msg: Message to process.
 522         :type msg: Message
 523         :returns: Nothing.
 524         """
 525
 526         if msg.message.count(u"return STDOUT Version:") or \
 527             msg.message.count(u"VPP Version:") or \
 528             msg.message.count(u"VPP version:"):
 529             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 530                                 group(2))
 531             self._data[u"metadata"][u"version"] = self._version
 532             self._msg_type = None
 533
 534     def _get_dpdk_version(self, msg):
 535         """Called when extraction of DPDK version is required.
 536
 537         :param msg: Message to process.
 538         :type msg: Message
 539         :returns: Nothing.
 540         """
 541
 542         if msg.message.count(u"DPDK Version:"):
 543             try:
 544                 self._version = str(re.search(
 545                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 546                 self._data[u"metadata"][u"version"] = self._version
 547             except IndexError:
 548                 pass
 549             finally:
 550                 self._msg_type = None
 551
 552     def _get_timestamp(self, msg):
 553         """Called when extraction of timestamp is required.
 554
 555         :param msg: Message to process.
 556         :type msg: Message
 557         :returns: Nothing.
 558         """
 559
 560         self._timestamp = msg.timestamp[:14]
 561         self._data[u"metadata"][u"generated"] = self._timestamp
 562         self._msg_type = None
 563
 564     def _get_vat_history(self, msg):
 565         """Called when extraction of VAT command history is required.
 566
 567         TODO: Remove when not needed.
 568
 569         :param msg: Message to process.
 570         :type msg: Message
 571         :returns: Nothing.
 572         """
 573         if msg.message.count(u"VAT command history:"):
 574             self._conf_history_lookup_nr += 1
 575             if self._conf_history_lookup_nr == 1:
 576                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 577             else:
 578                 self._msg_type = None
 579             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 580                           r"VAT command history:", u"",
 581                           msg.message, count=1).replace(u'\n', u' |br| ').\
 582                 replace(u'"', u"'")
 583
 584             self._data[u"tests"][self._test_id][u"conf-history"] += (
 585                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 586             )
 587
 588     def _get_papi_history(self, msg):
 589         """Called when extraction of PAPI command history is required.
 590
 591         :param msg: Message to process.
 592         :type msg: Message
 593         :returns: Nothing.
 594         """
 595         if msg.message.count(u"PAPI command history:"):
 596             self._conf_history_lookup_nr += 1
 597             if self._conf_history_lookup_nr == 1:
 598                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 599             else:
 600                 self._msg_type = None
 601             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 602                           r"PAPI command history:", u"",
 603                           msg.message, count=1).replace(u'\n', u' |br| ').\
 604                 replace(u'"', u"'")
 605             self._data[u"tests"][self._test_id][u"conf-history"] += (
 606                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 607             )
 608
 609     def _get_show_run(self, msg):
 610         """Called when extraction of VPP operational data (output of CLI command
 611         Show Runtime) is required.
 612
 613         :param msg: Message to process.
 614         :type msg: Message
 615         :returns: Nothing.
 616         """
 617
 618         if not msg.message.count(u"stats runtime"):
 619             return
 620
 621         # Temporary solution
 622         if self._sh_run_counter > 1:
 623             return
 624
 625         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 626             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 627
 628         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 629         if not groups:
 630             return
 631         try:
 632             host = groups.group(1)
 633         except (AttributeError, IndexError):
 634             host = u""
 635         try:
 636             sock = groups.group(2)
 637         except (AttributeError, IndexError):
 638             sock = u""
 639
 640         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 641                         replace(u"'", u'"').replace(u'b"', u'"').
 642                         replace(u'u"', u'"').split(u":", 1)[1])
 643
 644         try:
 645             threads_nr = len(runtime[0][u"clocks"])
 646         except (IndexError, KeyError):
 647             return
 648
 649         dut = u"DUT{nr}".format(
 650             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 651
 652         oper = {
 653             u"host": host,
 654             u"socket": sock,
 655             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 656         }
 657
 658         for item in runtime:
 659             for idx in range(threads_nr):
 660                 if item[u"vectors"][idx] > 0:
 661                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 662                 elif item[u"calls"][idx] > 0:
 663                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 664                 elif item[u"suspends"][idx] > 0:
 665                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 666                 else:
 667                     clocks = 0.0
 668
 669                 if item[u"calls"][idx] > 0:
 670                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 671                 else:
 672                     vectors_call = 0.0
 673
 674                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 675                         int(item[u"suspends"][idx]):
 676                     oper[u"threads"][idx].append([
 677                         item[u"name"],
 678                         item[u"calls"][idx],
 679                         item[u"vectors"][idx],
 680                         item[u"suspends"][idx],
 681                         clocks,
 682                         vectors_call
 683                     ])
 684
 685         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 686
 687     def _get_ndrpdr_throughput(self, msg):
 688         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 689         message.
 690
 691         :param msg: The test message to be parsed.
 692         :type msg: str
 693         :returns: Parsed data as a dict and the status (PASS/FAIL).
 694         :rtype: tuple(dict, str)
 695         """
 696
 697         throughput = {
 698             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 699             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 700         }
 701         status = u"FAIL"
 702         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 703
 704         if groups is not None:
 705             try:
 706                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 707                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 708                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 709                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 710                 status = u"PASS"
 711             except (IndexError, ValueError):
 712                 pass
 713
 714         return throughput, status
 715
 716     def _get_plr_throughput(self, msg):
 717         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 718         message.
 719
 720         :param msg: The test message to be parsed.
 721         :type msg: str
 722         :returns: Parsed data as a dict and the status (PASS/FAIL).
 723         :rtype: tuple(dict, str)
 724         """
 725
 726         throughput = {
 727             u"LOWER": -1.0,
 728             u"UPPER": -1.0
 729         }
 730         status = u"FAIL"
 731         groups = re.search(self.REGEX_PLR_RATE, msg)
 732
 733         if groups is not None:
 734             try:
 735                 throughput[u"LOWER"] = float(groups.group(1))
 736                 throughput[u"UPPER"] = float(groups.group(2))
 737                 status = u"PASS"
 738             except (IndexError, ValueError):
 739                 pass
 740
 741         return throughput, status
 742
 743     def _get_ndrpdr_latency(self, msg):
 744         """Get LATENCY from the test message.
 745
 746         :param msg: The test message to be parsed.
 747         :type msg: str
 748         :returns: Parsed data as a dict and the status (PASS/FAIL).
 749         :rtype: tuple(dict, str)
 750         """
 751         latency_default = {
 752             u"min": -1.0,
 753             u"avg": -1.0,
 754             u"max": -1.0,
 755             u"hdrh": u""
 756         }
 757         latency = {
 758             u"NDR": {
 759                 u"direction1": copy.copy(latency_default),
 760                 u"direction2": copy.copy(latency_default)
 761             },
 762             u"PDR": {
 763                 u"direction1": copy.copy(latency_default),
 764                 u"direction2": copy.copy(latency_default)
 765             },
 766             u"LAT0": {
 767                 u"direction1": copy.copy(latency_default),
 768                 u"direction2": copy.copy(latency_default)
 769             },
 770             u"PDR10": {
 771                 u"direction1": copy.copy(latency_default),
 772                 u"direction2": copy.copy(latency_default)
 773             },
 774             u"PDR50": {
 775                 u"direction1": copy.copy(latency_default),
 776                 u"direction2": copy.copy(latency_default)
 777             },
 778             u"PDR90": {
 779                 u"direction1": copy.copy(latency_default),
 780                 u"direction2": copy.copy(latency_default)
 781             },
 782         }
 783
 784         # TODO: Rewrite when long and base are not needed
 785         groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
 786         if groups is None:
 787             groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 788         if groups is None:
 789             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 790         if groups is None:
 791             return latency, u"FAIL"
 792
 793         def process_latency(in_str):
 794             """Return object with parsed latency values.
 795
 796             TODO: Define class for the return type.
 797
 798             :param in_str: Input string, min/avg/max/hdrh format.
 799             :type in_str: str
 800             :returns: Dict with corresponding keys, except hdrh float values.
 801             :rtype dict:
 802             :throws IndexError: If in_str does not have enough substrings.
 803             :throws ValueError: If a substring does not convert to float.
 804             """
 805             in_list = in_str.split('/', 3)
 806
 807             rval = {
 808                 u"min": float(in_list[0]),
 809                 u"avg": float(in_list[1]),
 810                 u"max": float(in_list[2]),
 811                 u"hdrh": u""
 812             }
 813
 814             if len(in_list) == 4:
 815                 rval[u"hdrh"] = str(in_list[3])
 816
 817             return rval
 818
 819         try:
 820             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 821             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 822             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 823             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 824             if groups.lastindex == 4:
 825                 return latency, u"PASS"
 826         except (IndexError, ValueError):
 827             pass
 828
 829         try:
 830             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 831             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 832             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 833             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 834             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 835             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 836             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 837             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 838             if groups.lastindex == 12:
 839                 return latency, u"PASS"
 840         except (IndexError, ValueError):
 841             pass
 842
 843         # TODO: Remove when not needed
 844         latency[u"NDR10"] = {
 845             u"direction1": copy.copy(latency_default),
 846             u"direction2": copy.copy(latency_default)
 847         }
 848         latency[u"NDR50"] = {
 849             u"direction1": copy.copy(latency_default),
 850             u"direction2": copy.copy(latency_default)
 851         }
 852         latency[u"NDR90"] = {
 853             u"direction1": copy.copy(latency_default),
 854             u"direction2": copy.copy(latency_default)
 855         }
 856         try:
 857             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
 858             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
 859             latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
 860             latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
 861             latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
 862             latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
 863             latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
 864             latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
 865             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
 866             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
 867             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
 868             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
 869             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
 870             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
 871             return latency, u"PASS"
 872         except (IndexError, ValueError):
 873             pass
 874
 875         return latency, u"FAIL"
 876
 877     @staticmethod
 878     def _get_hoststack_data(msg, tags):
 879         """Get data from the hoststack test message.
 880
 881         :param msg: The test message to be parsed.
 882         :param tags: Test tags.
 883         :type msg: str
 884         :type tags: list
 885         :returns: Parsed data as a JSON dict and the status (PASS/FAIL).
 886         :rtype: tuple(dict, str)
 887         """
 888         result = dict()
 889         status = u"FAIL"
 890
 891         msg = msg.replace(u"'", u'"').replace(u" ", u"")
 892         if u"LDPRELOAD" in tags:
 893             try:
 894                 result = loads(msg)
 895                 status = u"PASS"
 896             except JSONDecodeError:
 897                 pass
 898         elif u"VPPECHO" in tags:
 899             try:
 900                 msg_lst = msg.replace(u"}{", u"} {").split(u" ")
 901                 result = dict(
 902                     client=loads(msg_lst[0]),
 903                     server=loads(msg_lst[1])
 904                 )
 905                 status = u"PASS"
 906             except (JSONDecodeError, IndexError):
 907                 pass
 908
 909         return result, status
 910
 911     def visit_suite(self, suite):
 912         """Implements traversing through the suite and its direct children.
 913
 914         :param suite: Suite to process.
 915         :type suite: Suite
 916         :returns: Nothing.
 917         """
 918         if self.start_suite(suite) is not False:
 919             suite.suites.visit(self)
 920             suite.tests.visit(self)
 921             self.end_suite(suite)
 922
 923     def start_suite(self, suite):
 924         """Called when suite starts.
 925
 926         :param suite: Suite to process.
 927         :type suite: Suite
 928         :returns: Nothing.
 929         """
 930
 931         try:
 932             parent_name = suite.parent.name
 933         except AttributeError:
 934             return
 935
 936         doc_str = suite.doc.\
 937             replace(u'"', u"'").\
 938             replace(u'\n', u' ').\
 939             replace(u'\r', u'').\
 940             replace(u'*[', u' |br| *[').\
 941             replace(u"*", u"**").\
 942             replace(u' |br| *[', u'*[', 1)
 943
 944         self._data[u"suites"][suite.longname.lower().
 945                               replace(u'"', u"'").
 946                               replace(u" ", u"_")] = {
 947                                   u"name": suite.name.lower(),
 948                                   u"doc": doc_str,
 949                                   u"parent": parent_name,
 950                                   u"level": len(suite.longname.split(u"."))
 951                               }
 952
 953         suite.keywords.visit(self)
 954
 955     def end_suite(self, suite):
 956         """Called when suite ends.
 957
 958         :param suite: Suite to process.
 959         :type suite: Suite
 960         :returns: Nothing.
 961         """
 962
 963     def visit_test(self, test):
 964         """Implements traversing through the test.
 965
 966         :param test: Test to process.
 967         :type test: Test
 968         :returns: Nothing.
 969         """
 970         if self.start_test(test) is not False:
 971             test.keywords.visit(self)
 972             self.end_test(test)
 973
 974     def start_test(self, test):
 975         """Called when test starts.
 976
 977         :param test: Test to process.
 978         :type test: Test
 979         :returns: Nothing.
 980         """
 981
 982         self._sh_run_counter = 0
 983
 984         longname_orig = test.longname.lower()
 985
 986         # Check the ignore list
 987         if longname_orig in self._ignore:
 988             return
 989
 990         tags = [str(tag) for tag in test.tags]
 991         test_result = dict()
 992
 993         # Change the TC long name and name if defined in the mapping table
 994         longname = self._mapping.get(longname_orig, None)
 995         if longname is not None:
 996             name = longname.split(u'.')[-1]
 997             logging.debug(
 998                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
 999                 f"{name}"
1000             )
1001         else:
1002             longname = longname_orig
1003             name = test.name.lower()
1004
1005         # Remove TC number from the TC long name (backward compatibility):
1006         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
1007         # Remove TC number from the TC name (not needed):
1008         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
1009
1010         test_result[u"parent"] = test.parent.name.lower()
1011         test_result[u"tags"] = tags
1012         test_result["doc"] = test.doc.\
1013             replace(u'"', u"'").\
1014             replace(u'\n', u' ').\
1015             replace(u'\r', u'').\
1016             replace(u'[', u' |br| [').\
1017             replace(u' |br| [', u'[', 1)
1018         test_result[u"type"] = u"FUNC"
1019         test_result[u"status"] = test.status
1020
1021         if test.status == u"PASS":
1022             if u"NDRPDR" in tags:
1023                 test_result[u"msg"] = self._get_data_from_perf_test_msg(
1024                     test.message).replace(u'\n', u' |br| ').\
1025                     replace(u'\r', u'').replace(u'"', u"'")
1026             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1027                 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
1028                     test.message).replace(u'\n', u' |br| ').\
1029                     replace(u'\r', u'').replace(u'"', u"'")
1030             else:
1031                 test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\
1032                     replace(u'\r', u'').replace(u'"', u"'")
1033         else:
1034             test_result[u"msg"] = u"Test Failed."
1035
1036         if u"PERFTEST" in tags:
1037             # Replace info about cores (e.g. -1c-) with the info about threads
1038             # and cores (e.g. -1t1c-) in the long test case names and in the
1039             # test case names if necessary.
1040             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1041             if not groups:
1042                 tag_count = 0
1043                 tag_tc = str()
1044                 for tag in test_result[u"tags"]:
1045                     groups = re.search(self.REGEX_TC_TAG, tag)
1046                     if groups:
1047                         tag_count += 1
1048                         tag_tc = tag
1049
1050                 if tag_count == 1:
1051                     self._test_id = re.sub(
1052                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1053                         self._test_id, count=1
1054                     )
1055                     test_result[u"name"] = re.sub(
1056                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1057                         test_result["name"], count=1
1058                     )
1059                 else:
1060                     test_result[u"status"] = u"FAIL"
1061                     self._data[u"tests"][self._test_id] = test_result
1062                     logging.debug(
1063                         f"The test {self._test_id} has no or more than one "
1064                         f"multi-threading tags.\n"
1065                         f"Tags: {test_result[u'tags']}"
1066                     )
1067                     return
1068
1069         if test.status == u"PASS":
1070             if u"NDRPDR" in tags:
1071                 test_result[u"type"] = u"NDRPDR"
1072                 test_result[u"throughput"], test_result[u"status"] = \
1073                     self._get_ndrpdr_throughput(test.message)
1074                 test_result[u"latency"], test_result[u"status"] = \
1075                     self._get_ndrpdr_latency(test.message)
1076             elif u"SOAK" in tags:
1077                 test_result[u"type"] = u"SOAK"
1078                 test_result[u"throughput"], test_result[u"status"] = \
1079                     self._get_plr_throughput(test.message)
1080             elif u"HOSTSTACK" in tags:
1081                 test_result[u"type"] = u"HOSTSTACK"
1082                 test_result[u"result"], test_result[u"status"] = \
1083                     self._get_hoststack_data(test.message, tags)
1084             elif u"TCP" in tags:
1085                 test_result[u"type"] = u"TCP"
1086                 groups = re.search(self.REGEX_TCP, test.message)
1087                 test_result[u"result"] = int(groups.group(2))
1088             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1089                 if u"MRR" in tags:
1090                     test_result[u"type"] = u"MRR"
1091                 else:
1092                     test_result[u"type"] = u"BMRR"
1093
1094                 test_result[u"result"] = dict()
1095                 groups = re.search(self.REGEX_BMRR, test.message)
1096                 if groups is not None:
1097                     items_str = groups.group(1)
1098                     items_float = [float(item.strip()) for item
1099                                    in items_str.split(",")]
1100                     # Use whole list in CSIT-1180.
1101                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1102                     test_result[u"result"][u"receive-rate"] = stats.avg
1103                 else:
1104                     groups = re.search(self.REGEX_MRR, test.message)
1105                     test_result[u"result"][u"receive-rate"] = \
1106                         float(groups.group(3)) / float(groups.group(1))
1107             elif u"RECONF" in tags:
1108                 test_result[u"type"] = u"RECONF"
1109                 test_result[u"result"] = None
1110                 try:
1111                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1112                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1113                     test_result[u"result"] = {
1114                         u"loss": int(grps_loss.group(1)),
1115                         u"time": float(grps_time.group(1))
1116                     }
1117                 except (AttributeError, IndexError, ValueError, TypeError):
1118                     test_result[u"status"] = u"FAIL"
1119             elif u"DEVICETEST" in tags:
1120                 test_result[u"type"] = u"DEVICETEST"
1121             else:
1122                 test_result[u"status"] = u"FAIL"
1123                 self._data[u"tests"][self._test_id] = test_result
1124                 return
1125
1126         self._data[u"tests"][self._test_id] = test_result
1127
1128     def end_test(self, test):
1129         """Called when test ends.
1130
1131         :param test: Test to process.
1132         :type test: Test
1133         :returns: Nothing.
1134         """
1135
1136     def visit_keyword(self, keyword):
1137         """Implements traversing through the keyword and its child keywords.
1138
1139         :param keyword: Keyword to process.
1140         :type keyword: Keyword
1141         :returns: Nothing.
1142         """
1143         if self.start_keyword(keyword) is not False:
1144             self.end_keyword(keyword)
1145
1146     def start_keyword(self, keyword):
1147         """Called when keyword starts. Default implementation does nothing.
1148
1149         :param keyword: Keyword to process.
1150         :type keyword: Keyword
1151         :returns: Nothing.
1152         """
1153         try:
1154             if keyword.type == u"setup":
1155                 self.visit_setup_kw(keyword)
1156             elif keyword.type == u"teardown":
1157                 self.visit_teardown_kw(keyword)
1158             else:
1159                 self.visit_test_kw(keyword)
1160         except AttributeError:
1161             pass
1162
1163     def end_keyword(self, keyword):
1164         """Called when keyword ends. Default implementation does nothing.
1165
1166         :param keyword: Keyword to process.
1167         :type keyword: Keyword
1168         :returns: Nothing.
1169         """
1170
1171     def visit_test_kw(self, test_kw):
1172         """Implements traversing through the test keyword and its child
1173         keywords.
1174
1175         :param test_kw: Keyword to process.
1176         :type test_kw: Keyword
1177         :returns: Nothing.
1178         """
1179         for keyword in test_kw.keywords:
1180             if self.start_test_kw(keyword) is not False:
1181                 self.visit_test_kw(keyword)
1182                 self.end_test_kw(keyword)
1183
1184     def start_test_kw(self, test_kw):
1185         """Called when test keyword starts. Default implementation does
1186         nothing.
1187
1188         :param test_kw: Keyword to process.
1189         :type test_kw: Keyword
1190         :returns: Nothing.
1191         """
1192         if test_kw.name.count(u"Show Runtime On All Duts") or \
1193                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
1194             self._msg_type = u"test-show-runtime"
1195             self._sh_run_counter += 1
1196         elif test_kw.name.count(u"Install Dpdk Test On All Duts") and \
1197                 not self._version:
1198             self._msg_type = u"dpdk-version"
1199         else:
1200             return
1201         test_kw.messages.visit(self)
1202
1203     def end_test_kw(self, test_kw):
1204         """Called when keyword ends. Default implementation does nothing.
1205
1206         :param test_kw: Keyword to process.
1207         :type test_kw: Keyword
1208         :returns: Nothing.
1209         """
1210
1211     def visit_setup_kw(self, setup_kw):
1212         """Implements traversing through the teardown keyword and its child
1213         keywords.
1214
1215         :param setup_kw: Keyword to process.
1216         :type setup_kw: Keyword
1217         :returns: Nothing.
1218         """
1219         for keyword in setup_kw.keywords:
1220             if self.start_setup_kw(keyword) is not False:
1221                 self.visit_setup_kw(keyword)
1222                 self.end_setup_kw(keyword)
1223
1224     def start_setup_kw(self, setup_kw):
1225         """Called when teardown keyword starts. Default implementation does
1226         nothing.
1227
1228         :param setup_kw: Keyword to process.
1229         :type setup_kw: Keyword
1230         :returns: Nothing.
1231         """
1232         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1233                 and not self._version:
1234             self._msg_type = u"vpp-version"
1235         elif setup_kw.name.count(u"Set Global Variable") \
1236                 and not self._timestamp:
1237             self._msg_type = u"timestamp"
1238         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1239             self._msg_type = u"testbed"
1240         else:
1241             return
1242         setup_kw.messages.visit(self)
1243
1244     def end_setup_kw(self, setup_kw):
1245         """Called when keyword ends. Default implementation does nothing.
1246
1247         :param setup_kw: Keyword to process.
1248         :type setup_kw: Keyword
1249         :returns: Nothing.
1250         """
1251
1252     def visit_teardown_kw(self, teardown_kw):
1253         """Implements traversing through the teardown keyword and its child
1254         keywords.
1255
1256         :param teardown_kw: Keyword to process.
1257         :type teardown_kw: Keyword
1258         :returns: Nothing.
1259         """
1260         for keyword in teardown_kw.keywords:
1261             if self.start_teardown_kw(keyword) is not False:
1262                 self.visit_teardown_kw(keyword)
1263                 self.end_teardown_kw(keyword)
1264
1265     def start_teardown_kw(self, teardown_kw):
1266         """Called when teardown keyword starts
1267
1268         :param teardown_kw: Keyword to process.
1269         :type teardown_kw: Keyword
1270         :returns: Nothing.
1271         """
1272
1273         if teardown_kw.name.count(u"Show Vat History On All Duts"):
1274             # TODO: Remove when not needed:
1275             self._conf_history_lookup_nr = 0
1276             self._msg_type = u"teardown-vat-history"
1277             teardown_kw.messages.visit(self)
1278         elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1279             self._conf_history_lookup_nr = 0
1280             self._msg_type = u"teardown-papi-history"
1281             teardown_kw.messages.visit(self)
1282
1283     def end_teardown_kw(self, teardown_kw):
1284         """Called when keyword ends. Default implementation does nothing.
1285
1286         :param teardown_kw: Keyword to process.
1287         :type teardown_kw: Keyword
1288         :returns: Nothing.
1289         """
1290
1291     def visit_message(self, msg):
1292         """Implements visiting the message.
1293
1294         :param msg: Message to process.
1295         :type msg: Message
1296         :returns: Nothing.
1297         """
1298         if self.start_message(msg) is not False:
1299             self.end_message(msg)
1300
1301     def start_message(self, msg):
1302         """Called when message starts. Get required information from messages:
1303         - VPP version.
1304
1305         :param msg: Message to process.
1306         :type msg: Message
1307         :returns: Nothing.
1308         """
1309         if self._msg_type:
1310             self.parse_msg[self._msg_type](msg)
1311
1312     def end_message(self, msg):
1313         """Called when message ends. Default implementation does nothing.
1314
1315         :param msg: Message to process.
1316         :type msg: Message
1317         :returns: Nothing.
1318         """
1319
1320
1321 class InputData:
1322     """Input data
1323
1324     The data is extracted from output.xml files generated by Jenkins jobs and
1325     stored in pandas' DataFrames.
1326
1327     The data structure:
1328     - job name
1329       - build number
1330         - metadata
1331           (as described in ExecutionChecker documentation)
1332         - suites
1333           (as described in ExecutionChecker documentation)
1334         - tests
1335           (as described in ExecutionChecker documentation)
1336     """
1337
1338     def __init__(self, spec):
1339         """Initialization.
1340
1341         :param spec: Specification.
1342         :type spec: Specification
1343         """
1344
1345         # Specification:
1346         self._cfg = spec
1347
1348         # Data store:
1349         self._input_data = pd.Series()
1350
1351     @property
1352     def data(self):
1353         """Getter - Input data.
1354
1355         :returns: Input data
1356         :rtype: pandas.Series
1357         """
1358         return self._input_data
1359
1360     def metadata(self, job, build):
1361         """Getter - metadata
1362
1363         :param job: Job which metadata we want.
1364         :param build: Build which metadata we want.
1365         :type job: str
1366         :type build: str
1367         :returns: Metadata
1368         :rtype: pandas.Series
1369         """
1370         return self.data[job][build][u"metadata"]
1371
1372     def suites(self, job, build):
1373         """Getter - suites
1374
1375         :param job: Job which suites we want.
1376         :param build: Build which suites we want.
1377         :type job: str
1378         :type build: str
1379         :returns: Suites.
1380         :rtype: pandas.Series
1381         """
1382         return self.data[job][str(build)][u"suites"]
1383
1384     def tests(self, job, build):
1385         """Getter - tests
1386
1387         :param job: Job which tests we want.
1388         :param build: Build which tests we want.
1389         :type job: str
1390         :type build: str
1391         :returns: Tests.
1392         :rtype: pandas.Series
1393         """
1394         return self.data[job][build][u"tests"]
1395
1396     def _parse_tests(self, job, build, log):
1397         """Process data from robot output.xml file and return JSON structured
1398         data.
1399
1400         :param job: The name of job which build output data will be processed.
1401         :param build: The build which output data will be processed.
1402         :param log: List of log messages.
1403         :type job: str
1404         :type build: dict
1405         :type log: list of tuples (severity, msg)
1406         :returns: JSON data structure.
1407         :rtype: dict
1408         """
1409
1410         metadata = {
1411             u"job": job,
1412             u"build": build
1413         }
1414
1415         with open(build[u"file-name"], u'r') as data_file:
1416             try:
1417                 result = ExecutionResult(data_file)
1418             except errors.DataError as err:
1419                 log.append(
1420                     (u"ERROR", f"Error occurred while parsing output.xml: "
1421                                f"{repr(err)}")
1422                 )
1423                 return None
1424         checker = ExecutionChecker(metadata, self._cfg.mapping,
1425                                    self._cfg.ignore)
1426         result.visit(checker)
1427
1428         return checker.data
1429
1430     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1431         """Download and parse the input data file.
1432
1433         :param pid: PID of the process executing this method.
1434         :param job: Name of the Jenkins job which generated the processed input
1435             file.
1436         :param build: Information about the Jenkins build which generated the
1437             processed input file.
1438         :param repeat: Repeat the download specified number of times if not
1439             successful.
1440         :type pid: int
1441         :type job: str
1442         :type build: dict
1443         :type repeat: int
1444         """
1445
1446         logs = list()
1447
1448         logs.append(
1449             (u"INFO", f"  Processing the job/build: {job}: {build[u'build']}")
1450         )
1451
1452         state = u"failed"
1453         success = False
1454         data = None
1455         do_repeat = repeat
1456         while do_repeat:
1457             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1458                                                    logs)
1459             if success:
1460                 break
1461             do_repeat -= 1
1462         if not success:
1463             logs.append(
1464                 (u"ERROR",
1465                  f"It is not possible to download the input data file from the "
1466                  f"job {job}, build {build[u'build']}, or it is damaged. "
1467                  f"Skipped.")
1468             )
1469         if success:
1470             logs.append(
1471                 (u"INFO",
1472                  f"    Processing data from the build {build[u'build']} ...")
1473             )
1474             data = self._parse_tests(job, build, logs)
1475             if data is None:
1476                 logs.append(
1477                     (u"ERROR",
1478                      f"Input data file from the job {job}, build "
1479                      f"{build[u'build']} is damaged. Skipped.")
1480                 )
1481             else:
1482                 state = u"processed"
1483
1484             try:
1485                 remove(build[u"file-name"])
1486             except OSError as err:
1487                 logs.append(
1488                     ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
1489                               f"{repr(err)}")
1490                 )
1491
1492         # If the time-period is defined in the specification file, remove all
1493         # files which are outside the time period.
1494         timeperiod = self._cfg.input.get(u"time-period", None)
1495         if timeperiod and data:
1496             now = dt.utcnow()
1497             timeperiod = timedelta(int(timeperiod))
1498             metadata = data.get(u"metadata", None)
1499             if metadata:
1500                 generated = metadata.get(u"generated", None)
1501                 if generated:
1502                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1503                     if (now - generated) > timeperiod:
1504                         # Remove the data and the file:
1505                         state = u"removed"
1506                         data = None
1507                         logs.append(
1508                             (u"INFO",
1509                              f"    The build {job}/{build[u'build']} is "
1510                              f"outdated, will be removed.")
1511                         )
1512         logs.append((u"INFO", u"  Done."))
1513
1514         for level, line in logs:
1515             if level == u"INFO":
1516                 logging.info(line)
1517             elif level == u"ERROR":
1518                 logging.error(line)
1519             elif level == u"DEBUG":
1520                 logging.debug(line)
1521             elif level == u"CRITICAL":
1522                 logging.critical(line)
1523             elif level == u"WARNING":
1524                 logging.warning(line)
1525
1526         return {u"data": data, u"state": state, u"job": job, u"build": build}
1527
1528     def download_and_parse_data(self, repeat=1):
1529         """Download the input data files, parse input data from input files and
1530         store in pandas' Series.
1531
1532         :param repeat: Repeat the download specified number of times if not
1533             successful.
1534         :type repeat: int
1535         """
1536
1537         logging.info(u"Downloading and parsing input files ...")
1538
1539         for job, builds in self._cfg.builds.items():
1540             for build in builds:
1541
1542                 result = self._download_and_parse_build(job, build, repeat)
1543                 build_nr = result[u"build"][u"build"]
1544
1545                 if result[u"data"]:
1546                     data = result[u"data"]
1547                     build_data = pd.Series({
1548                         u"metadata": pd.Series(
1549                             list(data[u"metadata"].values()),
1550                             index=list(data[u"metadata"].keys())
1551                         ),
1552                         u"suites": pd.Series(
1553                             list(data[u"suites"].values()),
1554                             index=list(data[u"suites"].keys())
1555                         ),
1556                         u"tests": pd.Series(
1557                             list(data[u"tests"].values()),
1558                             index=list(data[u"tests"].keys())
1559                         )
1560                     })
1561
1562                     if self._input_data.get(job, None) is None:
1563                         self._input_data[job] = pd.Series()
1564                     self._input_data[job][str(build_nr)] = build_data
1565
1566                     self._cfg.set_input_file_name(
1567                         job, build_nr, result[u"build"][u"file-name"])
1568
1569                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1570
1571                 mem_alloc = \
1572                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1573                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1574
1575         logging.info(u"Done.")
1576
1577     def process_local_file(self, local_file, job=u"local", build_nr=1,
1578                            replace=True):
1579         """Process local XML file given as a command-line parameter.
1580
1581         :param local_file: The file to process.
1582         :param job: Job name.
1583         :param build_nr: Build number.
1584         :param replace: If True, the information about jobs and builds is
1585             replaced by the new one, otherwise the new jobs and builds are
1586             added.
1587         :type local_file: str
1588         :type job: str
1589         :type build_nr: int
1590         :type replace: bool
1591         :raises: PresentationError if an error occurs.
1592         """
1593         if not isfile(local_file):
1594             raise PresentationError(f"The file {local_file} does not exist.")
1595
1596         build = {
1597             u"build": build_nr,
1598             u"status": u"failed",
1599             u"file-name": local_file
1600         }
1601         if replace:
1602             self._cfg.builds = dict()
1603         self._cfg.add_build(job, build)
1604
1605         logging.info(f"Processing {job}: {build_nr:2d}: {local_file}")
1606         data = self._parse_tests(job, build, list())
1607         if data is None:
1608             raise PresentationError(
1609                 f"Error occurred while parsing the file {local_file}"
1610             )
1611
1612         build_data = pd.Series({
1613             u"metadata": pd.Series(
1614                 list(data[u"metadata"].values()),
1615                 index=list(data[u"metadata"].keys())
1616             ),
1617             u"suites": pd.Series(
1618                 list(data[u"suites"].values()),
1619                 index=list(data[u"suites"].keys())
1620             ),
1621             u"tests": pd.Series(
1622                 list(data[u"tests"].values()),
1623                 index=list(data[u"tests"].keys())
1624             )
1625         })
1626
1627         if self._input_data.get(job, None) is None:
1628             self._input_data[job] = pd.Series()
1629         self._input_data[job][str(build_nr)] = build_data
1630
1631         self._cfg.set_input_state(job, build_nr, u"processed")
1632
1633     def process_local_directory(self, local_dir, replace=True):
1634         """Process local directory with XML file(s). The directory is processed
1635         as a 'job' and the XML files in it as builds.
1636         If the given directory contains only sub-directories, these
1637         sub-directories processed as jobs and corresponding XML files as builds
1638         of their job.
1639
1640         :param local_dir: Local directory to process.
1641         :param replace: If True, the information about jobs and builds is
1642             replaced by the new one, otherwise the new jobs and builds are
1643             added.
1644         :type local_dir: str
1645         :type replace: bool
1646         """
1647         if not isdir(local_dir):
1648             raise PresentationError(
1649                 f"The directory {local_dir} does not exist."
1650             )
1651
1652         # Check if the given directory includes only files, or only directories
1653         _, dirnames, filenames = next(walk(local_dir))
1654
1655         if filenames and not dirnames:
1656             filenames.sort()
1657             # local_builds:
1658             # key: dir (job) name, value: list of file names (builds)
1659             local_builds = {
1660                 local_dir: [join(local_dir, name) for name in filenames]
1661             }
1662
1663         elif dirnames and not filenames:
1664             dirnames.sort()
1665             # local_builds:
1666             # key: dir (job) name, value: list of file names (builds)
1667             local_builds = dict()
1668             for dirname in dirnames:
1669                 builds = [
1670                     join(local_dir, dirname, name)
1671                     for name in listdir(join(local_dir, dirname))
1672                     if isfile(join(local_dir, dirname, name))
1673                 ]
1674                 if builds:
1675                     local_builds[dirname] = sorted(builds)
1676
1677         elif not filenames and not dirnames:
1678             raise PresentationError(f"The directory {local_dir} is empty.")
1679         else:
1680             raise PresentationError(
1681                 f"The directory {local_dir} can include only files or only "
1682                 f"directories, not both.\nThe directory {local_dir} includes "
1683                 f"file(s):\n{filenames}\nand directories:\n{dirnames}"
1684             )
1685
1686         if replace:
1687             self._cfg.builds = dict()
1688
1689         for job, files in local_builds.items():
1690             for idx, local_file in enumerate(files):
1691                 self.process_local_file(local_file, job, idx + 1, replace=False)
1692
1693     @staticmethod
1694     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1695         """Return the index of character in the string which is the end of tag.
1696
1697         :param tag_filter: The string where the end of tag is being searched.
1698         :param start: The index where the searching is stated.
1699         :param closer: The character which is the tag closer.
1700         :type tag_filter: str
1701         :type start: int
1702         :type closer: str
1703         :returns: The index of the tag closer.
1704         :rtype: int
1705         """
1706         try:
1707             idx_opener = tag_filter.index(closer, start)
1708             return tag_filter.index(closer, idx_opener + 1)
1709         except ValueError:
1710             return None
1711
1712     @staticmethod
1713     def _condition(tag_filter):
1714         """Create a conditional statement from the given tag filter.
1715
1716         :param tag_filter: Filter based on tags from the element specification.
1717         :type tag_filter: str
1718         :returns: Conditional statement which can be evaluated.
1719         :rtype: str
1720         """
1721         index = 0
1722         while True:
1723             index = InputData._end_of_tag(tag_filter, index)
1724             if index is None:
1725                 return tag_filter
1726             index += 1
1727             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1728
1729     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1730                     continue_on_error=False):
1731         """Filter required data from the given jobs and builds.
1732
1733         The output data structure is:
1734         - job 1
1735           - build 1
1736             - test (or suite) 1 ID:
1737               - param 1
1738               - param 2
1739               ...
1740               - param n
1741             ...
1742             - test (or suite) n ID:
1743             ...
1744           ...
1745           - build n
1746         ...
1747         - job n
1748
1749         :param element: Element which will use the filtered data.
1750         :param params: Parameters which will be included in the output. If None,
1751             all parameters are included.
1752         :param data: If not None, this data is used instead of data specified
1753             in the element.
1754         :param data_set: The set of data to be filtered: tests, suites,
1755             metadata.
1756         :param continue_on_error: Continue if there is error while reading the
1757             data. The Item will be empty then
1758         :type element: pandas.Series
1759         :type params: list
1760         :type data: dict
1761         :type data_set: str
1762         :type continue_on_error: bool
1763         :returns: Filtered data.
1764         :rtype pandas.Series
1765         """
1766
1767         try:
1768             if data_set == "suites":
1769                 cond = u"True"
1770             elif element[u"filter"] in (u"all", u"template"):
1771                 cond = u"True"
1772             else:
1773                 cond = InputData._condition(element[u"filter"])
1774             logging.debug(f"   Filter: {cond}")
1775         except KeyError:
1776             logging.error(u"  No filter defined.")
1777             return None
1778
1779         if params is None:
1780             params = element.get(u"parameters", None)
1781             if params:
1782                 params.append(u"type")
1783
1784         data_to_filter = data if data else element[u"data"]
1785         data = pd.Series()
1786         try:
1787             for job, builds in data_to_filter.items():
1788                 data[job] = pd.Series()
1789                 for build in builds:
1790                     data[job][str(build)] = pd.Series()
1791                     try:
1792                         data_dict = dict(
1793                             self.data[job][str(build)][data_set].items())
1794                     except KeyError:
1795                         if continue_on_error:
1796                             continue
1797                         return None
1798
1799                     for test_id, test_data in data_dict.items():
1800                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1801                             data[job][str(build)][test_id] = pd.Series()
1802                             if params is None:
1803                                 for param, val in test_data.items():
1804                                     data[job][str(build)][test_id][param] = val
1805                             else:
1806                                 for param in params:
1807                                     try:
1808                                         data[job][str(build)][test_id][param] =\
1809                                             test_data[param]
1810                                     except KeyError:
1811                                         data[job][str(build)][test_id][param] =\
1812                                             u"No Data"
1813             return data
1814
1815         except (KeyError, IndexError, ValueError) as err:
1816             logging.error(
1817                 f"Missing mandatory parameter in the element specification: "
1818                 f"{repr(err)}"
1819             )
1820             return None
1821         except AttributeError as err:
1822             logging.error(repr(err))
1823             return None
1824         except SyntaxError as err:
1825             logging.error(
1826                 f"The filter {cond} is not correct. Check if all tags are "
1827                 f"enclosed by apostrophes.\n{repr(err)}"
1828             )
1829             return None
1830
1831     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1832                              continue_on_error=False):
1833         """Filter required data from the given jobs and builds.
1834
1835         The output data structure is:
1836         - job 1
1837           - build 1
1838             - test (or suite) 1 ID:
1839               - param 1
1840               - param 2
1841               ...
1842               - param n
1843             ...
1844             - test (or suite) n ID:
1845             ...
1846           ...
1847           - build n
1848         ...
1849         - job n
1850
1851         :param element: Element which will use the filtered data.
1852         :param params: Parameters which will be included in the output. If None,
1853         all parameters are included.
1854         :param data_set: The set of data to be filtered: tests, suites,
1855         metadata.
1856         :param continue_on_error: Continue if there is error while reading the
1857         data. The Item will be empty then
1858         :type element: pandas.Series
1859         :type params: list
1860         :type data_set: str
1861         :type continue_on_error: bool
1862         :returns: Filtered data.
1863         :rtype pandas.Series
1864         """
1865
1866         include = element.get(u"include", None)
1867         if not include:
1868             logging.warning(u"No tests to include, skipping the element.")
1869             return None
1870
1871         if params is None:
1872             params = element.get(u"parameters", None)
1873             if params:
1874                 params.append(u"type")
1875
1876         data = pd.Series()
1877         try:
1878             for job, builds in element[u"data"].items():
1879                 data[job] = pd.Series()
1880                 for build in builds:
1881                     data[job][str(build)] = pd.Series()
1882                     for test in include:
1883                         try:
1884                             reg_ex = re.compile(str(test).lower())
1885                             for test_id in self.data[job][
1886                                     str(build)][data_set].keys():
1887                                 if re.match(reg_ex, str(test_id).lower()):
1888                                     test_data = self.data[job][
1889                                         str(build)][data_set][test_id]
1890                                     data[job][str(build)][test_id] = pd.Series()
1891                                     if params is None:
1892                                         for param, val in test_data.items():
1893                                             data[job][str(build)][test_id]\
1894                                                 [param] = val
1895                                     else:
1896                                         for param in params:
1897                                             try:
1898                                                 data[job][str(build)][
1899                                                     test_id][param] = \
1900                                                     test_data[param]
1901                                             except KeyError:
1902                                                 data[job][str(build)][
1903                                                     test_id][param] = u"No Data"
1904                         except KeyError as err:
1905                             logging.error(repr(err))
1906                             if continue_on_error:
1907                                 continue
1908                             return None
1909             return data
1910
1911         except (KeyError, IndexError, ValueError) as err:
1912             logging.error(
1913                 f"Missing mandatory parameter in the element "
1914                 f"specification: {repr(err)}"
1915             )
1916             return None
1917         except AttributeError as err:
1918             logging.error(repr(err))
1919             return None
1920
1921     @staticmethod
1922     def merge_data(data):
1923         """Merge data from more jobs and builds to a simple data structure.
1924
1925         The output data structure is:
1926
1927         - test (suite) 1 ID:
1928           - param 1
1929           - param 2
1930           ...
1931           - param n
1932         ...
1933         - test (suite) n ID:
1934         ...
1935
1936         :param data: Data to merge.
1937         :type data: pandas.Series
1938         :returns: Merged data.
1939         :rtype: pandas.Series
1940         """
1941
1942         logging.info(u"    Merging data ...")
1943
1944         merged_data = pd.Series()
1945         for builds in data.values:
1946             for item in builds.values:
1947                 for item_id, item_data in item.items():
1948                     merged_data[item_id] = item_data
1949         return merged_data
1950
1951     def print_all_oper_data(self):
1952         """Print all operational data to console.
1953         """
1954
1955         tbl_hdr = (
1956             u"Name",
1957             u"Nr of Vectors",
1958             u"Nr of Packets",
1959             u"Suspends",
1960             u"Cycles per Packet",
1961             u"Average Vector Size"
1962         )
1963
1964         for job in self._input_data.values:
1965             for build in job.values:
1966                 for test_id, test_data in build[u"tests"].items():
1967                     print(f"{test_id}")
1968                     if test_data.get(u"show-run", None) is None:
1969                         continue
1970                     for dut_name, data in test_data[u"show-run"].items():
1971                         if data.get(u"threads", None) is None:
1972                             continue
1973                         print(f"Host IP: {data.get(u'host', '')}, "
1974                               f"Socket: {data.get(u'socket', '')}")
1975                         for thread_nr, thread in data[u"threads"].items():
1976                             txt_table = prettytable.PrettyTable(tbl_hdr)
1977                             avg = 0.0
1978                             for row in thread:
1979                                 txt_table.add_row(row)
1980                                 avg += row[-1]
1981                             if len(thread) == 0:
1982                                 avg = u""
1983                             else:
1984                                 avg = f", Average Vector Size per Node: " \
1985                                       f"{(avg / len(thread)):.2f}"
1986                             th_name = u"main" if thread_nr == 0 \
1987                                 else f"worker_{thread_nr}"
1988                             print(f"{dut_name}, {th_name}{avg}")
1989                             txt_table.float_format = u".2"
1990                             txt_table.align = u"r"
1991                             txt_table.align[u"Name"] = u"l"
1992                             print(f"{txt_table.get_string()}\n")