resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove
  29 from datetime import datetime as dt
  30 from datetime import timedelta
  31 from json import loads
  32
  33 import hdrh.histogram
  34 import hdrh.codec
  35 import prettytable
  36 import pandas as pd
  37
  38 from robot.api import ExecutionResult, ResultVisitor
  39 from robot import errors
  40
  41 from resources.libraries.python import jumpavg
  42 from input_data_files import download_and_unzip_data_file
  43
  44
  45 # Separator used in file names
  46 SEPARATOR = u"__"
  47
  48
  49 class ExecutionChecker(ResultVisitor):
  50     """Class to traverse through the test suite structure.
  51
  52     The functionality implemented in this class generates a json structure:
  53
  54     Performance tests:
  55
  56     {
  57         "metadata": {
  58             "generated": "Timestamp",
  59             "version": "SUT version",
  60             "job": "Jenkins job name",
  61             "build": "Information about the build"
  62         },
  63         "suites": {
  64             "Suite long name 1": {
  65                 "name": Suite name,
  66                 "doc": "Suite 1 documentation",
  67                 "parent": "Suite 1 parent",
  68                 "level": "Level of the suite in the suite hierarchy"
  69             }
  70             "Suite long name N": {
  71                 "name": Suite name,
  72                 "doc": "Suite N documentation",
  73                 "parent": "Suite 2 parent",
  74                 "level": "Level of the suite in the suite hierarchy"
  75             }
  76         }
  77         "tests": {
  78             # NDRPDR tests:
  79             "ID": {
  80                 "name": "Test name",
  81                 "parent": "Name of the parent of the test",
  82                 "doc": "Test documentation",
  83                 "msg": "Test message",
  84                 "conf-history": "DUT1 and DUT2 VAT History",
  85                 "show-run": "Show Run",
  86                 "tags": ["tag 1", "tag 2", "tag n"],
  87                 "type": "NDRPDR",
  88                 "status": "PASS" | "FAIL",
  89                 "throughput": {
  90                     "NDR": {
  91                         "LOWER": float,
  92                         "UPPER": float
  93                     },
  94                     "PDR": {
  95                         "LOWER": float,
  96                         "UPPER": float
  97                     }
  98                 },
  99                 "latency": {
 100                     "NDR": {
 101                         "direction1": {
 102                             "min": float,
 103                             "avg": float,
 104                             "max": float,
 105                             "hdrh": str
 106                         },
 107                         "direction2": {
 108                             "min": float,
 109                             "avg": float,
 110                             "max": float,
 111                             "hdrh": str
 112                         }
 113                     },
 114                     "PDR": {
 115                         "direction1": {
 116                             "min": float,
 117                             "avg": float,
 118                             "max": float,
 119                             "hdrh": str
 120                         },
 121                         "direction2": {
 122                             "min": float,
 123                             "avg": float,
 124                             "max": float,
 125                             "hdrh": str
 126                         }
 127                     }
 128                 }
 129             }
 130
 131             # TCP tests:
 132             "ID": {
 133                 "name": "Test name",
 134                 "parent": "Name of the parent of the test",
 135                 "doc": "Test documentation",
 136                 "msg": "Test message",
 137                 "tags": ["tag 1", "tag 2", "tag n"],
 138                 "type": "TCP",
 139                 "status": "PASS" | "FAIL",
 140                 "result": int
 141             }
 142
 143             # MRR, BMRR tests:
 144             "ID": {
 145                 "name": "Test name",
 146                 "parent": "Name of the parent of the test",
 147                 "doc": "Test documentation",
 148                 "msg": "Test message",
 149                 "tags": ["tag 1", "tag 2", "tag n"],
 150                 "type": "MRR" | "BMRR",
 151                 "status": "PASS" | "FAIL",
 152                 "result": {
 153                     "receive-rate": float,
 154                     # Average of a list, computed using AvgStdevStats.
 155                     # In CSIT-1180, replace with List[float].
 156                 }
 157             }
 158
 159             "ID" {
 160                 # next test
 161             }
 162         }
 163     }
 164
 165
 166     Functional tests:
 167
 168     {
 169         "metadata": {  # Optional
 170             "version": "VPP version",
 171             "job": "Jenkins job name",
 172             "build": "Information about the build"
 173         },
 174         "suites": {
 175             "Suite name 1": {
 176                 "doc": "Suite 1 documentation",
 177                 "parent": "Suite 1 parent",
 178                 "level": "Level of the suite in the suite hierarchy"
 179             }
 180             "Suite name N": {
 181                 "doc": "Suite N documentation",
 182                 "parent": "Suite 2 parent",
 183                 "level": "Level of the suite in the suite hierarchy"
 184             }
 185         }
 186         "tests": {
 187             "ID": {
 188                 "name": "Test name",
 189                 "parent": "Name of the parent of the test",
 190                 "doc": "Test documentation"
 191                 "msg": "Test message"
 192                 "tags": ["tag 1", "tag 2", "tag n"],
 193                 "conf-history": "DUT1 and DUT2 VAT History"
 194                 "show-run": "Show Run"
 195                 "status": "PASS" | "FAIL"
 196             },
 197             "ID" {
 198                 # next test
 199             }
 200         }
 201     }
 202
 203     .. note:: ID is the lowercase full path to the test.
 204     """
 205
 206     REGEX_PLR_RATE = re.compile(
 207         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 208         r'PLRsearch upper bound::?\s(\d+.\d+)'
 209     )
 210     REGEX_NDRPDR_RATE = re.compile(
 211         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 212         r'NDR_UPPER:\s(\d+.\d+).*\n'
 213         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 214         r'PDR_UPPER:\s(\d+.\d+)'
 215     )
 216     REGEX_PERF_MSG_INFO = re.compile(
 217         r'NDR_LOWER:\s(\d+.\d+)\s([a-zA-Z]*).*\s(\d+.\d+)\s([a-zA-Z]*).*\n'
 218         r'LATENCY.*\[\'(.*)\', \'(.*)\'\].*\n'
 219         r'NDR_UPPER:\s(\d+.\d+)\s([a-zA-Z]*).*\s(\d+.\d+)\s([a-zA-Z]*).*\n'
 220         r'PDR_LOWER:\s(\d+.\d+)\s([a-zA-Z]*).*\s(\d+.\d+)\s([a-zA-Z]*).*\n'
 221         r'LATENCY.*\[\'(.*)\', \'(.*)\'\].*\n'
 222         r'PDR_UPPER:\s(\d+.\d+)\s([a-zA-Z]*).*\s(\d+.\d+)\s([a-zA-Z]*)'
 223     )
 224     # TODO: Remove when not needed
 225     REGEX_NDRPDR_LAT_BASE = re.compile(
 226         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 227         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 228     )
 229     REGEX_NDRPDR_LAT = re.compile(
 230         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 231         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 232         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 233         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 234         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 235         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 236     )
 237     # TODO: Remove when not needed
 238     REGEX_NDRPDR_LAT_LONG = re.compile(
 239         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 240         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 241         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 242         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 243         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 244         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 245         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 246         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 247         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 248     )
 249     REGEX_VERSION_VPP = re.compile(
 250         r"(return STDOUT Version:\s*|"
 251         r"VPP Version:\s*|VPP version:\s*)(.*)"
 252     )
 253     REGEX_VERSION_DPDK = re.compile(
 254         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 255     )
 256     REGEX_TCP = re.compile(
 257         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 258     )
 259     REGEX_MRR = re.compile(
 260         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 261         r'tx\s(\d*),\srx\s(\d*)'
 262     )
 263     REGEX_BMRR = re.compile(
 264         r'Maximum Receive Rate trial results'
 265         r' in packets per second: \[(.*)\]'
 266     )
 267     REGEX_RECONF_LOSS = re.compile(
 268         r'Packets lost due to reconfig: (\d*)'
 269     )
 270     REGEX_RECONF_TIME = re.compile(
 271         r'Implied time lost: (\d*.[\de-]*)'
 272     )
 273     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 274
 275     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 276
 277     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 278
 279     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 280
 281     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 282
 283     def __init__(self, metadata, mapping, ignore):
 284         """Initialisation.
 285
 286         :param metadata: Key-value pairs to be included in "metadata" part of
 287             JSON structure.
 288         :param mapping: Mapping of the old names of test cases to the new
 289             (actual) one.
 290         :param ignore: List of TCs to be ignored.
 291         :type metadata: dict
 292         :type mapping: dict
 293         :type ignore: list
 294         """
 295
 296         # Type of message to parse out from the test messages
 297         self._msg_type = None
 298
 299         # VPP version
 300         self._version = None
 301
 302         # Timestamp
 303         self._timestamp = None
 304
 305         # Testbed. The testbed is identified by TG node IP address.
 306         self._testbed = None
 307
 308         # Mapping of TCs long names
 309         self._mapping = mapping
 310
 311         # Ignore list
 312         self._ignore = ignore
 313
 314         # Number of PAPI History messages found:
 315         # 0 - no message
 316         # 1 - PAPI History of DUT1
 317         # 2 - PAPI History of DUT2
 318         self._conf_history_lookup_nr = 0
 319
 320         self._sh_run_counter = 0
 321
 322         # Test ID of currently processed test- the lowercase full path to the
 323         # test
 324         self._test_id = None
 325
 326         # The main data structure
 327         self._data = {
 328             u"metadata": OrderedDict(),
 329             u"suites": OrderedDict(),
 330             u"tests": OrderedDict()
 331         }
 332
 333         # Save the provided metadata
 334         for key, val in metadata.items():
 335             self._data[u"metadata"][key] = val
 336
 337         # Dictionary defining the methods used to parse different types of
 338         # messages
 339         self.parse_msg = {
 340             u"timestamp": self._get_timestamp,
 341             u"vpp-version": self._get_vpp_version,
 342             u"dpdk-version": self._get_dpdk_version,
 343             # TODO: Remove when not needed:
 344             u"teardown-vat-history": self._get_vat_history,
 345             u"teardown-papi-history": self._get_papi_history,
 346             u"test-show-runtime": self._get_show_run,
 347             u"testbed": self._get_testbed
 348         }
 349
 350     @property
 351     def data(self):
 352         """Getter - Data parsed from the XML file.
 353
 354         :returns: Data parsed from the XML file.
 355         :rtype: dict
 356         """
 357         return self._data
 358
 359     def _get_data_from_perf_test_msg(self, msg):
 360         """Get
 361             - NDR_LOWER
 362             - LATENCY
 363             - NDR_UPPER
 364             - PDR_LOWER
 365             - LATENCY
 366             - PDR_UPPER
 367         from message of NDRPDR performance tests.
 368
 369         :param msg: Message to be processed.
 370         :type msg: str
 371         :returns: Processed message or original message if a problem occurs.
 372         :rtype: str
 373         """
 374
 375         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 376         if not groups or groups.lastindex != 20:
 377             return msg
 378
 379         try:
 380             data = {
 381                 u"ndr_low": float(groups.group(1)),
 382                 u"ndr_low_unit": groups.group(2),
 383                 u"ndr_low_b": float(groups.group(3)),
 384                 u"ndr_low_b_unit": groups.group(4),
 385                 u"ndr_lat_1": groups.group(5),
 386                 u"ndr_lat_2": groups.group(6),
 387                 u"ndr_up": float(groups.group(7)),
 388                 u"ndr_up_unit": groups.group(8),
 389                 u"ndr_up_b": float(groups.group(9)),
 390                 u"ndr_up_b_unit": groups.group(10),
 391                 u"pdr_low": float(groups.group(11)),
 392                 u"pdr_low_unit": groups.group(12),
 393                 u"pdr_low_b": float(groups.group(13)),
 394                 u"pdr_low_b_unit": groups.group(14),
 395                 u"pdr_lat_1": groups.group(15),
 396                 u"pdr_lat_2": groups.group(16),
 397                 u"pdr_up": float(groups.group(17)),
 398                 u"pdr_up_unit": groups.group(18),
 399                 u"pdr_up_b": float(groups.group(19)),
 400                 u"pdr_up_b_unit": groups.group(20)
 401             }
 402         except (AttributeError, IndexError, ValueError, KeyError):
 403             return msg
 404
 405         def _process_lat(in_str_1, in_str_2):
 406             """Extract min, avg, max values from latency string.
 407
 408             :param in_str_1: Latency string for one direction produced by robot
 409                 framework.
 410             :param in_str_2: Latency string for second direction produced by
 411                 robot framework.
 412             :type in_str_1: str
 413             :type in_str_2: str
 414             :returns: Processed latency string or original string if a problem
 415                 occurs.
 416             :rtype: tuple(str, str)
 417             """
 418             in_list_1 = in_str_1.split('/', 3)
 419             if len(in_list_1) < 3:
 420                 return u"Not Measured.", u"Not Measured."
 421
 422             in_list_2 = in_str_2.split('/', 3)
 423             if len(in_list_2) < 3:
 424                 return u"Not Measured.", u"Not Measured."
 425
 426             hdr_lat_1 = u""
 427             if len(in_list_1) == 4:
 428                 in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 429                 try:
 430                     hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 431                 except hdrh.codec.HdrLengthException:
 432                     pass
 433             hdr_lat_2 = u""
 434             if len(in_list_2) == 4:
 435                 in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 436                 try:
 437                     hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 438                 except hdrh.codec.HdrLengthException:
 439                     pass
 440
 441             hdr_lat = u"Not Measured."
 442             if hdr_lat_1 and hdr_lat_2:
 443                 hdr_lat = (
 444                     f"50%/90%/99%/99.9%, "
 445                     f"{hdr_lat_1.get_value_at_percentile(50.0)}/"
 446                     f"{hdr_lat_1.get_value_at_percentile(90.0)}/"
 447                     f"{hdr_lat_1.get_value_at_percentile(99.0)}/"
 448                     f"{hdr_lat_1.get_value_at_percentile(99.9)}, "
 449                     f"{hdr_lat_2.get_value_at_percentile(50.0)}/"
 450                     f"{hdr_lat_2.get_value_at_percentile(90.0)}/"
 451                     f"{hdr_lat_2.get_value_at_percentile(99.0)}/"
 452                     f"{hdr_lat_2.get_value_at_percentile(99.9)} "
 453                     f"uSec."
 454                 )
 455
 456             return (
 457                 f"Min/Avg/Max, "
 458                 f"{in_list_1[0]}/{in_list_1[1]}/{in_list_1[2]}, "
 459                 f"{in_list_2[0]}/{in_list_2[1]}/{in_list_2[2]} uSec.",
 460                 hdr_lat
 461             )
 462
 463         try:
 464             pdr_lat = _process_lat(data[u'pdr_lat_1'], data[u'pdr_lat_2'])
 465             ndr_lat = _process_lat(data[u'ndr_lat_1'], data[u'ndr_lat_2'])
 466             return (
 467                 f"NDR Throughput: {(data[u'ndr_low'] / 1e6):.2f} "
 468                 f"M{data[u'ndr_low_unit']}, "
 469                 f"{data[u'ndr_low_b']:.2f} {data[u'ndr_low_b_unit']}.\n"
 470                 f"One-Way Latency at NDR: {ndr_lat[0]}\n"
 471                 f"One-Way Latency at NDR by percentiles: {ndr_lat[1]}\n"
 472                 f"PDR Throughput: {(data[u'pdr_low'] / 1e6):.2f} "
 473                 f"M{data[u'pdr_low_unit']}, "
 474                 f"{data[u'pdr_low_b']:.2f} {data[u'pdr_low_b_unit']}.\n"
 475                 f"One-Way Latency at PDR: {pdr_lat[0]}\n"
 476                 f"One-Way Latency at PDR by percentiles: {pdr_lat[1]}"
 477             )
 478         except (AttributeError, IndexError, ValueError, KeyError):
 479             return msg
 480
 481     def _get_testbed(self, msg):
 482         """Called when extraction of testbed IP is required.
 483         The testbed is identified by TG node IP address.
 484
 485         :param msg: Message to process.
 486         :type msg: Message
 487         :returns: Nothing.
 488         """
 489
 490         if msg.message.count(u"Setup of TG node") or \
 491                 msg.message.count(u"Setup of node TG host"):
 492             reg_tg_ip = re.compile(
 493                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 494             try:
 495                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 496             except (KeyError, ValueError, IndexError, AttributeError):
 497                 pass
 498             finally:
 499                 self._data[u"metadata"][u"testbed"] = self._testbed
 500                 self._msg_type = None
 501
 502     def _get_vpp_version(self, msg):
 503         """Called when extraction of VPP version is required.
 504
 505         :param msg: Message to process.
 506         :type msg: Message
 507         :returns: Nothing.
 508         """
 509
 510         if msg.message.count(u"return STDOUT Version:") or \
 511             msg.message.count(u"VPP Version:") or \
 512             msg.message.count(u"VPP version:"):
 513             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 514                                 group(2))
 515             self._data[u"metadata"][u"version"] = self._version
 516             self._msg_type = None
 517
 518     def _get_dpdk_version(self, msg):
 519         """Called when extraction of DPDK version is required.
 520
 521         :param msg: Message to process.
 522         :type msg: Message
 523         :returns: Nothing.
 524         """
 525
 526         if msg.message.count(u"DPDK Version:"):
 527             try:
 528                 self._version = str(re.search(
 529                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 530                 self._data[u"metadata"][u"version"] = self._version
 531             except IndexError:
 532                 pass
 533             finally:
 534                 self._msg_type = None
 535
 536     def _get_timestamp(self, msg):
 537         """Called when extraction of timestamp is required.
 538
 539         :param msg: Message to process.
 540         :type msg: Message
 541         :returns: Nothing.
 542         """
 543
 544         self._timestamp = msg.timestamp[:14]
 545         self._data[u"metadata"][u"generated"] = self._timestamp
 546         self._msg_type = None
 547
 548     def _get_vat_history(self, msg):
 549         """Called when extraction of VAT command history is required.
 550
 551         TODO: Remove when not needed.
 552
 553         :param msg: Message to process.
 554         :type msg: Message
 555         :returns: Nothing.
 556         """
 557         if msg.message.count(u"VAT command history:"):
 558             self._conf_history_lookup_nr += 1
 559             if self._conf_history_lookup_nr == 1:
 560                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 561             else:
 562                 self._msg_type = None
 563             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 564                           r"VAT command history:", u"",
 565                           msg.message, count=1).replace(u'\n', u' |br| ').\
 566                 replace(u'"', u"'")
 567
 568             self._data[u"tests"][self._test_id][u"conf-history"] += (
 569                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 570             )
 571
 572     def _get_papi_history(self, msg):
 573         """Called when extraction of PAPI command history is required.
 574
 575         :param msg: Message to process.
 576         :type msg: Message
 577         :returns: Nothing.
 578         """
 579         if msg.message.count(u"PAPI command history:"):
 580             self._conf_history_lookup_nr += 1
 581             if self._conf_history_lookup_nr == 1:
 582                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 583             else:
 584                 self._msg_type = None
 585             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 586                           r"PAPI command history:", u"",
 587                           msg.message, count=1).replace(u'\n', u' |br| ').\
 588                 replace(u'"', u"'")
 589             self._data[u"tests"][self._test_id][u"conf-history"] += (
 590                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 591             )
 592
 593     def _get_show_run(self, msg):
 594         """Called when extraction of VPP operational data (output of CLI command
 595         Show Runtime) is required.
 596
 597         :param msg: Message to process.
 598         :type msg: Message
 599         :returns: Nothing.
 600         """
 601
 602         if not msg.message.count(u"stats runtime"):
 603             return
 604
 605         # Temporary solution
 606         if self._sh_run_counter > 1:
 607             return
 608
 609         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 610             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 611
 612         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 613         if not groups:
 614             return
 615         try:
 616             host = groups.group(1)
 617         except (AttributeError, IndexError):
 618             host = u""
 619         try:
 620             sock = groups.group(2)
 621         except (AttributeError, IndexError):
 622             sock = u""
 623
 624         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 625                         replace(u"'", u'"').replace(u'b"', u'"').
 626                         replace(u'u"', u'"').split(u":", 1)[1])
 627
 628         try:
 629             threads_nr = len(runtime[0][u"clocks"])
 630         except (IndexError, KeyError):
 631             return
 632
 633         dut = u"DUT{nr}".format(
 634             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 635
 636         oper = {
 637             u"host": host,
 638             u"socket": sock,
 639             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 640         }
 641
 642         for item in runtime:
 643             for idx in range(threads_nr):
 644                 if item[u"vectors"][idx] > 0:
 645                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 646                 elif item[u"calls"][idx] > 0:
 647                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 648                 elif item[u"suspends"][idx] > 0:
 649                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 650                 else:
 651                     clocks = 0.0
 652
 653                 if item[u"calls"][idx] > 0:
 654                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 655                 else:
 656                     vectors_call = 0.0
 657
 658                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 659                         int(item[u"suspends"][idx]):
 660                     oper[u"threads"][idx].append([
 661                         item[u"name"],
 662                         item[u"calls"][idx],
 663                         item[u"vectors"][idx],
 664                         item[u"suspends"][idx],
 665                         clocks,
 666                         vectors_call
 667                     ])
 668
 669         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 670
 671     def _get_ndrpdr_throughput(self, msg):
 672         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 673         message.
 674
 675         :param msg: The test message to be parsed.
 676         :type msg: str
 677         :returns: Parsed data as a dict and the status (PASS/FAIL).
 678         :rtype: tuple(dict, str)
 679         """
 680
 681         throughput = {
 682             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 683             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 684         }
 685         status = u"FAIL"
 686         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 687
 688         if groups is not None:
 689             try:
 690                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 691                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 692                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 693                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 694                 status = u"PASS"
 695             except (IndexError, ValueError):
 696                 pass
 697
 698         return throughput, status
 699
 700     def _get_plr_throughput(self, msg):
 701         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 702         message.
 703
 704         :param msg: The test message to be parsed.
 705         :type msg: str
 706         :returns: Parsed data as a dict and the status (PASS/FAIL).
 707         :rtype: tuple(dict, str)
 708         """
 709
 710         throughput = {
 711             u"LOWER": -1.0,
 712             u"UPPER": -1.0
 713         }
 714         status = u"FAIL"
 715         groups = re.search(self.REGEX_PLR_RATE, msg)
 716
 717         if groups is not None:
 718             try:
 719                 throughput[u"LOWER"] = float(groups.group(1))
 720                 throughput[u"UPPER"] = float(groups.group(2))
 721                 status = u"PASS"
 722             except (IndexError, ValueError):
 723                 pass
 724
 725         return throughput, status
 726
 727     def _get_ndrpdr_latency(self, msg):
 728         """Get LATENCY from the test message.
 729
 730         :param msg: The test message to be parsed.
 731         :type msg: str
 732         :returns: Parsed data as a dict and the status (PASS/FAIL).
 733         :rtype: tuple(dict, str)
 734         """
 735         latency_default = {
 736             u"min": -1.0,
 737             u"avg": -1.0,
 738             u"max": -1.0,
 739             u"hdrh": u""
 740         }
 741         latency = {
 742             u"NDR": {
 743                 u"direction1": copy.copy(latency_default),
 744                 u"direction2": copy.copy(latency_default)
 745             },
 746             u"PDR": {
 747                 u"direction1": copy.copy(latency_default),
 748                 u"direction2": copy.copy(latency_default)
 749             },
 750             u"LAT0": {
 751                 u"direction1": copy.copy(latency_default),
 752                 u"direction2": copy.copy(latency_default)
 753             },
 754             u"PDR10": {
 755                 u"direction1": copy.copy(latency_default),
 756                 u"direction2": copy.copy(latency_default)
 757             },
 758             u"PDR50": {
 759                 u"direction1": copy.copy(latency_default),
 760                 u"direction2": copy.copy(latency_default)
 761             },
 762             u"PDR90": {
 763                 u"direction1": copy.copy(latency_default),
 764                 u"direction2": copy.copy(latency_default)
 765             },
 766         }
 767
 768         # TODO: Rewrite when long and base are not needed
 769         groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
 770         if groups is None:
 771             groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 772         if groups is None:
 773             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 774         if groups is None:
 775             return latency, u"FAIL"
 776
 777         def process_latency(in_str):
 778             """Return object with parsed latency values.
 779
 780             TODO: Define class for the return type.
 781
 782             :param in_str: Input string, min/avg/max/hdrh format.
 783             :type in_str: str
 784             :returns: Dict with corresponding keys, except hdrh float values.
 785             :rtype dict:
 786             :throws IndexError: If in_str does not have enough substrings.
 787             :throws ValueError: If a substring does not convert to float.
 788             """
 789             in_list = in_str.split('/', 3)
 790
 791             rval = {
 792                 u"min": float(in_list[0]),
 793                 u"avg": float(in_list[1]),
 794                 u"max": float(in_list[2]),
 795                 u"hdrh": u""
 796             }
 797
 798             if len(in_list) == 4:
 799                 rval[u"hdrh"] = str(in_list[3])
 800
 801             return rval
 802
 803         try:
 804             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 805             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 806             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 807             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 808             if groups.lastindex == 4:
 809                 return latency, u"PASS"
 810         except (IndexError, ValueError):
 811             pass
 812
 813         try:
 814             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 815             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 816             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 817             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 818             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 819             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 820             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 821             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 822             if groups.lastindex == 12:
 823                 return latency, u"PASS"
 824         except (IndexError, ValueError):
 825             pass
 826
 827         # TODO: Remove when not needed
 828         latency[u"NDR10"] = {
 829             u"direction1": copy.copy(latency_default),
 830             u"direction2": copy.copy(latency_default)
 831         }
 832         latency[u"NDR50"] = {
 833             u"direction1": copy.copy(latency_default),
 834             u"direction2": copy.copy(latency_default)
 835         }
 836         latency[u"NDR90"] = {
 837             u"direction1": copy.copy(latency_default),
 838             u"direction2": copy.copy(latency_default)
 839         }
 840         try:
 841             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
 842             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
 843             latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
 844             latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
 845             latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
 846             latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
 847             latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
 848             latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
 849             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
 850             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
 851             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
 852             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
 853             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
 854             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
 855             return latency, u"PASS"
 856         except (IndexError, ValueError):
 857             pass
 858
 859         return latency, u"FAIL"
 860
 861     def visit_suite(self, suite):
 862         """Implements traversing through the suite and its direct children.
 863
 864         :param suite: Suite to process.
 865         :type suite: Suite
 866         :returns: Nothing.
 867         """
 868         if self.start_suite(suite) is not False:
 869             suite.suites.visit(self)
 870             suite.tests.visit(self)
 871             self.end_suite(suite)
 872
 873     def start_suite(self, suite):
 874         """Called when suite starts.
 875
 876         :param suite: Suite to process.
 877         :type suite: Suite
 878         :returns: Nothing.
 879         """
 880
 881         try:
 882             parent_name = suite.parent.name
 883         except AttributeError:
 884             return
 885
 886         doc_str = suite.doc.\
 887             replace(u'"', u"'").\
 888             replace(u'\n', u' ').\
 889             replace(u'\r', u'').\
 890             replace(u'*[', u' |br| *[').\
 891             replace(u"*", u"**").\
 892             replace(u' |br| *[', u'*[', 1)
 893
 894         self._data[u"suites"][suite.longname.lower().
 895                               replace(u'"', u"'").
 896                               replace(u" ", u"_")] = {
 897                                   u"name": suite.name.lower(),
 898                                   u"doc": doc_str,
 899                                   u"parent": parent_name,
 900                                   u"level": len(suite.longname.split(u"."))
 901                               }
 902
 903         suite.keywords.visit(self)
 904
 905     def end_suite(self, suite):
 906         """Called when suite ends.
 907
 908         :param suite: Suite to process.
 909         :type suite: Suite
 910         :returns: Nothing.
 911         """
 912
 913     def visit_test(self, test):
 914         """Implements traversing through the test.
 915
 916         :param test: Test to process.
 917         :type test: Test
 918         :returns: Nothing.
 919         """
 920         if self.start_test(test) is not False:
 921             test.keywords.visit(self)
 922             self.end_test(test)
 923
 924     def start_test(self, test):
 925         """Called when test starts.
 926
 927         :param test: Test to process.
 928         :type test: Test
 929         :returns: Nothing.
 930         """
 931
 932         self._sh_run_counter = 0
 933
 934         longname_orig = test.longname.lower()
 935
 936         # Check the ignore list
 937         if longname_orig in self._ignore:
 938             return
 939
 940         tags = [str(tag) for tag in test.tags]
 941         test_result = dict()
 942
 943         # Change the TC long name and name if defined in the mapping table
 944         longname = self._mapping.get(longname_orig, None)
 945         if longname is not None:
 946             name = longname.split(u'.')[-1]
 947             logging.debug(
 948                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
 949                 f"{name}"
 950             )
 951         else:
 952             longname = longname_orig
 953             name = test.name.lower()
 954
 955         # Remove TC number from the TC long name (backward compatibility):
 956         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
 957         # Remove TC number from the TC name (not needed):
 958         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
 959
 960         test_result[u"parent"] = test.parent.name.lower()
 961         test_result[u"tags"] = tags
 962         test_result["doc"] = test.doc.\
 963             replace(u'"', u"'").\
 964             replace(u'\n', u' ').\
 965             replace(u'\r', u'').\
 966             replace(u'[', u' |br| [').\
 967             replace(u' |br| [', u'[', 1)
 968         test_result[u"msg"] = self._get_data_from_perf_test_msg(test.message).\
 969             replace(u'\n', u' |br| ').\
 970             replace(u'\r', u'').\
 971             replace(u'"', u"'")
 972         test_result[u"type"] = u"FUNC"
 973         test_result[u"status"] = test.status
 974
 975         if u"PERFTEST" in tags:
 976             # Replace info about cores (e.g. -1c-) with the info about threads
 977             # and cores (e.g. -1t1c-) in the long test case names and in the
 978             # test case names if necessary.
 979             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
 980             if not groups:
 981                 tag_count = 0
 982                 tag_tc = str()
 983                 for tag in test_result[u"tags"]:
 984                     groups = re.search(self.REGEX_TC_TAG, tag)
 985                     if groups:
 986                         tag_count += 1
 987                         tag_tc = tag
 988
 989                 if tag_count == 1:
 990                     self._test_id = re.sub(self.REGEX_TC_NAME_NEW,
 991                                            f"-{tag_tc.lower()}-",
 992                                            self._test_id,
 993                                            count=1)
 994                     test_result[u"name"] = re.sub(self.REGEX_TC_NAME_NEW,
 995                                                   f"-{tag_tc.lower()}-",
 996                                                   test_result["name"],
 997                                                   count=1)
 998                 else:
 999                     test_result[u"status"] = u"FAIL"
1000                     self._data[u"tests"][self._test_id] = test_result
1001                     logging.debug(
1002                         f"The test {self._test_id} has no or more than one "
1003                         f"multi-threading tags.\n"
1004                         f"Tags: {test_result[u'tags']}"
1005                     )
1006                     return
1007
1008         if test.status == u"PASS":
1009             if u"NDRPDR" in tags:
1010                 test_result[u"type"] = u"NDRPDR"
1011                 test_result[u"throughput"], test_result[u"status"] = \
1012                     self._get_ndrpdr_throughput(test.message)
1013                 test_result[u"latency"], test_result[u"status"] = \
1014                     self._get_ndrpdr_latency(test.message)
1015             elif u"SOAK" in tags:
1016                 test_result[u"type"] = u"SOAK"
1017                 test_result[u"throughput"], test_result[u"status"] = \
1018                     self._get_plr_throughput(test.message)
1019             elif u"TCP" in tags:
1020                 test_result[u"type"] = u"TCP"
1021                 groups = re.search(self.REGEX_TCP, test.message)
1022                 test_result[u"result"] = int(groups.group(2))
1023             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1024                 if u"MRR" in tags:
1025                     test_result[u"type"] = u"MRR"
1026                 else:
1027                     test_result[u"type"] = u"BMRR"
1028
1029                 test_result[u"result"] = dict()
1030                 groups = re.search(self.REGEX_BMRR, test.message)
1031                 if groups is not None:
1032                     items_str = groups.group(1)
1033                     items_float = [float(item.strip()) for item
1034                                    in items_str.split(",")]
1035                     # Use whole list in CSIT-1180.
1036                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1037                     test_result[u"result"][u"receive-rate"] = stats.avg
1038                 else:
1039                     groups = re.search(self.REGEX_MRR, test.message)
1040                     test_result[u"result"][u"receive-rate"] = \
1041                         float(groups.group(3)) / float(groups.group(1))
1042             elif u"RECONF" in tags:
1043                 test_result[u"type"] = u"RECONF"
1044                 test_result[u"result"] = None
1045                 try:
1046                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1047                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1048                     test_result[u"result"] = {
1049                         u"loss": int(grps_loss.group(1)),
1050                         u"time": float(grps_time.group(1))
1051                     }
1052                 except (AttributeError, IndexError, ValueError, TypeError):
1053                     test_result[u"status"] = u"FAIL"
1054             else:
1055                 test_result[u"status"] = u"FAIL"
1056                 self._data[u"tests"][self._test_id] = test_result
1057                 return
1058
1059         self._data[u"tests"][self._test_id] = test_result
1060
1061     def end_test(self, test):
1062         """Called when test ends.
1063
1064         :param test: Test to process.
1065         :type test: Test
1066         :returns: Nothing.
1067         """
1068
1069     def visit_keyword(self, keyword):
1070         """Implements traversing through the keyword and its child keywords.
1071
1072         :param keyword: Keyword to process.
1073         :type keyword: Keyword
1074         :returns: Nothing.
1075         """
1076         if self.start_keyword(keyword) is not False:
1077             self.end_keyword(keyword)
1078
1079     def start_keyword(self, keyword):
1080         """Called when keyword starts. Default implementation does nothing.
1081
1082         :param keyword: Keyword to process.
1083         :type keyword: Keyword
1084         :returns: Nothing.
1085         """
1086         try:
1087             if keyword.type == u"setup":
1088                 self.visit_setup_kw(keyword)
1089             elif keyword.type == u"teardown":
1090                 self.visit_teardown_kw(keyword)
1091             else:
1092                 self.visit_test_kw(keyword)
1093         except AttributeError:
1094             pass
1095
1096     def end_keyword(self, keyword):
1097         """Called when keyword ends. Default implementation does nothing.
1098
1099         :param keyword: Keyword to process.
1100         :type keyword: Keyword
1101         :returns: Nothing.
1102         """
1103
1104     def visit_test_kw(self, test_kw):
1105         """Implements traversing through the test keyword and its child
1106         keywords.
1107
1108         :param test_kw: Keyword to process.
1109         :type test_kw: Keyword
1110         :returns: Nothing.
1111         """
1112         for keyword in test_kw.keywords:
1113             if self.start_test_kw(keyword) is not False:
1114                 self.visit_test_kw(keyword)
1115                 self.end_test_kw(keyword)
1116
1117     def start_test_kw(self, test_kw):
1118         """Called when test keyword starts. Default implementation does
1119         nothing.
1120
1121         :param test_kw: Keyword to process.
1122         :type test_kw: Keyword
1123         :returns: Nothing.
1124         """
1125         if test_kw.name.count(u"Show Runtime On All Duts") or \
1126                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
1127             self._msg_type = u"test-show-runtime"
1128             self._sh_run_counter += 1
1129         elif test_kw.name.count(u"Install Dpdk Test") and not self._version:
1130             self._msg_type = u"dpdk-version"
1131         else:
1132             return
1133         test_kw.messages.visit(self)
1134
1135     def end_test_kw(self, test_kw):
1136         """Called when keyword ends. Default implementation does nothing.
1137
1138         :param test_kw: Keyword to process.
1139         :type test_kw: Keyword
1140         :returns: Nothing.
1141         """
1142
1143     def visit_setup_kw(self, setup_kw):
1144         """Implements traversing through the teardown keyword and its child
1145         keywords.
1146
1147         :param setup_kw: Keyword to process.
1148         :type setup_kw: Keyword
1149         :returns: Nothing.
1150         """
1151         for keyword in setup_kw.keywords:
1152             if self.start_setup_kw(keyword) is not False:
1153                 self.visit_setup_kw(keyword)
1154                 self.end_setup_kw(keyword)
1155
1156     def start_setup_kw(self, setup_kw):
1157         """Called when teardown keyword starts. Default implementation does
1158         nothing.
1159
1160         :param setup_kw: Keyword to process.
1161         :type setup_kw: Keyword
1162         :returns: Nothing.
1163         """
1164         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1165                 and not self._version:
1166             self._msg_type = u"vpp-version"
1167         elif setup_kw.name.count(u"Set Global Variable") \
1168                 and not self._timestamp:
1169             self._msg_type = u"timestamp"
1170         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1171             self._msg_type = u"testbed"
1172         else:
1173             return
1174         setup_kw.messages.visit(self)
1175
1176     def end_setup_kw(self, setup_kw):
1177         """Called when keyword ends. Default implementation does nothing.
1178
1179         :param setup_kw: Keyword to process.
1180         :type setup_kw: Keyword
1181         :returns: Nothing.
1182         """
1183
1184     def visit_teardown_kw(self, teardown_kw):
1185         """Implements traversing through the teardown keyword and its child
1186         keywords.
1187
1188         :param teardown_kw: Keyword to process.
1189         :type teardown_kw: Keyword
1190         :returns: Nothing.
1191         """
1192         for keyword in teardown_kw.keywords:
1193             if self.start_teardown_kw(keyword) is not False:
1194                 self.visit_teardown_kw(keyword)
1195                 self.end_teardown_kw(keyword)
1196
1197     def start_teardown_kw(self, teardown_kw):
1198         """Called when teardown keyword starts
1199
1200         :param teardown_kw: Keyword to process.
1201         :type teardown_kw: Keyword
1202         :returns: Nothing.
1203         """
1204
1205         if teardown_kw.name.count(u"Show Vat History On All Duts"):
1206             # TODO: Remove when not needed:
1207             self._conf_history_lookup_nr = 0
1208             self._msg_type = u"teardown-vat-history"
1209             teardown_kw.messages.visit(self)
1210         elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1211             self._conf_history_lookup_nr = 0
1212             self._msg_type = u"teardown-papi-history"
1213             teardown_kw.messages.visit(self)
1214
1215     def end_teardown_kw(self, teardown_kw):
1216         """Called when keyword ends. Default implementation does nothing.
1217
1218         :param teardown_kw: Keyword to process.
1219         :type teardown_kw: Keyword
1220         :returns: Nothing.
1221         """
1222
1223     def visit_message(self, msg):
1224         """Implements visiting the message.
1225
1226         :param msg: Message to process.
1227         :type msg: Message
1228         :returns: Nothing.
1229         """
1230         if self.start_message(msg) is not False:
1231             self.end_message(msg)
1232
1233     def start_message(self, msg):
1234         """Called when message starts. Get required information from messages:
1235         - VPP version.
1236
1237         :param msg: Message to process.
1238         :type msg: Message
1239         :returns: Nothing.
1240         """
1241
1242         if self._msg_type:
1243             self.parse_msg[self._msg_type](msg)
1244
1245     def end_message(self, msg):
1246         """Called when message ends. Default implementation does nothing.
1247
1248         :param msg: Message to process.
1249         :type msg: Message
1250         :returns: Nothing.
1251         """
1252
1253
1254 class InputData:
1255     """Input data
1256
1257     The data is extracted from output.xml files generated by Jenkins jobs and
1258     stored in pandas' DataFrames.
1259
1260     The data structure:
1261     - job name
1262       - build number
1263         - metadata
1264           (as described in ExecutionChecker documentation)
1265         - suites
1266           (as described in ExecutionChecker documentation)
1267         - tests
1268           (as described in ExecutionChecker documentation)
1269     """
1270
1271     def __init__(self, spec):
1272         """Initialization.
1273
1274         :param spec: Specification.
1275         :type spec: Specification
1276         """
1277
1278         # Specification:
1279         self._cfg = spec
1280
1281         # Data store:
1282         self._input_data = pd.Series()
1283
1284     @property
1285     def data(self):
1286         """Getter - Input data.
1287
1288         :returns: Input data
1289         :rtype: pandas.Series
1290         """
1291         return self._input_data
1292
1293     def metadata(self, job, build):
1294         """Getter - metadata
1295
1296         :param job: Job which metadata we want.
1297         :param build: Build which metadata we want.
1298         :type job: str
1299         :type build: str
1300         :returns: Metadata
1301         :rtype: pandas.Series
1302         """
1303
1304         return self.data[job][build][u"metadata"]
1305
1306     def suites(self, job, build):
1307         """Getter - suites
1308
1309         :param job: Job which suites we want.
1310         :param build: Build which suites we want.
1311         :type job: str
1312         :type build: str
1313         :returns: Suites.
1314         :rtype: pandas.Series
1315         """
1316
1317         return self.data[job][str(build)][u"suites"]
1318
1319     def tests(self, job, build):
1320         """Getter - tests
1321
1322         :param job: Job which tests we want.
1323         :param build: Build which tests we want.
1324         :type job: str
1325         :type build: str
1326         :returns: Tests.
1327         :rtype: pandas.Series
1328         """
1329
1330         return self.data[job][build][u"tests"]
1331
1332     def _parse_tests(self, job, build, log):
1333         """Process data from robot output.xml file and return JSON structured
1334         data.
1335
1336         :param job: The name of job which build output data will be processed.
1337         :param build: The build which output data will be processed.
1338         :param log: List of log messages.
1339         :type job: str
1340         :type build: dict
1341         :type log: list of tuples (severity, msg)
1342         :returns: JSON data structure.
1343         :rtype: dict
1344         """
1345
1346         metadata = {
1347             u"job": job,
1348             u"build": build
1349         }
1350
1351         with open(build[u"file-name"], u'r') as data_file:
1352             try:
1353                 result = ExecutionResult(data_file)
1354             except errors.DataError as err:
1355                 log.append(
1356                     (u"ERROR", f"Error occurred while parsing output.xml: "
1357                                f"{repr(err)}")
1358                 )
1359                 return None
1360         checker = ExecutionChecker(metadata, self._cfg.mapping,
1361                                    self._cfg.ignore)
1362         result.visit(checker)
1363
1364         return checker.data
1365
1366     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1367         """Download and parse the input data file.
1368
1369         :param pid: PID of the process executing this method.
1370         :param job: Name of the Jenkins job which generated the processed input
1371             file.
1372         :param build: Information about the Jenkins build which generated the
1373             processed input file.
1374         :param repeat: Repeat the download specified number of times if not
1375             successful.
1376         :type pid: int
1377         :type job: str
1378         :type build: dict
1379         :type repeat: int
1380         """
1381
1382         logs = list()
1383
1384         logs.append(
1385             (u"INFO", f"  Processing the job/build: {job}: {build[u'build']}")
1386         )
1387
1388         state = u"failed"
1389         success = False
1390         data = None
1391         do_repeat = repeat
1392         while do_repeat:
1393             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1394                                                    logs)
1395             if success:
1396                 break
1397             do_repeat -= 1
1398         if not success:
1399             logs.append(
1400                 (u"ERROR",
1401                  f"It is not possible to download the input data file from the "
1402                  f"job {job}, build {build[u'build']}, or it is damaged. "
1403                  f"Skipped.")
1404             )
1405         if success:
1406             logs.append(
1407                 (u"INFO",
1408                  f"    Processing data from the build {build[u'build']} ...")
1409             )
1410             data = self._parse_tests(job, build, logs)
1411             if data is None:
1412                 logs.append(
1413                     (u"ERROR",
1414                      f"Input data file from the job {job}, build "
1415                      f"{build[u'build']} is damaged. Skipped.")
1416                 )
1417             else:
1418                 state = u"processed"
1419
1420             try:
1421                 remove(build[u"file-name"])
1422             except OSError as err:
1423                 logs.append(
1424                     ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
1425                               f"{repr(err)}")
1426                 )
1427
1428         # If the time-period is defined in the specification file, remove all
1429         # files which are outside the time period.
1430         timeperiod = self._cfg.input.get(u"time-period", None)
1431         if timeperiod and data:
1432             now = dt.utcnow()
1433             timeperiod = timedelta(int(timeperiod))
1434             metadata = data.get(u"metadata", None)
1435             if metadata:
1436                 generated = metadata.get(u"generated", None)
1437                 if generated:
1438                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1439                     if (now - generated) > timeperiod:
1440                         # Remove the data and the file:
1441                         state = u"removed"
1442                         data = None
1443                         logs.append(
1444                             (u"INFO",
1445                              f"    The build {job}/{build[u'build']} is "
1446                              f"outdated, will be removed.")
1447                         )
1448         logs.append((u"INFO", u"  Done."))
1449
1450         for level, line in logs:
1451             if level == u"INFO":
1452                 logging.info(line)
1453             elif level == u"ERROR":
1454                 logging.error(line)
1455             elif level == u"DEBUG":
1456                 logging.debug(line)
1457             elif level == u"CRITICAL":
1458                 logging.critical(line)
1459             elif level == u"WARNING":
1460                 logging.warning(line)
1461
1462         return {u"data": data, u"state": state, u"job": job, u"build": build}
1463
1464     def download_and_parse_data(self, repeat=1):
1465         """Download the input data files, parse input data from input files and
1466         store in pandas' Series.
1467
1468         :param repeat: Repeat the download specified number of times if not
1469             successful.
1470         :type repeat: int
1471         """
1472
1473         logging.info(u"Downloading and parsing input files ...")
1474
1475         for job, builds in self._cfg.builds.items():
1476             for build in builds:
1477
1478                 result = self._download_and_parse_build(job, build, repeat)
1479                 build_nr = result[u"build"][u"build"]
1480
1481                 if result[u"data"]:
1482                     data = result[u"data"]
1483                     build_data = pd.Series({
1484                         u"metadata": pd.Series(
1485                             list(data[u"metadata"].values()),
1486                             index=list(data[u"metadata"].keys())
1487                         ),
1488                         u"suites": pd.Series(
1489                             list(data[u"suites"].values()),
1490                             index=list(data[u"suites"].keys())
1491                         ),
1492                         u"tests": pd.Series(
1493                             list(data[u"tests"].values()),
1494                             index=list(data[u"tests"].keys())
1495                         )
1496                     })
1497
1498                     if self._input_data.get(job, None) is None:
1499                         self._input_data[job] = pd.Series()
1500                     self._input_data[job][str(build_nr)] = build_data
1501
1502                     self._cfg.set_input_file_name(
1503                         job, build_nr, result[u"build"][u"file-name"])
1504
1505                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1506
1507                 mem_alloc = \
1508                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1509                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1510
1511         logging.info(u"Done.")
1512
1513     @staticmethod
1514     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1515         """Return the index of character in the string which is the end of tag.
1516
1517         :param tag_filter: The string where the end of tag is being searched.
1518         :param start: The index where the searching is stated.
1519         :param closer: The character which is the tag closer.
1520         :type tag_filter: str
1521         :type start: int
1522         :type closer: str
1523         :returns: The index of the tag closer.
1524         :rtype: int
1525         """
1526
1527         try:
1528             idx_opener = tag_filter.index(closer, start)
1529             return tag_filter.index(closer, idx_opener + 1)
1530         except ValueError:
1531             return None
1532
1533     @staticmethod
1534     def _condition(tag_filter):
1535         """Create a conditional statement from the given tag filter.
1536
1537         :param tag_filter: Filter based on tags from the element specification.
1538         :type tag_filter: str
1539         :returns: Conditional statement which can be evaluated.
1540         :rtype: str
1541         """
1542
1543         index = 0
1544         while True:
1545             index = InputData._end_of_tag(tag_filter, index)
1546             if index is None:
1547                 return tag_filter
1548             index += 1
1549             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1550
1551     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1552                     continue_on_error=False):
1553         """Filter required data from the given jobs and builds.
1554
1555         The output data structure is:
1556
1557         - job 1
1558           - build 1
1559             - test (or suite) 1 ID:
1560               - param 1
1561               - param 2
1562               ...
1563               - param n
1564             ...
1565             - test (or suite) n ID:
1566             ...
1567           ...
1568           - build n
1569         ...
1570         - job n
1571
1572         :param element: Element which will use the filtered data.
1573         :param params: Parameters which will be included in the output. If None,
1574             all parameters are included.
1575         :param data: If not None, this data is used instead of data specified
1576             in the element.
1577         :param data_set: The set of data to be filtered: tests, suites,
1578             metadata.
1579         :param continue_on_error: Continue if there is error while reading the
1580             data. The Item will be empty then
1581         :type element: pandas.Series
1582         :type params: list
1583         :type data: dict
1584         :type data_set: str
1585         :type continue_on_error: bool
1586         :returns: Filtered data.
1587         :rtype pandas.Series
1588         """
1589
1590         try:
1591             if data_set == "suites":
1592                 cond = u"True"
1593             elif element[u"filter"] in (u"all", u"template"):
1594                 cond = u"True"
1595             else:
1596                 cond = InputData._condition(element[u"filter"])
1597             logging.debug(f"   Filter: {cond}")
1598         except KeyError:
1599             logging.error(u"  No filter defined.")
1600             return None
1601
1602         if params is None:
1603             params = element.get(u"parameters", None)
1604             if params:
1605                 params.append(u"type")
1606
1607         data_to_filter = data if data else element[u"data"]
1608         data = pd.Series()
1609         try:
1610             for job, builds in data_to_filter.items():
1611                 data[job] = pd.Series()
1612                 for build in builds:
1613                     data[job][str(build)] = pd.Series()
1614                     try:
1615                         data_dict = dict(
1616                             self.data[job][str(build)][data_set].items())
1617                     except KeyError:
1618                         if continue_on_error:
1619                             continue
1620                         return None
1621
1622                     for test_id, test_data in data_dict.items():
1623                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1624                             data[job][str(build)][test_id] = pd.Series()
1625                             if params is None:
1626                                 for param, val in test_data.items():
1627                                     data[job][str(build)][test_id][param] = val
1628                             else:
1629                                 for param in params:
1630                                     try:
1631                                         data[job][str(build)][test_id][param] =\
1632                                             test_data[param]
1633                                     except KeyError:
1634                                         data[job][str(build)][test_id][param] =\
1635                                             u"No Data"
1636             return data
1637
1638         except (KeyError, IndexError, ValueError) as err:
1639             logging.error(
1640                 f"Missing mandatory parameter in the element specification: "
1641                 f"{repr(err)}"
1642             )
1643             return None
1644         except AttributeError as err:
1645             logging.error(repr(err))
1646             return None
1647         except SyntaxError as err:
1648             logging.error(
1649                 f"The filter {cond} is not correct. Check if all tags are "
1650                 f"enclosed by apostrophes.\n{repr(err)}"
1651             )
1652             return None
1653
1654     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1655                              continue_on_error=False):
1656         """Filter required data from the given jobs and builds.
1657
1658         The output data structure is:
1659
1660         - job 1
1661           - build 1
1662             - test (or suite) 1 ID:
1663               - param 1
1664               - param 2
1665               ...
1666               - param n
1667             ...
1668             - test (or suite) n ID:
1669             ...
1670           ...
1671           - build n
1672         ...
1673         - job n
1674
1675         :param element: Element which will use the filtered data.
1676         :param params: Parameters which will be included in the output. If None,
1677         all parameters are included.
1678         :param data_set: The set of data to be filtered: tests, suites,
1679         metadata.
1680         :param continue_on_error: Continue if there is error while reading the
1681         data. The Item will be empty then
1682         :type element: pandas.Series
1683         :type params: list
1684         :type data_set: str
1685         :type continue_on_error: bool
1686         :returns: Filtered data.
1687         :rtype pandas.Series
1688         """
1689
1690         include = element.get(u"include", None)
1691         if not include:
1692             logging.warning(u"No tests to include, skipping the element.")
1693             return None
1694
1695         if params is None:
1696             params = element.get(u"parameters", None)
1697             if params:
1698                 params.append(u"type")
1699
1700         data = pd.Series()
1701         try:
1702             for job, builds in element[u"data"].items():
1703                 data[job] = pd.Series()
1704                 for build in builds:
1705                     data[job][str(build)] = pd.Series()
1706                     for test in include:
1707                         try:
1708                             reg_ex = re.compile(str(test).lower())
1709                             for test_id in self.data[job][
1710                                     str(build)][data_set].keys():
1711                                 if re.match(reg_ex, str(test_id).lower()):
1712                                     test_data = self.data[job][
1713                                         str(build)][data_set][test_id]
1714                                     data[job][str(build)][test_id] = pd.Series()
1715                                     if params is None:
1716                                         for param, val in test_data.items():
1717                                             data[job][str(build)][test_id]\
1718                                                 [param] = val
1719                                     else:
1720                                         for param in params:
1721                                             try:
1722                                                 data[job][str(build)][
1723                                                     test_id][param] = \
1724                                                     test_data[param]
1725                                             except KeyError:
1726                                                 data[job][str(build)][
1727                                                     test_id][param] = u"No Data"
1728                         except KeyError as err:
1729                             logging.error(repr(err))
1730                             if continue_on_error:
1731                                 continue
1732                             return None
1733             return data
1734
1735         except (KeyError, IndexError, ValueError) as err:
1736             logging.error(
1737                 f"Missing mandatory parameter in the element "
1738                 f"specification: {repr(err)}"
1739             )
1740             return None
1741         except AttributeError as err:
1742             logging.error(repr(err))
1743             return None
1744
1745     @staticmethod
1746     def merge_data(data):
1747         """Merge data from more jobs and builds to a simple data structure.
1748
1749         The output data structure is:
1750
1751         - test (suite) 1 ID:
1752           - param 1
1753           - param 2
1754           ...
1755           - param n
1756         ...
1757         - test (suite) n ID:
1758         ...
1759
1760         :param data: Data to merge.
1761         :type data: pandas.Series
1762         :returns: Merged data.
1763         :rtype: pandas.Series
1764         """
1765
1766         logging.info(u"    Merging data ...")
1767
1768         merged_data = pd.Series()
1769         for builds in data.values:
1770             for item in builds.values:
1771                 for item_id, item_data in item.items():
1772                     merged_data[item_id] = item_data
1773
1774         return merged_data
1775
1776     def print_all_oper_data(self):
1777         """Print all operational data to console.
1778         """
1779
1780         tbl_hdr = (
1781             u"Name",
1782             u"Nr of Vectors",
1783             u"Nr of Packets",
1784             u"Suspends",
1785             u"Cycles per Packet",
1786             u"Average Vector Size"
1787         )
1788
1789         for job in self._input_data.values:
1790             for build in job.values:
1791                 for test_id, test_data in build[u"tests"].items():
1792                     print(f"{test_id}")
1793                     if test_data.get(u"show-run", None) is None:
1794                         continue
1795                     for dut_name, data in test_data[u"show-run"].items():
1796                         if data.get(u"threads", None) is None:
1797                             continue
1798                         print(f"Host IP: {data.get(u'host', '')}, "
1799                               f"Socket: {data.get(u'socket', '')}")
1800                         for thread_nr, thread in data[u"threads"].items():
1801                             txt_table = prettytable.PrettyTable(tbl_hdr)
1802                             avg = 0.0
1803                             for row in thread:
1804                                 txt_table.add_row(row)
1805                                 avg += row[-1]
1806                             if len(thread) == 0:
1807                                 avg = u""
1808                             else:
1809                                 avg = f", Average Vector Size per Node: " \
1810                                       f"{(avg / len(thread)):.2f}"
1811                             th_name = u"main" if thread_nr == 0 \
1812                                 else f"worker_{thread_nr}"
1813                             print(f"{dut_name}, {th_name}{avg}")
1814                             txt_table.float_format = u".2"
1815                             txt_table.align = u"r"
1816                             txt_table.align[u"Name"] = u"l"
1817                             print(f"{txt_table.get_string()}\n")