resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove
  29 from datetime import datetime as dt
  30 from datetime import timedelta
  31 from json import loads
  32
  33 import hdrh.histogram
  34 import hdrh.codec
  35 import prettytable
  36 import pandas as pd
  37
  38 from robot.api import ExecutionResult, ResultVisitor
  39 from robot import errors
  40
  41 from resources.libraries.python import jumpavg
  42 from input_data_files import download_and_unzip_data_file
  43
  44
  45 # Separator used in file names
  46 SEPARATOR = u"__"
  47
  48
  49 class ExecutionChecker(ResultVisitor):
  50     """Class to traverse through the test suite structure.
  51
  52     The functionality implemented in this class generates a json structure:
  53
  54     Performance tests:
  55
  56     {
  57         "metadata": {
  58             "generated": "Timestamp",
  59             "version": "SUT version",
  60             "job": "Jenkins job name",
  61             "build": "Information about the build"
  62         },
  63         "suites": {
  64             "Suite long name 1": {
  65                 "name": Suite name,
  66                 "doc": "Suite 1 documentation",
  67                 "parent": "Suite 1 parent",
  68                 "level": "Level of the suite in the suite hierarchy"
  69             }
  70             "Suite long name N": {
  71                 "name": Suite name,
  72                 "doc": "Suite N documentation",
  73                 "parent": "Suite 2 parent",
  74                 "level": "Level of the suite in the suite hierarchy"
  75             }
  76         }
  77         "tests": {
  78             # NDRPDR tests:
  79             "ID": {
  80                 "name": "Test name",
  81                 "parent": "Name of the parent of the test",
  82                 "doc": "Test documentation",
  83                 "msg": "Test message",
  84                 "conf-history": "DUT1 and DUT2 VAT History",
  85                 "show-run": "Show Run",
  86                 "tags": ["tag 1", "tag 2", "tag n"],
  87                 "type": "NDRPDR",
  88                 "status": "PASS" | "FAIL",
  89                 "throughput": {
  90                     "NDR": {
  91                         "LOWER": float,
  92                         "UPPER": float
  93                     },
  94                     "PDR": {
  95                         "LOWER": float,
  96                         "UPPER": float
  97                     }
  98                 },
  99                 "latency": {
 100                     "NDR": {
 101                         "direction1": {
 102                             "min": float,
 103                             "avg": float,
 104                             "max": float,
 105                             "hdrh": str
 106                         },
 107                         "direction2": {
 108                             "min": float,
 109                             "avg": float,
 110                             "max": float,
 111                             "hdrh": str
 112                         }
 113                     },
 114                     "PDR": {
 115                         "direction1": {
 116                             "min": float,
 117                             "avg": float,
 118                             "max": float,
 119                             "hdrh": str
 120                         },
 121                         "direction2": {
 122                             "min": float,
 123                             "avg": float,
 124                             "max": float,
 125                             "hdrh": str
 126                         }
 127                     }
 128                 }
 129             }
 130
 131             # TCP tests:
 132             "ID": {
 133                 "name": "Test name",
 134                 "parent": "Name of the parent of the test",
 135                 "doc": "Test documentation",
 136                 "msg": "Test message",
 137                 "tags": ["tag 1", "tag 2", "tag n"],
 138                 "type": "TCP",
 139                 "status": "PASS" | "FAIL",
 140                 "result": int
 141             }
 142
 143             # MRR, BMRR tests:
 144             "ID": {
 145                 "name": "Test name",
 146                 "parent": "Name of the parent of the test",
 147                 "doc": "Test documentation",
 148                 "msg": "Test message",
 149                 "tags": ["tag 1", "tag 2", "tag n"],
 150                 "type": "MRR" | "BMRR",
 151                 "status": "PASS" | "FAIL",
 152                 "result": {
 153                     "receive-rate": float,
 154                     # Average of a list, computed using AvgStdevStats.
 155                     # In CSIT-1180, replace with List[float].
 156                 }
 157             }
 158
 159             "ID" {
 160                 # next test
 161             }
 162         }
 163     }
 164
 165
 166     Functional tests:
 167
 168     {
 169         "metadata": {  # Optional
 170             "version": "VPP version",
 171             "job": "Jenkins job name",
 172             "build": "Information about the build"
 173         },
 174         "suites": {
 175             "Suite name 1": {
 176                 "doc": "Suite 1 documentation",
 177                 "parent": "Suite 1 parent",
 178                 "level": "Level of the suite in the suite hierarchy"
 179             }
 180             "Suite name N": {
 181                 "doc": "Suite N documentation",
 182                 "parent": "Suite 2 parent",
 183                 "level": "Level of the suite in the suite hierarchy"
 184             }
 185         }
 186         "tests": {
 187             "ID": {
 188                 "name": "Test name",
 189                 "parent": "Name of the parent of the test",
 190                 "doc": "Test documentation"
 191                 "msg": "Test message"
 192                 "tags": ["tag 1", "tag 2", "tag n"],
 193                 "conf-history": "DUT1 and DUT2 VAT History"
 194                 "show-run": "Show Run"
 195                 "status": "PASS" | "FAIL"
 196             },
 197             "ID" {
 198                 # next test
 199             }
 200         }
 201     }
 202
 203     .. note:: ID is the lowercase full path to the test.
 204     """
 205
 206     REGEX_PLR_RATE = re.compile(
 207         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 208         r'PLRsearch upper bound::?\s(\d+.\d+)'
 209     )
 210     REGEX_NDRPDR_RATE = re.compile(
 211         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 212         r'NDR_UPPER:\s(\d+.\d+).*\n'
 213         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 214         r'PDR_UPPER:\s(\d+.\d+)'
 215     )
 216     REGEX_PERF_MSG_INFO = re.compile(
 217         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 218         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 219         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 220         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 221         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 222     )
 223     # TODO: Remove when not needed
 224     REGEX_NDRPDR_LAT_BASE = re.compile(
 225         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 226         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 227     )
 228     REGEX_NDRPDR_LAT = re.compile(
 229         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 230         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 231         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 232         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 233         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 234         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 235     )
 236     # TODO: Remove when not needed
 237     REGEX_NDRPDR_LAT_LONG = re.compile(
 238         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 239         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 240         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 241         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 242         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 243         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 244         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 245         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 246         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 247     )
 248     REGEX_VERSION_VPP = re.compile(
 249         r"(return STDOUT Version:\s*|"
 250         r"VPP Version:\s*|VPP version:\s*)(.*)"
 251     )
 252     REGEX_VERSION_DPDK = re.compile(
 253         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 254     )
 255     REGEX_TCP = re.compile(
 256         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 257     )
 258     REGEX_MRR = re.compile(
 259         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 260         r'tx\s(\d*),\srx\s(\d*)'
 261     )
 262     REGEX_BMRR = re.compile(
 263         r'Maximum Receive Rate trial results'
 264         r' in packets per second: \[(.*)\]'
 265     )
 266     REGEX_RECONF_LOSS = re.compile(
 267         r'Packets lost due to reconfig: (\d*)'
 268     )
 269     REGEX_RECONF_TIME = re.compile(
 270         r'Implied time lost: (\d*.[\de-]*)'
 271     )
 272     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 273
 274     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 275
 276     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 277
 278     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 279
 280     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 281
 282     def __init__(self, metadata, mapping, ignore):
 283         """Initialisation.
 284
 285         :param metadata: Key-value pairs to be included in "metadata" part of
 286             JSON structure.
 287         :param mapping: Mapping of the old names of test cases to the new
 288             (actual) one.
 289         :param ignore: List of TCs to be ignored.
 290         :type metadata: dict
 291         :type mapping: dict
 292         :type ignore: list
 293         """
 294
 295         # Type of message to parse out from the test messages
 296         self._msg_type = None
 297
 298         # VPP version
 299         self._version = None
 300
 301         # Timestamp
 302         self._timestamp = None
 303
 304         # Testbed. The testbed is identified by TG node IP address.
 305         self._testbed = None
 306
 307         # Mapping of TCs long names
 308         self._mapping = mapping
 309
 310         # Ignore list
 311         self._ignore = ignore
 312
 313         # Number of PAPI History messages found:
 314         # 0 - no message
 315         # 1 - PAPI History of DUT1
 316         # 2 - PAPI History of DUT2
 317         self._conf_history_lookup_nr = 0
 318
 319         self._sh_run_counter = 0
 320
 321         # Test ID of currently processed test- the lowercase full path to the
 322         # test
 323         self._test_id = None
 324
 325         # The main data structure
 326         self._data = {
 327             u"metadata": OrderedDict(),
 328             u"suites": OrderedDict(),
 329             u"tests": OrderedDict()
 330         }
 331
 332         # Save the provided metadata
 333         for key, val in metadata.items():
 334             self._data[u"metadata"][key] = val
 335
 336         # Dictionary defining the methods used to parse different types of
 337         # messages
 338         self.parse_msg = {
 339             u"timestamp": self._get_timestamp,
 340             u"vpp-version": self._get_vpp_version,
 341             u"dpdk-version": self._get_dpdk_version,
 342             # TODO: Remove when not needed:
 343             u"teardown-vat-history": self._get_vat_history,
 344             u"teardown-papi-history": self._get_papi_history,
 345             u"test-show-runtime": self._get_show_run,
 346             u"testbed": self._get_testbed
 347         }
 348
 349     @property
 350     def data(self):
 351         """Getter - Data parsed from the XML file.
 352
 353         :returns: Data parsed from the XML file.
 354         :rtype: dict
 355         """
 356         return self._data
 357
 358     def _get_data_from_perf_test_msg(self, msg):
 359         """Get
 360             - NDR_LOWER
 361             - LATENCY
 362             - NDR_UPPER
 363             - PDR_LOWER
 364             - LATENCY
 365             - PDR_UPPER
 366         from message of NDRPDR performance tests.
 367
 368         :param msg: Message to be processed.
 369         :type msg: str
 370         :returns: Processed message or original message if a problem occurs.
 371         :rtype: str
 372         """
 373
 374         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 375         if not groups or groups.lastindex != 10:
 376             return msg
 377
 378         try:
 379             data = {
 380                 u"ndr_low": float(groups.group(1)),
 381                 u"ndr_low_b": float(groups.group(2)),
 382                 u"pdr_low": float(groups.group(3)),
 383                 u"pdr_low_b": float(groups.group(4)),
 384                 u"pdr_lat_90_1": groups.group(5),
 385                 u"pdr_lat_90_2": groups.group(6),
 386                 u"pdr_lat_50_1": groups.group(7),
 387                 u"pdr_lat_50_2": groups.group(8),
 388                 u"pdr_lat_10_1": groups.group(9),
 389                 u"pdr_lat_10_2": groups.group(10),
 390             }
 391         except (AttributeError, IndexError, ValueError, KeyError):
 392             return msg
 393
 394         def _process_lat(in_str_1, in_str_2):
 395             """Extract min, avg, max values from latency string.
 396
 397             :param in_str_1: Latency string for one direction produced by robot
 398                 framework.
 399             :param in_str_2: Latency string for second direction produced by
 400                 robot framework.
 401             :type in_str_1: str
 402             :type in_str_2: str
 403             :returns: Processed latency string or empty string if a problem
 404                 occurs.
 405             :rtype: tuple(str, str)
 406             """
 407             in_list_1 = in_str_1.split('/', 3)
 408             in_list_2 = in_str_2.split('/', 3)
 409
 410             if len(in_list_1) != 4 and len(in_list_2) != 4:
 411                 return u""
 412
 413             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 414             try:
 415                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 416             except hdrh.codec.HdrLengthException:
 417                 return u""
 418
 419             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 420             try:
 421                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 422             except hdrh.codec.HdrLengthException:
 423                 return u""
 424
 425             if hdr_lat_1 and hdr_lat_2:
 426                 hdr_lat_1_50 = hdr_lat_1.get_value_at_percentile(50.0)
 427                 hdr_lat_1_90 = hdr_lat_1.get_value_at_percentile(90.0)
 428                 hdr_lat_1_99 = hdr_lat_1.get_value_at_percentile(99.0)
 429                 hdr_lat_2_50 = hdr_lat_2.get_value_at_percentile(50.0)
 430                 hdr_lat_2_90 = hdr_lat_2.get_value_at_percentile(90.0)
 431                 hdr_lat_2_99 = hdr_lat_2.get_value_at_percentile(99.0)
 432
 433                 if (hdr_lat_1_50 + hdr_lat_1_90 + hdr_lat_1_99 +
 434                         hdr_lat_2_50 + hdr_lat_2_90 + hdr_lat_2_99):
 435                     return (
 436                         f"{hdr_lat_1_50} {hdr_lat_1_90} {hdr_lat_1_99} , "
 437                         f"{hdr_lat_2_50} {hdr_lat_2_90} {hdr_lat_2_99}"
 438                     )
 439
 440             return u""
 441
 442         try:
 443             pdr_lat_10 = _process_lat(data[u'pdr_lat_10_1'],
 444                                       data[u'pdr_lat_10_2'])
 445             pdr_lat_50 = _process_lat(data[u'pdr_lat_50_1'],
 446                                       data[u'pdr_lat_50_2'])
 447             pdr_lat_90 = _process_lat(data[u'pdr_lat_90_1'],
 448                                       data[u'pdr_lat_90_2'])
 449             pdr_lat_10 = f"\n3. {pdr_lat_10}" if pdr_lat_10 else u""
 450             pdr_lat_50 = f"\n4. {pdr_lat_50}" if pdr_lat_50 else u""
 451             pdr_lat_90 = f"\n5. {pdr_lat_90}" if pdr_lat_90 else u""
 452
 453             return (
 454                 u" |prein| "
 455                 f"1. {(data[u'ndr_low'] / 1e6):.2f} {data[u'ndr_low_b']:.2f}"
 456                 f"\n2. {(data[u'pdr_low'] / 1e6):.2f} {data[u'pdr_low_b']:.2f}"
 457                 f"{pdr_lat_10}"
 458                 f"{pdr_lat_50}"
 459                 f"{pdr_lat_90}"
 460                 u" |preout| "
 461             )
 462         except (AttributeError, IndexError, ValueError, KeyError):
 463             return msg
 464
 465     def _get_testbed(self, msg):
 466         """Called when extraction of testbed IP is required.
 467         The testbed is identified by TG node IP address.
 468
 469         :param msg: Message to process.
 470         :type msg: Message
 471         :returns: Nothing.
 472         """
 473
 474         if msg.message.count(u"Setup of TG node") or \
 475                 msg.message.count(u"Setup of node TG host"):
 476             reg_tg_ip = re.compile(
 477                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 478             try:
 479                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 480             except (KeyError, ValueError, IndexError, AttributeError):
 481                 pass
 482             finally:
 483                 self._data[u"metadata"][u"testbed"] = self._testbed
 484                 self._msg_type = None
 485
 486     def _get_vpp_version(self, msg):
 487         """Called when extraction of VPP version is required.
 488
 489         :param msg: Message to process.
 490         :type msg: Message
 491         :returns: Nothing.
 492         """
 493
 494         if msg.message.count(u"return STDOUT Version:") or \
 495             msg.message.count(u"VPP Version:") or \
 496             msg.message.count(u"VPP version:"):
 497             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 498                                 group(2))
 499             self._data[u"metadata"][u"version"] = self._version
 500             self._msg_type = None
 501
 502     def _get_dpdk_version(self, msg):
 503         """Called when extraction of DPDK version is required.
 504
 505         :param msg: Message to process.
 506         :type msg: Message
 507         :returns: Nothing.
 508         """
 509
 510         if msg.message.count(u"DPDK Version:"):
 511             try:
 512                 self._version = str(re.search(
 513                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 514                 self._data[u"metadata"][u"version"] = self._version
 515             except IndexError:
 516                 pass
 517             finally:
 518                 self._msg_type = None
 519
 520     def _get_timestamp(self, msg):
 521         """Called when extraction of timestamp is required.
 522
 523         :param msg: Message to process.
 524         :type msg: Message
 525         :returns: Nothing.
 526         """
 527
 528         self._timestamp = msg.timestamp[:14]
 529         self._data[u"metadata"][u"generated"] = self._timestamp
 530         self._msg_type = None
 531
 532     def _get_vat_history(self, msg):
 533         """Called when extraction of VAT command history is required.
 534
 535         TODO: Remove when not needed.
 536
 537         :param msg: Message to process.
 538         :type msg: Message
 539         :returns: Nothing.
 540         """
 541         if msg.message.count(u"VAT command history:"):
 542             self._conf_history_lookup_nr += 1
 543             if self._conf_history_lookup_nr == 1:
 544                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 545             else:
 546                 self._msg_type = None
 547             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 548                           r"VAT command history:", u"",
 549                           msg.message, count=1).replace(u'\n', u' |br| ').\
 550                 replace(u'"', u"'")
 551
 552             self._data[u"tests"][self._test_id][u"conf-history"] += (
 553                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 554             )
 555
 556     def _get_papi_history(self, msg):
 557         """Called when extraction of PAPI command history is required.
 558
 559         :param msg: Message to process.
 560         :type msg: Message
 561         :returns: Nothing.
 562         """
 563         if msg.message.count(u"PAPI command history:"):
 564             self._conf_history_lookup_nr += 1
 565             if self._conf_history_lookup_nr == 1:
 566                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 567             else:
 568                 self._msg_type = None
 569             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 570                           r"PAPI command history:", u"",
 571                           msg.message, count=1).replace(u'\n', u' |br| ').\
 572                 replace(u'"', u"'")
 573             self._data[u"tests"][self._test_id][u"conf-history"] += (
 574                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 575             )
 576
 577     def _get_show_run(self, msg):
 578         """Called when extraction of VPP operational data (output of CLI command
 579         Show Runtime) is required.
 580
 581         :param msg: Message to process.
 582         :type msg: Message
 583         :returns: Nothing.
 584         """
 585
 586         if not msg.message.count(u"stats runtime"):
 587             return
 588
 589         # Temporary solution
 590         if self._sh_run_counter > 1:
 591             return
 592
 593         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 594             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 595
 596         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 597         if not groups:
 598             return
 599         try:
 600             host = groups.group(1)
 601         except (AttributeError, IndexError):
 602             host = u""
 603         try:
 604             sock = groups.group(2)
 605         except (AttributeError, IndexError):
 606             sock = u""
 607
 608         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 609                         replace(u"'", u'"').replace(u'b"', u'"').
 610                         replace(u'u"', u'"').split(u":", 1)[1])
 611
 612         try:
 613             threads_nr = len(runtime[0][u"clocks"])
 614         except (IndexError, KeyError):
 615             return
 616
 617         dut = u"DUT{nr}".format(
 618             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 619
 620         oper = {
 621             u"host": host,
 622             u"socket": sock,
 623             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 624         }
 625
 626         for item in runtime:
 627             for idx in range(threads_nr):
 628                 if item[u"vectors"][idx] > 0:
 629                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 630                 elif item[u"calls"][idx] > 0:
 631                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 632                 elif item[u"suspends"][idx] > 0:
 633                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 634                 else:
 635                     clocks = 0.0
 636
 637                 if item[u"calls"][idx] > 0:
 638                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 639                 else:
 640                     vectors_call = 0.0
 641
 642                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 643                         int(item[u"suspends"][idx]):
 644                     oper[u"threads"][idx].append([
 645                         item[u"name"],
 646                         item[u"calls"][idx],
 647                         item[u"vectors"][idx],
 648                         item[u"suspends"][idx],
 649                         clocks,
 650                         vectors_call
 651                     ])
 652
 653         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 654
 655     def _get_ndrpdr_throughput(self, msg):
 656         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 657         message.
 658
 659         :param msg: The test message to be parsed.
 660         :type msg: str
 661         :returns: Parsed data as a dict and the status (PASS/FAIL).
 662         :rtype: tuple(dict, str)
 663         """
 664
 665         throughput = {
 666             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 667             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 668         }
 669         status = u"FAIL"
 670         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 671
 672         if groups is not None:
 673             try:
 674                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 675                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 676                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 677                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 678                 status = u"PASS"
 679             except (IndexError, ValueError):
 680                 pass
 681
 682         return throughput, status
 683
 684     def _get_plr_throughput(self, msg):
 685         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 686         message.
 687
 688         :param msg: The test message to be parsed.
 689         :type msg: str
 690         :returns: Parsed data as a dict and the status (PASS/FAIL).
 691         :rtype: tuple(dict, str)
 692         """
 693
 694         throughput = {
 695             u"LOWER": -1.0,
 696             u"UPPER": -1.0
 697         }
 698         status = u"FAIL"
 699         groups = re.search(self.REGEX_PLR_RATE, msg)
 700
 701         if groups is not None:
 702             try:
 703                 throughput[u"LOWER"] = float(groups.group(1))
 704                 throughput[u"UPPER"] = float(groups.group(2))
 705                 status = u"PASS"
 706             except (IndexError, ValueError):
 707                 pass
 708
 709         return throughput, status
 710
 711     def _get_ndrpdr_latency(self, msg):
 712         """Get LATENCY from the test message.
 713
 714         :param msg: The test message to be parsed.
 715         :type msg: str
 716         :returns: Parsed data as a dict and the status (PASS/FAIL).
 717         :rtype: tuple(dict, str)
 718         """
 719         latency_default = {
 720             u"min": -1.0,
 721             u"avg": -1.0,
 722             u"max": -1.0,
 723             u"hdrh": u""
 724         }
 725         latency = {
 726             u"NDR": {
 727                 u"direction1": copy.copy(latency_default),
 728                 u"direction2": copy.copy(latency_default)
 729             },
 730             u"PDR": {
 731                 u"direction1": copy.copy(latency_default),
 732                 u"direction2": copy.copy(latency_default)
 733             },
 734             u"LAT0": {
 735                 u"direction1": copy.copy(latency_default),
 736                 u"direction2": copy.copy(latency_default)
 737             },
 738             u"PDR10": {
 739                 u"direction1": copy.copy(latency_default),
 740                 u"direction2": copy.copy(latency_default)
 741             },
 742             u"PDR50": {
 743                 u"direction1": copy.copy(latency_default),
 744                 u"direction2": copy.copy(latency_default)
 745             },
 746             u"PDR90": {
 747                 u"direction1": copy.copy(latency_default),
 748                 u"direction2": copy.copy(latency_default)
 749             },
 750         }
 751
 752         # TODO: Rewrite when long and base are not needed
 753         groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
 754         if groups is None:
 755             groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 756         if groups is None:
 757             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 758         if groups is None:
 759             return latency, u"FAIL"
 760
 761         def process_latency(in_str):
 762             """Return object with parsed latency values.
 763
 764             TODO: Define class for the return type.
 765
 766             :param in_str: Input string, min/avg/max/hdrh format.
 767             :type in_str: str
 768             :returns: Dict with corresponding keys, except hdrh float values.
 769             :rtype dict:
 770             :throws IndexError: If in_str does not have enough substrings.
 771             :throws ValueError: If a substring does not convert to float.
 772             """
 773             in_list = in_str.split('/', 3)
 774
 775             rval = {
 776                 u"min": float(in_list[0]),
 777                 u"avg": float(in_list[1]),
 778                 u"max": float(in_list[2]),
 779                 u"hdrh": u""
 780             }
 781
 782             if len(in_list) == 4:
 783                 rval[u"hdrh"] = str(in_list[3])
 784
 785             return rval
 786
 787         try:
 788             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 789             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 790             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 791             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 792             if groups.lastindex == 4:
 793                 return latency, u"PASS"
 794         except (IndexError, ValueError):
 795             pass
 796
 797         try:
 798             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 799             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 800             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 801             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 802             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 803             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 804             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 805             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 806             if groups.lastindex == 12:
 807                 return latency, u"PASS"
 808         except (IndexError, ValueError):
 809             pass
 810
 811         # TODO: Remove when not needed
 812         latency[u"NDR10"] = {
 813             u"direction1": copy.copy(latency_default),
 814             u"direction2": copy.copy(latency_default)
 815         }
 816         latency[u"NDR50"] = {
 817             u"direction1": copy.copy(latency_default),
 818             u"direction2": copy.copy(latency_default)
 819         }
 820         latency[u"NDR90"] = {
 821             u"direction1": copy.copy(latency_default),
 822             u"direction2": copy.copy(latency_default)
 823         }
 824         try:
 825             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
 826             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
 827             latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
 828             latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
 829             latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
 830             latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
 831             latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
 832             latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
 833             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
 834             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
 835             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
 836             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
 837             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
 838             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
 839             return latency, u"PASS"
 840         except (IndexError, ValueError):
 841             pass
 842
 843         return latency, u"FAIL"
 844
 845     def visit_suite(self, suite):
 846         """Implements traversing through the suite and its direct children.
 847
 848         :param suite: Suite to process.
 849         :type suite: Suite
 850         :returns: Nothing.
 851         """
 852         if self.start_suite(suite) is not False:
 853             suite.suites.visit(self)
 854             suite.tests.visit(self)
 855             self.end_suite(suite)
 856
 857     def start_suite(self, suite):
 858         """Called when suite starts.
 859
 860         :param suite: Suite to process.
 861         :type suite: Suite
 862         :returns: Nothing.
 863         """
 864
 865         try:
 866             parent_name = suite.parent.name
 867         except AttributeError:
 868             return
 869
 870         doc_str = suite.doc.\
 871             replace(u'"', u"'").\
 872             replace(u'\n', u' ').\
 873             replace(u'\r', u'').\
 874             replace(u'*[', u' |br| *[').\
 875             replace(u"*", u"**").\
 876             replace(u' |br| *[', u'*[', 1)
 877
 878         self._data[u"suites"][suite.longname.lower().
 879                               replace(u'"', u"'").
 880                               replace(u" ", u"_")] = {
 881                                   u"name": suite.name.lower(),
 882                                   u"doc": doc_str,
 883                                   u"parent": parent_name,
 884                                   u"level": len(suite.longname.split(u"."))
 885                               }
 886
 887         suite.keywords.visit(self)
 888
 889     def end_suite(self, suite):
 890         """Called when suite ends.
 891
 892         :param suite: Suite to process.
 893         :type suite: Suite
 894         :returns: Nothing.
 895         """
 896
 897     def visit_test(self, test):
 898         """Implements traversing through the test.
 899
 900         :param test: Test to process.
 901         :type test: Test
 902         :returns: Nothing.
 903         """
 904         if self.start_test(test) is not False:
 905             test.keywords.visit(self)
 906             self.end_test(test)
 907
 908     def start_test(self, test):
 909         """Called when test starts.
 910
 911         :param test: Test to process.
 912         :type test: Test
 913         :returns: Nothing.
 914         """
 915
 916         self._sh_run_counter = 0
 917
 918         longname_orig = test.longname.lower()
 919
 920         # Check the ignore list
 921         if longname_orig in self._ignore:
 922             return
 923
 924         tags = [str(tag) for tag in test.tags]
 925         test_result = dict()
 926
 927         # Change the TC long name and name if defined in the mapping table
 928         longname = self._mapping.get(longname_orig, None)
 929         if longname is not None:
 930             name = longname.split(u'.')[-1]
 931             logging.debug(
 932                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
 933                 f"{name}"
 934             )
 935         else:
 936             longname = longname_orig
 937             name = test.name.lower()
 938
 939         # Remove TC number from the TC long name (backward compatibility):
 940         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
 941         # Remove TC number from the TC name (not needed):
 942         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
 943
 944         test_result[u"parent"] = test.parent.name.lower()
 945         test_result[u"tags"] = tags
 946         test_result["doc"] = test.doc.\
 947             replace(u'"', u"'").\
 948             replace(u'\n', u' ').\
 949             replace(u'\r', u'').\
 950             replace(u'[', u' |br| [').\
 951             replace(u' |br| [', u'[', 1)
 952         test_result[u"msg"] = self._get_data_from_perf_test_msg(test.message).\
 953             replace(u'\n', u' |br| ').\
 954             replace(u'\r', u'').\
 955             replace(u'"', u"'")
 956         test_result[u"type"] = u"FUNC"
 957         test_result[u"status"] = test.status
 958
 959         if u"PERFTEST" in tags:
 960             # Replace info about cores (e.g. -1c-) with the info about threads
 961             # and cores (e.g. -1t1c-) in the long test case names and in the
 962             # test case names if necessary.
 963             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
 964             if not groups:
 965                 tag_count = 0
 966                 tag_tc = str()
 967                 for tag in test_result[u"tags"]:
 968                     groups = re.search(self.REGEX_TC_TAG, tag)
 969                     if groups:
 970                         tag_count += 1
 971                         tag_tc = tag
 972
 973                 if tag_count == 1:
 974                     self._test_id = re.sub(self.REGEX_TC_NAME_NEW,
 975                                            f"-{tag_tc.lower()}-",
 976                                            self._test_id,
 977                                            count=1)
 978                     test_result[u"name"] = re.sub(self.REGEX_TC_NAME_NEW,
 979                                                   f"-{tag_tc.lower()}-",
 980                                                   test_result["name"],
 981                                                   count=1)
 982                 else:
 983                     test_result[u"status"] = u"FAIL"
 984                     self._data[u"tests"][self._test_id] = test_result
 985                     logging.debug(
 986                         f"The test {self._test_id} has no or more than one "
 987                         f"multi-threading tags.\n"
 988                         f"Tags: {test_result[u'tags']}"
 989                     )
 990                     return
 991
 992         if test.status == u"PASS":
 993             if u"NDRPDR" in tags:
 994                 test_result[u"type"] = u"NDRPDR"
 995                 test_result[u"throughput"], test_result[u"status"] = \
 996                     self._get_ndrpdr_throughput(test.message)
 997                 test_result[u"latency"], test_result[u"status"] = \
 998                     self._get_ndrpdr_latency(test.message)
 999             elif u"SOAK" in tags:
1000                 test_result[u"type"] = u"SOAK"
1001                 test_result[u"throughput"], test_result[u"status"] = \
1002                     self._get_plr_throughput(test.message)
1003             elif u"TCP" in tags:
1004                 test_result[u"type"] = u"TCP"
1005                 groups = re.search(self.REGEX_TCP, test.message)
1006                 test_result[u"result"] = int(groups.group(2))
1007             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1008                 if u"MRR" in tags:
1009                     test_result[u"type"] = u"MRR"
1010                 else:
1011                     test_result[u"type"] = u"BMRR"
1012
1013                 test_result[u"result"] = dict()
1014                 groups = re.search(self.REGEX_BMRR, test.message)
1015                 if groups is not None:
1016                     items_str = groups.group(1)
1017                     items_float = [float(item.strip()) for item
1018                                    in items_str.split(",")]
1019                     # Use whole list in CSIT-1180.
1020                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1021                     test_result[u"result"][u"receive-rate"] = stats.avg
1022                 else:
1023                     groups = re.search(self.REGEX_MRR, test.message)
1024                     test_result[u"result"][u"receive-rate"] = \
1025                         float(groups.group(3)) / float(groups.group(1))
1026             elif u"RECONF" in tags:
1027                 test_result[u"type"] = u"RECONF"
1028                 test_result[u"result"] = None
1029                 try:
1030                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1031                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1032                     test_result[u"result"] = {
1033                         u"loss": int(grps_loss.group(1)),
1034                         u"time": float(grps_time.group(1))
1035                     }
1036                 except (AttributeError, IndexError, ValueError, TypeError):
1037                     test_result[u"status"] = u"FAIL"
1038             elif u"DEVICETEST" in tags:
1039                 test_result[u"type"] = u"DEVICETEST"
1040             else:
1041                 test_result[u"status"] = u"FAIL"
1042                 self._data[u"tests"][self._test_id] = test_result
1043                 return
1044
1045         self._data[u"tests"][self._test_id] = test_result
1046
1047     def end_test(self, test):
1048         """Called when test ends.
1049
1050         :param test: Test to process.
1051         :type test: Test
1052         :returns: Nothing.
1053         """
1054
1055     def visit_keyword(self, keyword):
1056         """Implements traversing through the keyword and its child keywords.
1057
1058         :param keyword: Keyword to process.
1059         :type keyword: Keyword
1060         :returns: Nothing.
1061         """
1062         if self.start_keyword(keyword) is not False:
1063             self.end_keyword(keyword)
1064
1065     def start_keyword(self, keyword):
1066         """Called when keyword starts. Default implementation does nothing.
1067
1068         :param keyword: Keyword to process.
1069         :type keyword: Keyword
1070         :returns: Nothing.
1071         """
1072         try:
1073             if keyword.type == u"setup":
1074                 self.visit_setup_kw(keyword)
1075             elif keyword.type == u"teardown":
1076                 self.visit_teardown_kw(keyword)
1077             else:
1078                 self.visit_test_kw(keyword)
1079         except AttributeError:
1080             pass
1081
1082     def end_keyword(self, keyword):
1083         """Called when keyword ends. Default implementation does nothing.
1084
1085         :param keyword: Keyword to process.
1086         :type keyword: Keyword
1087         :returns: Nothing.
1088         """
1089
1090     def visit_test_kw(self, test_kw):
1091         """Implements traversing through the test keyword and its child
1092         keywords.
1093
1094         :param test_kw: Keyword to process.
1095         :type test_kw: Keyword
1096         :returns: Nothing.
1097         """
1098         for keyword in test_kw.keywords:
1099             if self.start_test_kw(keyword) is not False:
1100                 self.visit_test_kw(keyword)
1101                 self.end_test_kw(keyword)
1102
1103     def start_test_kw(self, test_kw):
1104         """Called when test keyword starts. Default implementation does
1105         nothing.
1106
1107         :param test_kw: Keyword to process.
1108         :type test_kw: Keyword
1109         :returns: Nothing.
1110         """
1111         if test_kw.name.count(u"Show Runtime On All Duts") or \
1112                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
1113             self._msg_type = u"test-show-runtime"
1114             self._sh_run_counter += 1
1115         elif test_kw.name.count(u"Install Dpdk Test") and not self._version:
1116             self._msg_type = u"dpdk-version"
1117         else:
1118             return
1119         test_kw.messages.visit(self)
1120
1121     def end_test_kw(self, test_kw):
1122         """Called when keyword ends. Default implementation does nothing.
1123
1124         :param test_kw: Keyword to process.
1125         :type test_kw: Keyword
1126         :returns: Nothing.
1127         """
1128
1129     def visit_setup_kw(self, setup_kw):
1130         """Implements traversing through the teardown keyword and its child
1131         keywords.
1132
1133         :param setup_kw: Keyword to process.
1134         :type setup_kw: Keyword
1135         :returns: Nothing.
1136         """
1137         for keyword in setup_kw.keywords:
1138             if self.start_setup_kw(keyword) is not False:
1139                 self.visit_setup_kw(keyword)
1140                 self.end_setup_kw(keyword)
1141
1142     def start_setup_kw(self, setup_kw):
1143         """Called when teardown keyword starts. Default implementation does
1144         nothing.
1145
1146         :param setup_kw: Keyword to process.
1147         :type setup_kw: Keyword
1148         :returns: Nothing.
1149         """
1150         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1151                 and not self._version:
1152             self._msg_type = u"vpp-version"
1153         elif setup_kw.name.count(u"Set Global Variable") \
1154                 and not self._timestamp:
1155             self._msg_type = u"timestamp"
1156         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1157             self._msg_type = u"testbed"
1158         else:
1159             return
1160         setup_kw.messages.visit(self)
1161
1162     def end_setup_kw(self, setup_kw):
1163         """Called when keyword ends. Default implementation does nothing.
1164
1165         :param setup_kw: Keyword to process.
1166         :type setup_kw: Keyword
1167         :returns: Nothing.
1168         """
1169
1170     def visit_teardown_kw(self, teardown_kw):
1171         """Implements traversing through the teardown keyword and its child
1172         keywords.
1173
1174         :param teardown_kw: Keyword to process.
1175         :type teardown_kw: Keyword
1176         :returns: Nothing.
1177         """
1178         for keyword in teardown_kw.keywords:
1179             if self.start_teardown_kw(keyword) is not False:
1180                 self.visit_teardown_kw(keyword)
1181                 self.end_teardown_kw(keyword)
1182
1183     def start_teardown_kw(self, teardown_kw):
1184         """Called when teardown keyword starts
1185
1186         :param teardown_kw: Keyword to process.
1187         :type teardown_kw: Keyword
1188         :returns: Nothing.
1189         """
1190
1191         if teardown_kw.name.count(u"Show Vat History On All Duts"):
1192             # TODO: Remove when not needed:
1193             self._conf_history_lookup_nr = 0
1194             self._msg_type = u"teardown-vat-history"
1195             teardown_kw.messages.visit(self)
1196         elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1197             self._conf_history_lookup_nr = 0
1198             self._msg_type = u"teardown-papi-history"
1199             teardown_kw.messages.visit(self)
1200
1201     def end_teardown_kw(self, teardown_kw):
1202         """Called when keyword ends. Default implementation does nothing.
1203
1204         :param teardown_kw: Keyword to process.
1205         :type teardown_kw: Keyword
1206         :returns: Nothing.
1207         """
1208
1209     def visit_message(self, msg):
1210         """Implements visiting the message.
1211
1212         :param msg: Message to process.
1213         :type msg: Message
1214         :returns: Nothing.
1215         """
1216         if self.start_message(msg) is not False:
1217             self.end_message(msg)
1218
1219     def start_message(self, msg):
1220         """Called when message starts. Get required information from messages:
1221         - VPP version.
1222
1223         :param msg: Message to process.
1224         :type msg: Message
1225         :returns: Nothing.
1226         """
1227
1228         if self._msg_type:
1229             self.parse_msg[self._msg_type](msg)
1230
1231     def end_message(self, msg):
1232         """Called when message ends. Default implementation does nothing.
1233
1234         :param msg: Message to process.
1235         :type msg: Message
1236         :returns: Nothing.
1237         """
1238
1239
1240 class InputData:
1241     """Input data
1242
1243     The data is extracted from output.xml files generated by Jenkins jobs and
1244     stored in pandas' DataFrames.
1245
1246     The data structure:
1247     - job name
1248       - build number
1249         - metadata
1250           (as described in ExecutionChecker documentation)
1251         - suites
1252           (as described in ExecutionChecker documentation)
1253         - tests
1254           (as described in ExecutionChecker documentation)
1255     """
1256
1257     def __init__(self, spec):
1258         """Initialization.
1259
1260         :param spec: Specification.
1261         :type spec: Specification
1262         """
1263
1264         # Specification:
1265         self._cfg = spec
1266
1267         # Data store:
1268         self._input_data = pd.Series()
1269
1270     @property
1271     def data(self):
1272         """Getter - Input data.
1273
1274         :returns: Input data
1275         :rtype: pandas.Series
1276         """
1277         return self._input_data
1278
1279     def metadata(self, job, build):
1280         """Getter - metadata
1281
1282         :param job: Job which metadata we want.
1283         :param build: Build which metadata we want.
1284         :type job: str
1285         :type build: str
1286         :returns: Metadata
1287         :rtype: pandas.Series
1288         """
1289
1290         return self.data[job][build][u"metadata"]
1291
1292     def suites(self, job, build):
1293         """Getter - suites
1294
1295         :param job: Job which suites we want.
1296         :param build: Build which suites we want.
1297         :type job: str
1298         :type build: str
1299         :returns: Suites.
1300         :rtype: pandas.Series
1301         """
1302
1303         return self.data[job][str(build)][u"suites"]
1304
1305     def tests(self, job, build):
1306         """Getter - tests
1307
1308         :param job: Job which tests we want.
1309         :param build: Build which tests we want.
1310         :type job: str
1311         :type build: str
1312         :returns: Tests.
1313         :rtype: pandas.Series
1314         """
1315
1316         return self.data[job][build][u"tests"]
1317
1318     def _parse_tests(self, job, build, log):
1319         """Process data from robot output.xml file and return JSON structured
1320         data.
1321
1322         :param job: The name of job which build output data will be processed.
1323         :param build: The build which output data will be processed.
1324         :param log: List of log messages.
1325         :type job: str
1326         :type build: dict
1327         :type log: list of tuples (severity, msg)
1328         :returns: JSON data structure.
1329         :rtype: dict
1330         """
1331
1332         metadata = {
1333             u"job": job,
1334             u"build": build
1335         }
1336
1337         with open(build[u"file-name"], u'r') as data_file:
1338             try:
1339                 result = ExecutionResult(data_file)
1340             except errors.DataError as err:
1341                 log.append(
1342                     (u"ERROR", f"Error occurred while parsing output.xml: "
1343                                f"{repr(err)}")
1344                 )
1345                 return None
1346         checker = ExecutionChecker(metadata, self._cfg.mapping,
1347                                    self._cfg.ignore)
1348         result.visit(checker)
1349
1350         return checker.data
1351
1352     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1353         """Download and parse the input data file.
1354
1355         :param pid: PID of the process executing this method.
1356         :param job: Name of the Jenkins job which generated the processed input
1357             file.
1358         :param build: Information about the Jenkins build which generated the
1359             processed input file.
1360         :param repeat: Repeat the download specified number of times if not
1361             successful.
1362         :type pid: int
1363         :type job: str
1364         :type build: dict
1365         :type repeat: int
1366         """
1367
1368         logs = list()
1369
1370         logs.append(
1371             (u"INFO", f"  Processing the job/build: {job}: {build[u'build']}")
1372         )
1373
1374         state = u"failed"
1375         success = False
1376         data = None
1377         do_repeat = repeat
1378         while do_repeat:
1379             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1380                                                    logs)
1381             if success:
1382                 break
1383             do_repeat -= 1
1384         if not success:
1385             logs.append(
1386                 (u"ERROR",
1387                  f"It is not possible to download the input data file from the "
1388                  f"job {job}, build {build[u'build']}, or it is damaged. "
1389                  f"Skipped.")
1390             )
1391         if success:
1392             logs.append(
1393                 (u"INFO",
1394                  f"    Processing data from the build {build[u'build']} ...")
1395             )
1396             data = self._parse_tests(job, build, logs)
1397             if data is None:
1398                 logs.append(
1399                     (u"ERROR",
1400                      f"Input data file from the job {job}, build "
1401                      f"{build[u'build']} is damaged. Skipped.")
1402                 )
1403             else:
1404                 state = u"processed"
1405
1406             try:
1407                 remove(build[u"file-name"])
1408             except OSError as err:
1409                 logs.append(
1410                     ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
1411                               f"{repr(err)}")
1412                 )
1413
1414         # If the time-period is defined in the specification file, remove all
1415         # files which are outside the time period.
1416         timeperiod = self._cfg.input.get(u"time-period", None)
1417         if timeperiod and data:
1418             now = dt.utcnow()
1419             timeperiod = timedelta(int(timeperiod))
1420             metadata = data.get(u"metadata", None)
1421             if metadata:
1422                 generated = metadata.get(u"generated", None)
1423                 if generated:
1424                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1425                     if (now - generated) > timeperiod:
1426                         # Remove the data and the file:
1427                         state = u"removed"
1428                         data = None
1429                         logs.append(
1430                             (u"INFO",
1431                              f"    The build {job}/{build[u'build']} is "
1432                              f"outdated, will be removed.")
1433                         )
1434         logs.append((u"INFO", u"  Done."))
1435
1436         for level, line in logs:
1437             if level == u"INFO":
1438                 logging.info(line)
1439             elif level == u"ERROR":
1440                 logging.error(line)
1441             elif level == u"DEBUG":
1442                 logging.debug(line)
1443             elif level == u"CRITICAL":
1444                 logging.critical(line)
1445             elif level == u"WARNING":
1446                 logging.warning(line)
1447
1448         return {u"data": data, u"state": state, u"job": job, u"build": build}
1449
1450     def download_and_parse_data(self, repeat=1):
1451         """Download the input data files, parse input data from input files and
1452         store in pandas' Series.
1453
1454         :param repeat: Repeat the download specified number of times if not
1455             successful.
1456         :type repeat: int
1457         """
1458
1459         logging.info(u"Downloading and parsing input files ...")
1460
1461         for job, builds in self._cfg.builds.items():
1462             for build in builds:
1463
1464                 result = self._download_and_parse_build(job, build, repeat)
1465                 build_nr = result[u"build"][u"build"]
1466
1467                 if result[u"data"]:
1468                     data = result[u"data"]
1469                     build_data = pd.Series({
1470                         u"metadata": pd.Series(
1471                             list(data[u"metadata"].values()),
1472                             index=list(data[u"metadata"].keys())
1473                         ),
1474                         u"suites": pd.Series(
1475                             list(data[u"suites"].values()),
1476                             index=list(data[u"suites"].keys())
1477                         ),
1478                         u"tests": pd.Series(
1479                             list(data[u"tests"].values()),
1480                             index=list(data[u"tests"].keys())
1481                         )
1482                     })
1483
1484                     if self._input_data.get(job, None) is None:
1485                         self._input_data[job] = pd.Series()
1486                     self._input_data[job][str(build_nr)] = build_data
1487
1488                     self._cfg.set_input_file_name(
1489                         job, build_nr, result[u"build"][u"file-name"])
1490
1491                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1492
1493                 mem_alloc = \
1494                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1495                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1496
1497         logging.info(u"Done.")
1498
1499     @staticmethod
1500     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1501         """Return the index of character in the string which is the end of tag.
1502
1503         :param tag_filter: The string where the end of tag is being searched.
1504         :param start: The index where the searching is stated.
1505         :param closer: The character which is the tag closer.
1506         :type tag_filter: str
1507         :type start: int
1508         :type closer: str
1509         :returns: The index of the tag closer.
1510         :rtype: int
1511         """
1512
1513         try:
1514             idx_opener = tag_filter.index(closer, start)
1515             return tag_filter.index(closer, idx_opener + 1)
1516         except ValueError:
1517             return None
1518
1519     @staticmethod
1520     def _condition(tag_filter):
1521         """Create a conditional statement from the given tag filter.
1522
1523         :param tag_filter: Filter based on tags from the element specification.
1524         :type tag_filter: str
1525         :returns: Conditional statement which can be evaluated.
1526         :rtype: str
1527         """
1528
1529         index = 0
1530         while True:
1531             index = InputData._end_of_tag(tag_filter, index)
1532             if index is None:
1533                 return tag_filter
1534             index += 1
1535             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1536
1537     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1538                     continue_on_error=False):
1539         """Filter required data from the given jobs and builds.
1540
1541         The output data structure is:
1542
1543         - job 1
1544           - build 1
1545             - test (or suite) 1 ID:
1546               - param 1
1547               - param 2
1548               ...
1549               - param n
1550             ...
1551             - test (or suite) n ID:
1552             ...
1553           ...
1554           - build n
1555         ...
1556         - job n
1557
1558         :param element: Element which will use the filtered data.
1559         :param params: Parameters which will be included in the output. If None,
1560             all parameters are included.
1561         :param data: If not None, this data is used instead of data specified
1562             in the element.
1563         :param data_set: The set of data to be filtered: tests, suites,
1564             metadata.
1565         :param continue_on_error: Continue if there is error while reading the
1566             data. The Item will be empty then
1567         :type element: pandas.Series
1568         :type params: list
1569         :type data: dict
1570         :type data_set: str
1571         :type continue_on_error: bool
1572         :returns: Filtered data.
1573         :rtype pandas.Series
1574         """
1575
1576         try:
1577             if data_set == "suites":
1578                 cond = u"True"
1579             elif element[u"filter"] in (u"all", u"template"):
1580                 cond = u"True"
1581             else:
1582                 cond = InputData._condition(element[u"filter"])
1583             logging.debug(f"   Filter: {cond}")
1584         except KeyError:
1585             logging.error(u"  No filter defined.")
1586             return None
1587
1588         if params is None:
1589             params = element.get(u"parameters", None)
1590             if params:
1591                 params.append(u"type")
1592
1593         data_to_filter = data if data else element[u"data"]
1594         data = pd.Series()
1595         try:
1596             for job, builds in data_to_filter.items():
1597                 data[job] = pd.Series()
1598                 for build in builds:
1599                     data[job][str(build)] = pd.Series()
1600                     try:
1601                         data_dict = dict(
1602                             self.data[job][str(build)][data_set].items())
1603                     except KeyError:
1604                         if continue_on_error:
1605                             continue
1606                         return None
1607
1608                     for test_id, test_data in data_dict.items():
1609                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1610                             data[job][str(build)][test_id] = pd.Series()
1611                             if params is None:
1612                                 for param, val in test_data.items():
1613                                     data[job][str(build)][test_id][param] = val
1614                             else:
1615                                 for param in params:
1616                                     try:
1617                                         data[job][str(build)][test_id][param] =\
1618                                             test_data[param]
1619                                     except KeyError:
1620                                         data[job][str(build)][test_id][param] =\
1621                                             u"No Data"
1622             return data
1623
1624         except (KeyError, IndexError, ValueError) as err:
1625             logging.error(
1626                 f"Missing mandatory parameter in the element specification: "
1627                 f"{repr(err)}"
1628             )
1629             return None
1630         except AttributeError as err:
1631             logging.error(repr(err))
1632             return None
1633         except SyntaxError as err:
1634             logging.error(
1635                 f"The filter {cond} is not correct. Check if all tags are "
1636                 f"enclosed by apostrophes.\n{repr(err)}"
1637             )
1638             return None
1639
1640     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1641                              continue_on_error=False):
1642         """Filter required data from the given jobs and builds.
1643
1644         The output data structure is:
1645
1646         - job 1
1647           - build 1
1648             - test (or suite) 1 ID:
1649               - param 1
1650               - param 2
1651               ...
1652               - param n
1653             ...
1654             - test (or suite) n ID:
1655             ...
1656           ...
1657           - build n
1658         ...
1659         - job n
1660
1661         :param element: Element which will use the filtered data.
1662         :param params: Parameters which will be included in the output. If None,
1663         all parameters are included.
1664         :param data_set: The set of data to be filtered: tests, suites,
1665         metadata.
1666         :param continue_on_error: Continue if there is error while reading the
1667         data. The Item will be empty then
1668         :type element: pandas.Series
1669         :type params: list
1670         :type data_set: str
1671         :type continue_on_error: bool
1672         :returns: Filtered data.
1673         :rtype pandas.Series
1674         """
1675
1676         include = element.get(u"include", None)
1677         if not include:
1678             logging.warning(u"No tests to include, skipping the element.")
1679             return None
1680
1681         if params is None:
1682             params = element.get(u"parameters", None)
1683             if params:
1684                 params.append(u"type")
1685
1686         data = pd.Series()
1687         try:
1688             for job, builds in element[u"data"].items():
1689                 data[job] = pd.Series()
1690                 for build in builds:
1691                     data[job][str(build)] = pd.Series()
1692                     for test in include:
1693                         try:
1694                             reg_ex = re.compile(str(test).lower())
1695                             for test_id in self.data[job][
1696                                     str(build)][data_set].keys():
1697                                 if re.match(reg_ex, str(test_id).lower()):
1698                                     test_data = self.data[job][
1699                                         str(build)][data_set][test_id]
1700                                     data[job][str(build)][test_id] = pd.Series()
1701                                     if params is None:
1702                                         for param, val in test_data.items():
1703                                             data[job][str(build)][test_id]\
1704                                                 [param] = val
1705                                     else:
1706                                         for param in params:
1707                                             try:
1708                                                 data[job][str(build)][
1709                                                     test_id][param] = \
1710                                                     test_data[param]
1711                                             except KeyError:
1712                                                 data[job][str(build)][
1713                                                     test_id][param] = u"No Data"
1714                         except KeyError as err:
1715                             logging.error(repr(err))
1716                             if continue_on_error:
1717                                 continue
1718                             return None
1719             return data
1720
1721         except (KeyError, IndexError, ValueError) as err:
1722             logging.error(
1723                 f"Missing mandatory parameter in the element "
1724                 f"specification: {repr(err)}"
1725             )
1726             return None
1727         except AttributeError as err:
1728             logging.error(repr(err))
1729             return None
1730
1731     @staticmethod
1732     def merge_data(data):
1733         """Merge data from more jobs and builds to a simple data structure.
1734
1735         The output data structure is:
1736
1737         - test (suite) 1 ID:
1738           - param 1
1739           - param 2
1740           ...
1741           - param n
1742         ...
1743         - test (suite) n ID:
1744         ...
1745
1746         :param data: Data to merge.
1747         :type data: pandas.Series
1748         :returns: Merged data.
1749         :rtype: pandas.Series
1750         """
1751
1752         logging.info(u"    Merging data ...")
1753
1754         merged_data = pd.Series()
1755         for builds in data.values:
1756             for item in builds.values:
1757                 for item_id, item_data in item.items():
1758                     merged_data[item_id] = item_data
1759
1760         return merged_data
1761
1762     def print_all_oper_data(self):
1763         """Print all operational data to console.
1764         """
1765
1766         tbl_hdr = (
1767             u"Name",
1768             u"Nr of Vectors",
1769             u"Nr of Packets",
1770             u"Suspends",
1771             u"Cycles per Packet",
1772             u"Average Vector Size"
1773         )
1774
1775         for job in self._input_data.values:
1776             for build in job.values:
1777                 for test_id, test_data in build[u"tests"].items():
1778                     print(f"{test_id}")
1779                     if test_data.get(u"show-run", None) is None:
1780                         continue
1781                     for dut_name, data in test_data[u"show-run"].items():
1782                         if data.get(u"threads", None) is None:
1783                             continue
1784                         print(f"Host IP: {data.get(u'host', '')}, "
1785                               f"Socket: {data.get(u'socket', '')}")
1786                         for thread_nr, thread in data[u"threads"].items():
1787                             txt_table = prettytable.PrettyTable(tbl_hdr)
1788                             avg = 0.0
1789                             for row in thread:
1790                                 txt_table.add_row(row)
1791                                 avg += row[-1]
1792                             if len(thread) == 0:
1793                                 avg = u""
1794                             else:
1795                                 avg = f", Average Vector Size per Node: " \
1796                                       f"{(avg / len(thread)):.2f}"
1797                             th_name = u"main" if thread_nr == 0 \
1798                                 else f"worker_{thread_nr}"
1799                             print(f"{dut_name}, {th_name}{avg}")
1800                             txt_table.float_format = u".2"
1801                             txt_table.align = u"r"
1802                             txt_table.align[u"Name"] = u"l"
1803                             print(f"{txt_table.get_string()}\n")