resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove
  29 from datetime import datetime as dt
  30 from datetime import timedelta
  31 from json import loads
  32
  33 import hdrh.histogram
  34 import hdrh.codec
  35 import prettytable
  36 import pandas as pd
  37
  38 from robot.api import ExecutionResult, ResultVisitor
  39 from robot import errors
  40
  41 from resources.libraries.python import jumpavg
  42 from input_data_files import download_and_unzip_data_file
  43
  44
  45 # Separator used in file names
  46 SEPARATOR = u"__"
  47
  48
  49 class ExecutionChecker(ResultVisitor):
  50     """Class to traverse through the test suite structure.
  51
  52     The functionality implemented in this class generates a json structure:
  53
  54     Performance tests:
  55
  56     {
  57         "metadata": {
  58             "generated": "Timestamp",
  59             "version": "SUT version",
  60             "job": "Jenkins job name",
  61             "build": "Information about the build"
  62         },
  63         "suites": {
  64             "Suite long name 1": {
  65                 "name": Suite name,
  66                 "doc": "Suite 1 documentation",
  67                 "parent": "Suite 1 parent",
  68                 "level": "Level of the suite in the suite hierarchy"
  69             }
  70             "Suite long name N": {
  71                 "name": Suite name,
  72                 "doc": "Suite N documentation",
  73                 "parent": "Suite 2 parent",
  74                 "level": "Level of the suite in the suite hierarchy"
  75             }
  76         }
  77         "tests": {
  78             # NDRPDR tests:
  79             "ID": {
  80                 "name": "Test name",
  81                 "parent": "Name of the parent of the test",
  82                 "doc": "Test documentation",
  83                 "msg": "Test message",
  84                 "conf-history": "DUT1 and DUT2 VAT History",
  85                 "show-run": "Show Run",
  86                 "tags": ["tag 1", "tag 2", "tag n"],
  87                 "type": "NDRPDR",
  88                 "status": "PASS" | "FAIL",
  89                 "throughput": {
  90                     "NDR": {
  91                         "LOWER": float,
  92                         "UPPER": float
  93                     },
  94                     "PDR": {
  95                         "LOWER": float,
  96                         "UPPER": float
  97                     }
  98                 },
  99                 "latency": {
 100                     "NDR": {
 101                         "direction1": {
 102                             "min": float,
 103                             "avg": float,
 104                             "max": float,
 105                             "hdrh": str
 106                         },
 107                         "direction2": {
 108                             "min": float,
 109                             "avg": float,
 110                             "max": float,
 111                             "hdrh": str
 112                         }
 113                     },
 114                     "PDR": {
 115                         "direction1": {
 116                             "min": float,
 117                             "avg": float,
 118                             "max": float,
 119                             "hdrh": str
 120                         },
 121                         "direction2": {
 122                             "min": float,
 123                             "avg": float,
 124                             "max": float,
 125                             "hdrh": str
 126                         }
 127                     }
 128                 }
 129             }
 130
 131             # TCP tests:
 132             "ID": {
 133                 "name": "Test name",
 134                 "parent": "Name of the parent of the test",
 135                 "doc": "Test documentation",
 136                 "msg": "Test message",
 137                 "tags": ["tag 1", "tag 2", "tag n"],
 138                 "type": "TCP",
 139                 "status": "PASS" | "FAIL",
 140                 "result": int
 141             }
 142
 143             # MRR, BMRR tests:
 144             "ID": {
 145                 "name": "Test name",
 146                 "parent": "Name of the parent of the test",
 147                 "doc": "Test documentation",
 148                 "msg": "Test message",
 149                 "tags": ["tag 1", "tag 2", "tag n"],
 150                 "type": "MRR" | "BMRR",
 151                 "status": "PASS" | "FAIL",
 152                 "result": {
 153                     "receive-rate": float,
 154                     # Average of a list, computed using AvgStdevStats.
 155                     # In CSIT-1180, replace with List[float].
 156                 }
 157             }
 158
 159             "ID" {
 160                 # next test
 161             }
 162         }
 163     }
 164
 165
 166     Functional tests:
 167
 168     {
 169         "metadata": {  # Optional
 170             "version": "VPP version",
 171             "job": "Jenkins job name",
 172             "build": "Information about the build"
 173         },
 174         "suites": {
 175             "Suite name 1": {
 176                 "doc": "Suite 1 documentation",
 177                 "parent": "Suite 1 parent",
 178                 "level": "Level of the suite in the suite hierarchy"
 179             }
 180             "Suite name N": {
 181                 "doc": "Suite N documentation",
 182                 "parent": "Suite 2 parent",
 183                 "level": "Level of the suite in the suite hierarchy"
 184             }
 185         }
 186         "tests": {
 187             "ID": {
 188                 "name": "Test name",
 189                 "parent": "Name of the parent of the test",
 190                 "doc": "Test documentation"
 191                 "msg": "Test message"
 192                 "tags": ["tag 1", "tag 2", "tag n"],
 193                 "conf-history": "DUT1 and DUT2 VAT History"
 194                 "show-run": "Show Run"
 195                 "status": "PASS" | "FAIL"
 196             },
 197             "ID" {
 198                 # next test
 199             }
 200         }
 201     }
 202
 203     .. note:: ID is the lowercase full path to the test.
 204     """
 205
 206     REGEX_PLR_RATE = re.compile(
 207         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 208         r'PLRsearch upper bound::?\s(\d+.\d+)'
 209     )
 210     REGEX_NDRPDR_RATE = re.compile(
 211         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 212         r'NDR_UPPER:\s(\d+.\d+).*\n'
 213         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 214         r'PDR_UPPER:\s(\d+.\d+)'
 215     )
 216     REGEX_PERF_MSG_INFO = re.compile(
 217         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 218         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 219         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 220         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 221         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 222     )
 223     # TODO: Remove when not needed
 224     REGEX_NDRPDR_LAT_BASE = re.compile(
 225         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 226         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 227     )
 228     REGEX_NDRPDR_LAT = re.compile(
 229         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 230         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 231         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 232         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 233         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 234         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 235     )
 236     # TODO: Remove when not needed
 237     REGEX_NDRPDR_LAT_LONG = re.compile(
 238         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 239         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 240         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 241         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 242         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 243         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 244         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 245         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 246         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 247     )
 248     REGEX_VERSION_VPP = re.compile(
 249         r"(return STDOUT Version:\s*|"
 250         r"VPP Version:\s*|VPP version:\s*)(.*)"
 251     )
 252     REGEX_VERSION_DPDK = re.compile(
 253         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 254     )
 255     REGEX_TCP = re.compile(
 256         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 257     )
 258     REGEX_MRR = re.compile(
 259         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 260         r'tx\s(\d*),\srx\s(\d*)'
 261     )
 262     REGEX_BMRR = re.compile(
 263         r'Maximum Receive Rate trial results'
 264         r' in packets per second: \[(.*)\]'
 265     )
 266     REGEX_RECONF_LOSS = re.compile(
 267         r'Packets lost due to reconfig: (\d*)'
 268     )
 269     REGEX_RECONF_TIME = re.compile(
 270         r'Implied time lost: (\d*.[\de-]*)'
 271     )
 272     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 273
 274     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 275
 276     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 277
 278     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 279
 280     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 281
 282     def __init__(self, metadata, mapping, ignore):
 283         """Initialisation.
 284
 285         :param metadata: Key-value pairs to be included in "metadata" part of
 286             JSON structure.
 287         :param mapping: Mapping of the old names of test cases to the new
 288             (actual) one.
 289         :param ignore: List of TCs to be ignored.
 290         :type metadata: dict
 291         :type mapping: dict
 292         :type ignore: list
 293         """
 294
 295         # Type of message to parse out from the test messages
 296         self._msg_type = None
 297
 298         # VPP version
 299         self._version = None
 300
 301         # Timestamp
 302         self._timestamp = None
 303
 304         # Testbed. The testbed is identified by TG node IP address.
 305         self._testbed = None
 306
 307         # Mapping of TCs long names
 308         self._mapping = mapping
 309
 310         # Ignore list
 311         self._ignore = ignore
 312
 313         # Number of PAPI History messages found:
 314         # 0 - no message
 315         # 1 - PAPI History of DUT1
 316         # 2 - PAPI History of DUT2
 317         self._conf_history_lookup_nr = 0
 318
 319         self._sh_run_counter = 0
 320
 321         # Test ID of currently processed test- the lowercase full path to the
 322         # test
 323         self._test_id = None
 324
 325         # The main data structure
 326         self._data = {
 327             u"metadata": OrderedDict(),
 328             u"suites": OrderedDict(),
 329             u"tests": OrderedDict()
 330         }
 331
 332         # Save the provided metadata
 333         for key, val in metadata.items():
 334             self._data[u"metadata"][key] = val
 335
 336         # Dictionary defining the methods used to parse different types of
 337         # messages
 338         self.parse_msg = {
 339             u"timestamp": self._get_timestamp,
 340             u"vpp-version": self._get_vpp_version,
 341             u"dpdk-version": self._get_dpdk_version,
 342             # TODO: Remove when not needed:
 343             u"teardown-vat-history": self._get_vat_history,
 344             u"teardown-papi-history": self._get_papi_history,
 345             u"test-show-runtime": self._get_show_run,
 346             u"testbed": self._get_testbed
 347         }
 348
 349     @property
 350     def data(self):
 351         """Getter - Data parsed from the XML file.
 352
 353         :returns: Data parsed from the XML file.
 354         :rtype: dict
 355         """
 356         return self._data
 357
 358     def _get_data_from_perf_test_msg(self, msg):
 359         """Get
 360             - NDR_LOWER
 361             - LATENCY
 362             - NDR_UPPER
 363             - PDR_LOWER
 364             - LATENCY
 365             - PDR_UPPER
 366         from message of NDRPDR performance tests.
 367
 368         :param msg: Message to be processed.
 369         :type msg: str
 370         :returns: Processed message or original message if a problem occurs.
 371         :rtype: str
 372         """
 373
 374         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 375         if not groups or groups.lastindex != 10:
 376             return msg
 377
 378         try:
 379             data = {
 380                 u"ndr_low": float(groups.group(1)),
 381                 u"ndr_low_b": float(groups.group(2)),
 382                 u"pdr_low": float(groups.group(3)),
 383                 u"pdr_low_b": float(groups.group(4)),
 384                 u"pdr_lat_90_1": groups.group(5),
 385                 u"pdr_lat_90_2": groups.group(6),
 386                 u"pdr_lat_50_1": groups.group(7),
 387                 u"pdr_lat_50_2": groups.group(8),
 388                 u"pdr_lat_10_1": groups.group(9),
 389                 u"pdr_lat_10_2": groups.group(10),
 390             }
 391         except (AttributeError, IndexError, ValueError, KeyError):
 392             return msg
 393
 394         def _process_lat(in_str_1, in_str_2):
 395             """Extract min, avg, max values from latency string.
 396
 397             :param in_str_1: Latency string for one direction produced by robot
 398                 framework.
 399             :param in_str_2: Latency string for second direction produced by
 400                 robot framework.
 401             :type in_str_1: str
 402             :type in_str_2: str
 403             :returns: Processed latency string or empty string if a problem
 404                 occurs.
 405             :rtype: tuple(str, str)
 406             """
 407             in_list_1 = in_str_1.split('/', 3)
 408             in_list_2 = in_str_2.split('/', 3)
 409
 410             if len(in_list_1) != 4 and len(in_list_2) != 4:
 411                 return u""
 412
 413             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 414             try:
 415                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 416             except hdrh.codec.HdrLengthException:
 417                 return u""
 418
 419             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 420             try:
 421                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 422             except hdrh.codec.HdrLengthException:
 423                 return u""
 424
 425             if hdr_lat_1 and hdr_lat_2:
 426                 hdr_lat_1_50 = hdr_lat_1.get_value_at_percentile(50.0)
 427                 hdr_lat_1_90 = hdr_lat_1.get_value_at_percentile(90.0)
 428                 hdr_lat_1_99 = hdr_lat_1.get_value_at_percentile(99.0)
 429                 hdr_lat_2_50 = hdr_lat_2.get_value_at_percentile(50.0)
 430                 hdr_lat_2_90 = hdr_lat_2.get_value_at_percentile(90.0)
 431                 hdr_lat_2_99 = hdr_lat_2.get_value_at_percentile(99.0)
 432
 433                 if (hdr_lat_1_50 + hdr_lat_1_90 + hdr_lat_1_99 +
 434                         hdr_lat_2_50 + hdr_lat_2_90 + hdr_lat_2_99):
 435                     return (
 436                         f"{hdr_lat_1_50} {hdr_lat_1_90} {hdr_lat_1_99} , "
 437                         f"{hdr_lat_2_50} {hdr_lat_2_90} {hdr_lat_2_99}"
 438                     )
 439
 440             return u""
 441
 442         try:
 443             pdr_lat_10 = _process_lat(data[u'pdr_lat_10_1'],
 444                                       data[u'pdr_lat_10_2'])
 445             pdr_lat_50 = _process_lat(data[u'pdr_lat_50_1'],
 446                                       data[u'pdr_lat_50_2'])
 447             pdr_lat_90 = _process_lat(data[u'pdr_lat_90_1'],
 448                                       data[u'pdr_lat_90_2'])
 449             pdr_lat_10 = f"\n3. {pdr_lat_10}" if pdr_lat_10 else u""
 450             pdr_lat_50 = f"\n4. {pdr_lat_50}" if pdr_lat_50 else u""
 451             pdr_lat_90 = f"\n5. {pdr_lat_90}" if pdr_lat_90 else u""
 452
 453             return (
 454                 f"1. {(data[u'ndr_low'] / 1e6):.2f} {data[u'ndr_low_b']:.2f}"
 455                 f"\n2. {(data[u'pdr_low'] / 1e6):.2f} {data[u'pdr_low_b']:.2f}"
 456                 f"{pdr_lat_10}"
 457                 f"{pdr_lat_50}"
 458                 f"{pdr_lat_90}"
 459             )
 460         except (AttributeError, IndexError, ValueError, KeyError):
 461             return msg
 462
 463     def _get_testbed(self, msg):
 464         """Called when extraction of testbed IP is required.
 465         The testbed is identified by TG node IP address.
 466
 467         :param msg: Message to process.
 468         :type msg: Message
 469         :returns: Nothing.
 470         """
 471
 472         if msg.message.count(u"Setup of TG node") or \
 473                 msg.message.count(u"Setup of node TG host"):
 474             reg_tg_ip = re.compile(
 475                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 476             try:
 477                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 478             except (KeyError, ValueError, IndexError, AttributeError):
 479                 pass
 480             finally:
 481                 self._data[u"metadata"][u"testbed"] = self._testbed
 482                 self._msg_type = None
 483
 484     def _get_vpp_version(self, msg):
 485         """Called when extraction of VPP version is required.
 486
 487         :param msg: Message to process.
 488         :type msg: Message
 489         :returns: Nothing.
 490         """
 491
 492         if msg.message.count(u"return STDOUT Version:") or \
 493             msg.message.count(u"VPP Version:") or \
 494             msg.message.count(u"VPP version:"):
 495             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 496                                 group(2))
 497             self._data[u"metadata"][u"version"] = self._version
 498             self._msg_type = None
 499
 500     def _get_dpdk_version(self, msg):
 501         """Called when extraction of DPDK version is required.
 502
 503         :param msg: Message to process.
 504         :type msg: Message
 505         :returns: Nothing.
 506         """
 507
 508         if msg.message.count(u"DPDK Version:"):
 509             try:
 510                 self._version = str(re.search(
 511                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 512                 self._data[u"metadata"][u"version"] = self._version
 513             except IndexError:
 514                 pass
 515             finally:
 516                 self._msg_type = None
 517
 518     def _get_timestamp(self, msg):
 519         """Called when extraction of timestamp is required.
 520
 521         :param msg: Message to process.
 522         :type msg: Message
 523         :returns: Nothing.
 524         """
 525
 526         self._timestamp = msg.timestamp[:14]
 527         self._data[u"metadata"][u"generated"] = self._timestamp
 528         self._msg_type = None
 529
 530     def _get_vat_history(self, msg):
 531         """Called when extraction of VAT command history is required.
 532
 533         TODO: Remove when not needed.
 534
 535         :param msg: Message to process.
 536         :type msg: Message
 537         :returns: Nothing.
 538         """
 539         if msg.message.count(u"VAT command history:"):
 540             self._conf_history_lookup_nr += 1
 541             if self._conf_history_lookup_nr == 1:
 542                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 543             else:
 544                 self._msg_type = None
 545             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 546                           r"VAT command history:", u"",
 547                           msg.message, count=1).replace(u'\n', u' |br| ').\
 548                 replace(u'"', u"'")
 549
 550             self._data[u"tests"][self._test_id][u"conf-history"] += (
 551                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 552             )
 553
 554     def _get_papi_history(self, msg):
 555         """Called when extraction of PAPI command history is required.
 556
 557         :param msg: Message to process.
 558         :type msg: Message
 559         :returns: Nothing.
 560         """
 561         if msg.message.count(u"PAPI command history:"):
 562             self._conf_history_lookup_nr += 1
 563             if self._conf_history_lookup_nr == 1:
 564                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 565             else:
 566                 self._msg_type = None
 567             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 568                           r"PAPI command history:", u"",
 569                           msg.message, count=1).replace(u'\n', u' |br| ').\
 570                 replace(u'"', u"'")
 571             self._data[u"tests"][self._test_id][u"conf-history"] += (
 572                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 573             )
 574
 575     def _get_show_run(self, msg):
 576         """Called when extraction of VPP operational data (output of CLI command
 577         Show Runtime) is required.
 578
 579         :param msg: Message to process.
 580         :type msg: Message
 581         :returns: Nothing.
 582         """
 583
 584         if not msg.message.count(u"stats runtime"):
 585             return
 586
 587         # Temporary solution
 588         if self._sh_run_counter > 1:
 589             return
 590
 591         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 592             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 593
 594         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 595         if not groups:
 596             return
 597         try:
 598             host = groups.group(1)
 599         except (AttributeError, IndexError):
 600             host = u""
 601         try:
 602             sock = groups.group(2)
 603         except (AttributeError, IndexError):
 604             sock = u""
 605
 606         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 607                         replace(u"'", u'"').replace(u'b"', u'"').
 608                         replace(u'u"', u'"').split(u":", 1)[1])
 609
 610         try:
 611             threads_nr = len(runtime[0][u"clocks"])
 612         except (IndexError, KeyError):
 613             return
 614
 615         dut = u"DUT{nr}".format(
 616             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 617
 618         oper = {
 619             u"host": host,
 620             u"socket": sock,
 621             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 622         }
 623
 624         for item in runtime:
 625             for idx in range(threads_nr):
 626                 if item[u"vectors"][idx] > 0:
 627                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 628                 elif item[u"calls"][idx] > 0:
 629                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 630                 elif item[u"suspends"][idx] > 0:
 631                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 632                 else:
 633                     clocks = 0.0
 634
 635                 if item[u"calls"][idx] > 0:
 636                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 637                 else:
 638                     vectors_call = 0.0
 639
 640                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 641                         int(item[u"suspends"][idx]):
 642                     oper[u"threads"][idx].append([
 643                         item[u"name"],
 644                         item[u"calls"][idx],
 645                         item[u"vectors"][idx],
 646                         item[u"suspends"][idx],
 647                         clocks,
 648                         vectors_call
 649                     ])
 650
 651         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 652
 653     def _get_ndrpdr_throughput(self, msg):
 654         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 655         message.
 656
 657         :param msg: The test message to be parsed.
 658         :type msg: str
 659         :returns: Parsed data as a dict and the status (PASS/FAIL).
 660         :rtype: tuple(dict, str)
 661         """
 662
 663         throughput = {
 664             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 665             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 666         }
 667         status = u"FAIL"
 668         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 669
 670         if groups is not None:
 671             try:
 672                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 673                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 674                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 675                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 676                 status = u"PASS"
 677             except (IndexError, ValueError):
 678                 pass
 679
 680         return throughput, status
 681
 682     def _get_plr_throughput(self, msg):
 683         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 684         message.
 685
 686         :param msg: The test message to be parsed.
 687         :type msg: str
 688         :returns: Parsed data as a dict and the status (PASS/FAIL).
 689         :rtype: tuple(dict, str)
 690         """
 691
 692         throughput = {
 693             u"LOWER": -1.0,
 694             u"UPPER": -1.0
 695         }
 696         status = u"FAIL"
 697         groups = re.search(self.REGEX_PLR_RATE, msg)
 698
 699         if groups is not None:
 700             try:
 701                 throughput[u"LOWER"] = float(groups.group(1))
 702                 throughput[u"UPPER"] = float(groups.group(2))
 703                 status = u"PASS"
 704             except (IndexError, ValueError):
 705                 pass
 706
 707         return throughput, status
 708
 709     def _get_ndrpdr_latency(self, msg):
 710         """Get LATENCY from the test message.
 711
 712         :param msg: The test message to be parsed.
 713         :type msg: str
 714         :returns: Parsed data as a dict and the status (PASS/FAIL).
 715         :rtype: tuple(dict, str)
 716         """
 717         latency_default = {
 718             u"min": -1.0,
 719             u"avg": -1.0,
 720             u"max": -1.0,
 721             u"hdrh": u""
 722         }
 723         latency = {
 724             u"NDR": {
 725                 u"direction1": copy.copy(latency_default),
 726                 u"direction2": copy.copy(latency_default)
 727             },
 728             u"PDR": {
 729                 u"direction1": copy.copy(latency_default),
 730                 u"direction2": copy.copy(latency_default)
 731             },
 732             u"LAT0": {
 733                 u"direction1": copy.copy(latency_default),
 734                 u"direction2": copy.copy(latency_default)
 735             },
 736             u"PDR10": {
 737                 u"direction1": copy.copy(latency_default),
 738                 u"direction2": copy.copy(latency_default)
 739             },
 740             u"PDR50": {
 741                 u"direction1": copy.copy(latency_default),
 742                 u"direction2": copy.copy(latency_default)
 743             },
 744             u"PDR90": {
 745                 u"direction1": copy.copy(latency_default),
 746                 u"direction2": copy.copy(latency_default)
 747             },
 748         }
 749
 750         # TODO: Rewrite when long and base are not needed
 751         groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
 752         if groups is None:
 753             groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 754         if groups is None:
 755             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 756         if groups is None:
 757             return latency, u"FAIL"
 758
 759         def process_latency(in_str):
 760             """Return object with parsed latency values.
 761
 762             TODO: Define class for the return type.
 763
 764             :param in_str: Input string, min/avg/max/hdrh format.
 765             :type in_str: str
 766             :returns: Dict with corresponding keys, except hdrh float values.
 767             :rtype dict:
 768             :throws IndexError: If in_str does not have enough substrings.
 769             :throws ValueError: If a substring does not convert to float.
 770             """
 771             in_list = in_str.split('/', 3)
 772
 773             rval = {
 774                 u"min": float(in_list[0]),
 775                 u"avg": float(in_list[1]),
 776                 u"max": float(in_list[2]),
 777                 u"hdrh": u""
 778             }
 779
 780             if len(in_list) == 4:
 781                 rval[u"hdrh"] = str(in_list[3])
 782
 783             return rval
 784
 785         try:
 786             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 787             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 788             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 789             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 790             if groups.lastindex == 4:
 791                 return latency, u"PASS"
 792         except (IndexError, ValueError):
 793             pass
 794
 795         try:
 796             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 797             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 798             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 799             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 800             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 801             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 802             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 803             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 804             if groups.lastindex == 12:
 805                 return latency, u"PASS"
 806         except (IndexError, ValueError):
 807             pass
 808
 809         # TODO: Remove when not needed
 810         latency[u"NDR10"] = {
 811             u"direction1": copy.copy(latency_default),
 812             u"direction2": copy.copy(latency_default)
 813         }
 814         latency[u"NDR50"] = {
 815             u"direction1": copy.copy(latency_default),
 816             u"direction2": copy.copy(latency_default)
 817         }
 818         latency[u"NDR90"] = {
 819             u"direction1": copy.copy(latency_default),
 820             u"direction2": copy.copy(latency_default)
 821         }
 822         try:
 823             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
 824             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
 825             latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
 826             latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
 827             latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
 828             latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
 829             latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
 830             latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
 831             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
 832             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
 833             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
 834             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
 835             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
 836             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
 837             return latency, u"PASS"
 838         except (IndexError, ValueError):
 839             pass
 840
 841         return latency, u"FAIL"
 842
 843     def visit_suite(self, suite):
 844         """Implements traversing through the suite and its direct children.
 845
 846         :param suite: Suite to process.
 847         :type suite: Suite
 848         :returns: Nothing.
 849         """
 850         if self.start_suite(suite) is not False:
 851             suite.suites.visit(self)
 852             suite.tests.visit(self)
 853             self.end_suite(suite)
 854
 855     def start_suite(self, suite):
 856         """Called when suite starts.
 857
 858         :param suite: Suite to process.
 859         :type suite: Suite
 860         :returns: Nothing.
 861         """
 862
 863         try:
 864             parent_name = suite.parent.name
 865         except AttributeError:
 866             return
 867
 868         doc_str = suite.doc.\
 869             replace(u'"', u"'").\
 870             replace(u'\n', u' ').\
 871             replace(u'\r', u'').\
 872             replace(u'*[', u' |br| *[').\
 873             replace(u"*", u"**").\
 874             replace(u' |br| *[', u'*[', 1)
 875
 876         self._data[u"suites"][suite.longname.lower().
 877                               replace(u'"', u"'").
 878                               replace(u" ", u"_")] = {
 879                                   u"name": suite.name.lower(),
 880                                   u"doc": doc_str,
 881                                   u"parent": parent_name,
 882                                   u"level": len(suite.longname.split(u"."))
 883                               }
 884
 885         suite.keywords.visit(self)
 886
 887     def end_suite(self, suite):
 888         """Called when suite ends.
 889
 890         :param suite: Suite to process.
 891         :type suite: Suite
 892         :returns: Nothing.
 893         """
 894
 895     def visit_test(self, test):
 896         """Implements traversing through the test.
 897
 898         :param test: Test to process.
 899         :type test: Test
 900         :returns: Nothing.
 901         """
 902         if self.start_test(test) is not False:
 903             test.keywords.visit(self)
 904             self.end_test(test)
 905
 906     def start_test(self, test):
 907         """Called when test starts.
 908
 909         :param test: Test to process.
 910         :type test: Test
 911         :returns: Nothing.
 912         """
 913
 914         self._sh_run_counter = 0
 915
 916         longname_orig = test.longname.lower()
 917
 918         # Check the ignore list
 919         if longname_orig in self._ignore:
 920             return
 921
 922         tags = [str(tag) for tag in test.tags]
 923         test_result = dict()
 924
 925         # Change the TC long name and name if defined in the mapping table
 926         longname = self._mapping.get(longname_orig, None)
 927         if longname is not None:
 928             name = longname.split(u'.')[-1]
 929             logging.debug(
 930                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
 931                 f"{name}"
 932             )
 933         else:
 934             longname = longname_orig
 935             name = test.name.lower()
 936
 937         # Remove TC number from the TC long name (backward compatibility):
 938         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
 939         # Remove TC number from the TC name (not needed):
 940         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
 941
 942         test_result[u"parent"] = test.parent.name.lower()
 943         test_result[u"tags"] = tags
 944         test_result["doc"] = test.doc.\
 945             replace(u'"', u"'").\
 946             replace(u'\n', u' ').\
 947             replace(u'\r', u'').\
 948             replace(u'[', u' |br| [').\
 949             replace(u' |br| [', u'[', 1)
 950         test_result[u"msg"] = self._get_data_from_perf_test_msg(test.message).\
 951             replace(u'\n', u' |br| ').\
 952             replace(u'\r', u'').\
 953             replace(u'"', u"'")
 954         test_result[u"type"] = u"FUNC"
 955         test_result[u"status"] = test.status
 956
 957         if u"PERFTEST" in tags:
 958             # Replace info about cores (e.g. -1c-) with the info about threads
 959             # and cores (e.g. -1t1c-) in the long test case names and in the
 960             # test case names if necessary.
 961             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
 962             if not groups:
 963                 tag_count = 0
 964                 tag_tc = str()
 965                 for tag in test_result[u"tags"]:
 966                     groups = re.search(self.REGEX_TC_TAG, tag)
 967                     if groups:
 968                         tag_count += 1
 969                         tag_tc = tag
 970
 971                 if tag_count == 1:
 972                     self._test_id = re.sub(self.REGEX_TC_NAME_NEW,
 973                                            f"-{tag_tc.lower()}-",
 974                                            self._test_id,
 975                                            count=1)
 976                     test_result[u"name"] = re.sub(self.REGEX_TC_NAME_NEW,
 977                                                   f"-{tag_tc.lower()}-",
 978                                                   test_result["name"],
 979                                                   count=1)
 980                 else:
 981                     test_result[u"status"] = u"FAIL"
 982                     self._data[u"tests"][self._test_id] = test_result
 983                     logging.debug(
 984                         f"The test {self._test_id} has no or more than one "
 985                         f"multi-threading tags.\n"
 986                         f"Tags: {test_result[u'tags']}"
 987                     )
 988                     return
 989
 990         if test.status == u"PASS":
 991             if u"NDRPDR" in tags:
 992                 test_result[u"type"] = u"NDRPDR"
 993                 test_result[u"throughput"], test_result[u"status"] = \
 994                     self._get_ndrpdr_throughput(test.message)
 995                 test_result[u"latency"], test_result[u"status"] = \
 996                     self._get_ndrpdr_latency(test.message)
 997             elif u"SOAK" in tags:
 998                 test_result[u"type"] = u"SOAK"
 999                 test_result[u"throughput"], test_result[u"status"] = \
1000                     self._get_plr_throughput(test.message)
1001             elif u"TCP" in tags:
1002                 test_result[u"type"] = u"TCP"
1003                 groups = re.search(self.REGEX_TCP, test.message)
1004                 test_result[u"result"] = int(groups.group(2))
1005             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1006                 if u"MRR" in tags:
1007                     test_result[u"type"] = u"MRR"
1008                 else:
1009                     test_result[u"type"] = u"BMRR"
1010
1011                 test_result[u"result"] = dict()
1012                 groups = re.search(self.REGEX_BMRR, test.message)
1013                 if groups is not None:
1014                     items_str = groups.group(1)
1015                     items_float = [float(item.strip()) for item
1016                                    in items_str.split(",")]
1017                     # Use whole list in CSIT-1180.
1018                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1019                     test_result[u"result"][u"receive-rate"] = stats.avg
1020                 else:
1021                     groups = re.search(self.REGEX_MRR, test.message)
1022                     test_result[u"result"][u"receive-rate"] = \
1023                         float(groups.group(3)) / float(groups.group(1))
1024             elif u"RECONF" in tags:
1025                 test_result[u"type"] = u"RECONF"
1026                 test_result[u"result"] = None
1027                 try:
1028                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1029                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1030                     test_result[u"result"] = {
1031                         u"loss": int(grps_loss.group(1)),
1032                         u"time": float(grps_time.group(1))
1033                     }
1034                 except (AttributeError, IndexError, ValueError, TypeError):
1035                     test_result[u"status"] = u"FAIL"
1036             elif u"DEVICETEST" in tags:
1037                 test_result[u"type"] = u"DEVICETEST"
1038             else:
1039                 test_result[u"status"] = u"FAIL"
1040                 self._data[u"tests"][self._test_id] = test_result
1041                 return
1042
1043         self._data[u"tests"][self._test_id] = test_result
1044
1045     def end_test(self, test):
1046         """Called when test ends.
1047
1048         :param test: Test to process.
1049         :type test: Test
1050         :returns: Nothing.
1051         """
1052
1053     def visit_keyword(self, keyword):
1054         """Implements traversing through the keyword and its child keywords.
1055
1056         :param keyword: Keyword to process.
1057         :type keyword: Keyword
1058         :returns: Nothing.
1059         """
1060         if self.start_keyword(keyword) is not False:
1061             self.end_keyword(keyword)
1062
1063     def start_keyword(self, keyword):
1064         """Called when keyword starts. Default implementation does nothing.
1065
1066         :param keyword: Keyword to process.
1067         :type keyword: Keyword
1068         :returns: Nothing.
1069         """
1070         try:
1071             if keyword.type == u"setup":
1072                 self.visit_setup_kw(keyword)
1073             elif keyword.type == u"teardown":
1074                 self.visit_teardown_kw(keyword)
1075             else:
1076                 self.visit_test_kw(keyword)
1077         except AttributeError:
1078             pass
1079
1080     def end_keyword(self, keyword):
1081         """Called when keyword ends. Default implementation does nothing.
1082
1083         :param keyword: Keyword to process.
1084         :type keyword: Keyword
1085         :returns: Nothing.
1086         """
1087
1088     def visit_test_kw(self, test_kw):
1089         """Implements traversing through the test keyword and its child
1090         keywords.
1091
1092         :param test_kw: Keyword to process.
1093         :type test_kw: Keyword
1094         :returns: Nothing.
1095         """
1096         for keyword in test_kw.keywords:
1097             if self.start_test_kw(keyword) is not False:
1098                 self.visit_test_kw(keyword)
1099                 self.end_test_kw(keyword)
1100
1101     def start_test_kw(self, test_kw):
1102         """Called when test keyword starts. Default implementation does
1103         nothing.
1104
1105         :param test_kw: Keyword to process.
1106         :type test_kw: Keyword
1107         :returns: Nothing.
1108         """
1109         if test_kw.name.count(u"Show Runtime On All Duts") or \
1110                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
1111             self._msg_type = u"test-show-runtime"
1112             self._sh_run_counter += 1
1113         elif test_kw.name.count(u"Install Dpdk Test") and not self._version:
1114             self._msg_type = u"dpdk-version"
1115         else:
1116             return
1117         test_kw.messages.visit(self)
1118
1119     def end_test_kw(self, test_kw):
1120         """Called when keyword ends. Default implementation does nothing.
1121
1122         :param test_kw: Keyword to process.
1123         :type test_kw: Keyword
1124         :returns: Nothing.
1125         """
1126
1127     def visit_setup_kw(self, setup_kw):
1128         """Implements traversing through the teardown keyword and its child
1129         keywords.
1130
1131         :param setup_kw: Keyword to process.
1132         :type setup_kw: Keyword
1133         :returns: Nothing.
1134         """
1135         for keyword in setup_kw.keywords:
1136             if self.start_setup_kw(keyword) is not False:
1137                 self.visit_setup_kw(keyword)
1138                 self.end_setup_kw(keyword)
1139
1140     def start_setup_kw(self, setup_kw):
1141         """Called when teardown keyword starts. Default implementation does
1142         nothing.
1143
1144         :param setup_kw: Keyword to process.
1145         :type setup_kw: Keyword
1146         :returns: Nothing.
1147         """
1148         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1149                 and not self._version:
1150             self._msg_type = u"vpp-version"
1151         elif setup_kw.name.count(u"Set Global Variable") \
1152                 and not self._timestamp:
1153             self._msg_type = u"timestamp"
1154         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1155             self._msg_type = u"testbed"
1156         else:
1157             return
1158         setup_kw.messages.visit(self)
1159
1160     def end_setup_kw(self, setup_kw):
1161         """Called when keyword ends. Default implementation does nothing.
1162
1163         :param setup_kw: Keyword to process.
1164         :type setup_kw: Keyword
1165         :returns: Nothing.
1166         """
1167
1168     def visit_teardown_kw(self, teardown_kw):
1169         """Implements traversing through the teardown keyword and its child
1170         keywords.
1171
1172         :param teardown_kw: Keyword to process.
1173         :type teardown_kw: Keyword
1174         :returns: Nothing.
1175         """
1176         for keyword in teardown_kw.keywords:
1177             if self.start_teardown_kw(keyword) is not False:
1178                 self.visit_teardown_kw(keyword)
1179                 self.end_teardown_kw(keyword)
1180
1181     def start_teardown_kw(self, teardown_kw):
1182         """Called when teardown keyword starts
1183
1184         :param teardown_kw: Keyword to process.
1185         :type teardown_kw: Keyword
1186         :returns: Nothing.
1187         """
1188
1189         if teardown_kw.name.count(u"Show Vat History On All Duts"):
1190             # TODO: Remove when not needed:
1191             self._conf_history_lookup_nr = 0
1192             self._msg_type = u"teardown-vat-history"
1193             teardown_kw.messages.visit(self)
1194         elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1195             self._conf_history_lookup_nr = 0
1196             self._msg_type = u"teardown-papi-history"
1197             teardown_kw.messages.visit(self)
1198
1199     def end_teardown_kw(self, teardown_kw):
1200         """Called when keyword ends. Default implementation does nothing.
1201
1202         :param teardown_kw: Keyword to process.
1203         :type teardown_kw: Keyword
1204         :returns: Nothing.
1205         """
1206
1207     def visit_message(self, msg):
1208         """Implements visiting the message.
1209
1210         :param msg: Message to process.
1211         :type msg: Message
1212         :returns: Nothing.
1213         """
1214         if self.start_message(msg) is not False:
1215             self.end_message(msg)
1216
1217     def start_message(self, msg):
1218         """Called when message starts. Get required information from messages:
1219         - VPP version.
1220
1221         :param msg: Message to process.
1222         :type msg: Message
1223         :returns: Nothing.
1224         """
1225
1226         if self._msg_type:
1227             self.parse_msg[self._msg_type](msg)
1228
1229     def end_message(self, msg):
1230         """Called when message ends. Default implementation does nothing.
1231
1232         :param msg: Message to process.
1233         :type msg: Message
1234         :returns: Nothing.
1235         """
1236
1237
1238 class InputData:
1239     """Input data
1240
1241     The data is extracted from output.xml files generated by Jenkins jobs and
1242     stored in pandas' DataFrames.
1243
1244     The data structure:
1245     - job name
1246       - build number
1247         - metadata
1248           (as described in ExecutionChecker documentation)
1249         - suites
1250           (as described in ExecutionChecker documentation)
1251         - tests
1252           (as described in ExecutionChecker documentation)
1253     """
1254
1255     def __init__(self, spec):
1256         """Initialization.
1257
1258         :param spec: Specification.
1259         :type spec: Specification
1260         """
1261
1262         # Specification:
1263         self._cfg = spec
1264
1265         # Data store:
1266         self._input_data = pd.Series()
1267
1268     @property
1269     def data(self):
1270         """Getter - Input data.
1271
1272         :returns: Input data
1273         :rtype: pandas.Series
1274         """
1275         return self._input_data
1276
1277     def metadata(self, job, build):
1278         """Getter - metadata
1279
1280         :param job: Job which metadata we want.
1281         :param build: Build which metadata we want.
1282         :type job: str
1283         :type build: str
1284         :returns: Metadata
1285         :rtype: pandas.Series
1286         """
1287
1288         return self.data[job][build][u"metadata"]
1289
1290     def suites(self, job, build):
1291         """Getter - suites
1292
1293         :param job: Job which suites we want.
1294         :param build: Build which suites we want.
1295         :type job: str
1296         :type build: str
1297         :returns: Suites.
1298         :rtype: pandas.Series
1299         """
1300
1301         return self.data[job][str(build)][u"suites"]
1302
1303     def tests(self, job, build):
1304         """Getter - tests
1305
1306         :param job: Job which tests we want.
1307         :param build: Build which tests we want.
1308         :type job: str
1309         :type build: str
1310         :returns: Tests.
1311         :rtype: pandas.Series
1312         """
1313
1314         return self.data[job][build][u"tests"]
1315
1316     def _parse_tests(self, job, build, log):
1317         """Process data from robot output.xml file and return JSON structured
1318         data.
1319
1320         :param job: The name of job which build output data will be processed.
1321         :param build: The build which output data will be processed.
1322         :param log: List of log messages.
1323         :type job: str
1324         :type build: dict
1325         :type log: list of tuples (severity, msg)
1326         :returns: JSON data structure.
1327         :rtype: dict
1328         """
1329
1330         metadata = {
1331             u"job": job,
1332             u"build": build
1333         }
1334
1335         with open(build[u"file-name"], u'r') as data_file:
1336             try:
1337                 result = ExecutionResult(data_file)
1338             except errors.DataError as err:
1339                 log.append(
1340                     (u"ERROR", f"Error occurred while parsing output.xml: "
1341                                f"{repr(err)}")
1342                 )
1343                 return None
1344         checker = ExecutionChecker(metadata, self._cfg.mapping,
1345                                    self._cfg.ignore)
1346         result.visit(checker)
1347
1348         return checker.data
1349
1350     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1351         """Download and parse the input data file.
1352
1353         :param pid: PID of the process executing this method.
1354         :param job: Name of the Jenkins job which generated the processed input
1355             file.
1356         :param build: Information about the Jenkins build which generated the
1357             processed input file.
1358         :param repeat: Repeat the download specified number of times if not
1359             successful.
1360         :type pid: int
1361         :type job: str
1362         :type build: dict
1363         :type repeat: int
1364         """
1365
1366         logs = list()
1367
1368         logs.append(
1369             (u"INFO", f"  Processing the job/build: {job}: {build[u'build']}")
1370         )
1371
1372         state = u"failed"
1373         success = False
1374         data = None
1375         do_repeat = repeat
1376         while do_repeat:
1377             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1378                                                    logs)
1379             if success:
1380                 break
1381             do_repeat -= 1
1382         if not success:
1383             logs.append(
1384                 (u"ERROR",
1385                  f"It is not possible to download the input data file from the "
1386                  f"job {job}, build {build[u'build']}, or it is damaged. "
1387                  f"Skipped.")
1388             )
1389         if success:
1390             logs.append(
1391                 (u"INFO",
1392                  f"    Processing data from the build {build[u'build']} ...")
1393             )
1394             data = self._parse_tests(job, build, logs)
1395             if data is None:
1396                 logs.append(
1397                     (u"ERROR",
1398                      f"Input data file from the job {job}, build "
1399                      f"{build[u'build']} is damaged. Skipped.")
1400                 )
1401             else:
1402                 state = u"processed"
1403
1404             try:
1405                 remove(build[u"file-name"])
1406             except OSError as err:
1407                 logs.append(
1408                     ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
1409                               f"{repr(err)}")
1410                 )
1411
1412         # If the time-period is defined in the specification file, remove all
1413         # files which are outside the time period.
1414         timeperiod = self._cfg.input.get(u"time-period", None)
1415         if timeperiod and data:
1416             now = dt.utcnow()
1417             timeperiod = timedelta(int(timeperiod))
1418             metadata = data.get(u"metadata", None)
1419             if metadata:
1420                 generated = metadata.get(u"generated", None)
1421                 if generated:
1422                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1423                     if (now - generated) > timeperiod:
1424                         # Remove the data and the file:
1425                         state = u"removed"
1426                         data = None
1427                         logs.append(
1428                             (u"INFO",
1429                              f"    The build {job}/{build[u'build']} is "
1430                              f"outdated, will be removed.")
1431                         )
1432         logs.append((u"INFO", u"  Done."))
1433
1434         for level, line in logs:
1435             if level == u"INFO":
1436                 logging.info(line)
1437             elif level == u"ERROR":
1438                 logging.error(line)
1439             elif level == u"DEBUG":
1440                 logging.debug(line)
1441             elif level == u"CRITICAL":
1442                 logging.critical(line)
1443             elif level == u"WARNING":
1444                 logging.warning(line)
1445
1446         return {u"data": data, u"state": state, u"job": job, u"build": build}
1447
1448     def download_and_parse_data(self, repeat=1):
1449         """Download the input data files, parse input data from input files and
1450         store in pandas' Series.
1451
1452         :param repeat: Repeat the download specified number of times if not
1453             successful.
1454         :type repeat: int
1455         """
1456
1457         logging.info(u"Downloading and parsing input files ...")
1458
1459         for job, builds in self._cfg.builds.items():
1460             for build in builds:
1461
1462                 result = self._download_and_parse_build(job, build, repeat)
1463                 build_nr = result[u"build"][u"build"]
1464
1465                 if result[u"data"]:
1466                     data = result[u"data"]
1467                     build_data = pd.Series({
1468                         u"metadata": pd.Series(
1469                             list(data[u"metadata"].values()),
1470                             index=list(data[u"metadata"].keys())
1471                         ),
1472                         u"suites": pd.Series(
1473                             list(data[u"suites"].values()),
1474                             index=list(data[u"suites"].keys())
1475                         ),
1476                         u"tests": pd.Series(
1477                             list(data[u"tests"].values()),
1478                             index=list(data[u"tests"].keys())
1479                         )
1480                     })
1481
1482                     if self._input_data.get(job, None) is None:
1483                         self._input_data[job] = pd.Series()
1484                     self._input_data[job][str(build_nr)] = build_data
1485
1486                     self._cfg.set_input_file_name(
1487                         job, build_nr, result[u"build"][u"file-name"])
1488
1489                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1490
1491                 mem_alloc = \
1492                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1493                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1494
1495         logging.info(u"Done.")
1496
1497     @staticmethod
1498     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1499         """Return the index of character in the string which is the end of tag.
1500
1501         :param tag_filter: The string where the end of tag is being searched.
1502         :param start: The index where the searching is stated.
1503         :param closer: The character which is the tag closer.
1504         :type tag_filter: str
1505         :type start: int
1506         :type closer: str
1507         :returns: The index of the tag closer.
1508         :rtype: int
1509         """
1510
1511         try:
1512             idx_opener = tag_filter.index(closer, start)
1513             return tag_filter.index(closer, idx_opener + 1)
1514         except ValueError:
1515             return None
1516
1517     @staticmethod
1518     def _condition(tag_filter):
1519         """Create a conditional statement from the given tag filter.
1520
1521         :param tag_filter: Filter based on tags from the element specification.
1522         :type tag_filter: str
1523         :returns: Conditional statement which can be evaluated.
1524         :rtype: str
1525         """
1526
1527         index = 0
1528         while True:
1529             index = InputData._end_of_tag(tag_filter, index)
1530             if index is None:
1531                 return tag_filter
1532             index += 1
1533             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1534
1535     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1536                     continue_on_error=False):
1537         """Filter required data from the given jobs and builds.
1538
1539         The output data structure is:
1540
1541         - job 1
1542           - build 1
1543             - test (or suite) 1 ID:
1544               - param 1
1545               - param 2
1546               ...
1547               - param n
1548             ...
1549             - test (or suite) n ID:
1550             ...
1551           ...
1552           - build n
1553         ...
1554         - job n
1555
1556         :param element: Element which will use the filtered data.
1557         :param params: Parameters which will be included in the output. If None,
1558             all parameters are included.
1559         :param data: If not None, this data is used instead of data specified
1560             in the element.
1561         :param data_set: The set of data to be filtered: tests, suites,
1562             metadata.
1563         :param continue_on_error: Continue if there is error while reading the
1564             data. The Item will be empty then
1565         :type element: pandas.Series
1566         :type params: list
1567         :type data: dict
1568         :type data_set: str
1569         :type continue_on_error: bool
1570         :returns: Filtered data.
1571         :rtype pandas.Series
1572         """
1573
1574         try:
1575             if data_set == "suites":
1576                 cond = u"True"
1577             elif element[u"filter"] in (u"all", u"template"):
1578                 cond = u"True"
1579             else:
1580                 cond = InputData._condition(element[u"filter"])
1581             logging.debug(f"   Filter: {cond}")
1582         except KeyError:
1583             logging.error(u"  No filter defined.")
1584             return None
1585
1586         if params is None:
1587             params = element.get(u"parameters", None)
1588             if params:
1589                 params.append(u"type")
1590
1591         data_to_filter = data if data else element[u"data"]
1592         data = pd.Series()
1593         try:
1594             for job, builds in data_to_filter.items():
1595                 data[job] = pd.Series()
1596                 for build in builds:
1597                     data[job][str(build)] = pd.Series()
1598                     try:
1599                         data_dict = dict(
1600                             self.data[job][str(build)][data_set].items())
1601                     except KeyError:
1602                         if continue_on_error:
1603                             continue
1604                         return None
1605
1606                     for test_id, test_data in data_dict.items():
1607                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1608                             data[job][str(build)][test_id] = pd.Series()
1609                             if params is None:
1610                                 for param, val in test_data.items():
1611                                     data[job][str(build)][test_id][param] = val
1612                             else:
1613                                 for param in params:
1614                                     try:
1615                                         data[job][str(build)][test_id][param] =\
1616                                             test_data[param]
1617                                     except KeyError:
1618                                         data[job][str(build)][test_id][param] =\
1619                                             u"No Data"
1620             return data
1621
1622         except (KeyError, IndexError, ValueError) as err:
1623             logging.error(
1624                 f"Missing mandatory parameter in the element specification: "
1625                 f"{repr(err)}"
1626             )
1627             return None
1628         except AttributeError as err:
1629             logging.error(repr(err))
1630             return None
1631         except SyntaxError as err:
1632             logging.error(
1633                 f"The filter {cond} is not correct. Check if all tags are "
1634                 f"enclosed by apostrophes.\n{repr(err)}"
1635             )
1636             return None
1637
1638     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1639                              continue_on_error=False):
1640         """Filter required data from the given jobs and builds.
1641
1642         The output data structure is:
1643
1644         - job 1
1645           - build 1
1646             - test (or suite) 1 ID:
1647               - param 1
1648               - param 2
1649               ...
1650               - param n
1651             ...
1652             - test (or suite) n ID:
1653             ...
1654           ...
1655           - build n
1656         ...
1657         - job n
1658
1659         :param element: Element which will use the filtered data.
1660         :param params: Parameters which will be included in the output. If None,
1661         all parameters are included.
1662         :param data_set: The set of data to be filtered: tests, suites,
1663         metadata.
1664         :param continue_on_error: Continue if there is error while reading the
1665         data. The Item will be empty then
1666         :type element: pandas.Series
1667         :type params: list
1668         :type data_set: str
1669         :type continue_on_error: bool
1670         :returns: Filtered data.
1671         :rtype pandas.Series
1672         """
1673
1674         include = element.get(u"include", None)
1675         if not include:
1676             logging.warning(u"No tests to include, skipping the element.")
1677             return None
1678
1679         if params is None:
1680             params = element.get(u"parameters", None)
1681             if params:
1682                 params.append(u"type")
1683
1684         data = pd.Series()
1685         try:
1686             for job, builds in element[u"data"].items():
1687                 data[job] = pd.Series()
1688                 for build in builds:
1689                     data[job][str(build)] = pd.Series()
1690                     for test in include:
1691                         try:
1692                             reg_ex = re.compile(str(test).lower())
1693                             for test_id in self.data[job][
1694                                     str(build)][data_set].keys():
1695                                 if re.match(reg_ex, str(test_id).lower()):
1696                                     test_data = self.data[job][
1697                                         str(build)][data_set][test_id]
1698                                     data[job][str(build)][test_id] = pd.Series()
1699                                     if params is None:
1700                                         for param, val in test_data.items():
1701                                             data[job][str(build)][test_id]\
1702                                                 [param] = val
1703                                     else:
1704                                         for param in params:
1705                                             try:
1706                                                 data[job][str(build)][
1707                                                     test_id][param] = \
1708                                                     test_data[param]
1709                                             except KeyError:
1710                                                 data[job][str(build)][
1711                                                     test_id][param] = u"No Data"
1712                         except KeyError as err:
1713                             logging.error(repr(err))
1714                             if continue_on_error:
1715                                 continue
1716                             return None
1717             return data
1718
1719         except (KeyError, IndexError, ValueError) as err:
1720             logging.error(
1721                 f"Missing mandatory parameter in the element "
1722                 f"specification: {repr(err)}"
1723             )
1724             return None
1725         except AttributeError as err:
1726             logging.error(repr(err))
1727             return None
1728
1729     @staticmethod
1730     def merge_data(data):
1731         """Merge data from more jobs and builds to a simple data structure.
1732
1733         The output data structure is:
1734
1735         - test (suite) 1 ID:
1736           - param 1
1737           - param 2
1738           ...
1739           - param n
1740         ...
1741         - test (suite) n ID:
1742         ...
1743
1744         :param data: Data to merge.
1745         :type data: pandas.Series
1746         :returns: Merged data.
1747         :rtype: pandas.Series
1748         """
1749
1750         logging.info(u"    Merging data ...")
1751
1752         merged_data = pd.Series()
1753         for builds in data.values:
1754             for item in builds.values:
1755                 for item_id, item_data in item.items():
1756                     merged_data[item_id] = item_data
1757
1758         return merged_data
1759
1760     def print_all_oper_data(self):
1761         """Print all operational data to console.
1762         """
1763
1764         tbl_hdr = (
1765             u"Name",
1766             u"Nr of Vectors",
1767             u"Nr of Packets",
1768             u"Suspends",
1769             u"Cycles per Packet",
1770             u"Average Vector Size"
1771         )
1772
1773         for job in self._input_data.values:
1774             for build in job.values:
1775                 for test_id, test_data in build[u"tests"].items():
1776                     print(f"{test_id}")
1777                     if test_data.get(u"show-run", None) is None:
1778                         continue
1779                     for dut_name, data in test_data[u"show-run"].items():
1780                         if data.get(u"threads", None) is None:
1781                             continue
1782                         print(f"Host IP: {data.get(u'host', '')}, "
1783                               f"Socket: {data.get(u'socket', '')}")
1784                         for thread_nr, thread in data[u"threads"].items():
1785                             txt_table = prettytable.PrettyTable(tbl_hdr)
1786                             avg = 0.0
1787                             for row in thread:
1788                                 txt_table.add_row(row)
1789                                 avg += row[-1]
1790                             if len(thread) == 0:
1791                                 avg = u""
1792                             else:
1793                                 avg = f", Average Vector Size per Node: " \
1794                                       f"{(avg / len(thread)):.2f}"
1795                             th_name = u"main" if thread_nr == 0 \
1796                                 else f"worker_{thread_nr}"
1797                             print(f"{dut_name}, {th_name}{avg}")
1798                             txt_table.float_format = u".2"
1799                             txt_table.align = u"r"
1800                             txt_table.align[u"Name"] = u"l"
1801                             print(f"{txt_table.get_string()}\n")