resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove
  29 from datetime import datetime as dt
  30 from datetime import timedelta
  31 from json import loads
  32
  33 import hdrh.histogram
  34 import hdrh.codec
  35 import prettytable
  36 import pandas as pd
  37
  38 from robot.api import ExecutionResult, ResultVisitor
  39 from robot import errors
  40
  41 from resources.libraries.python import jumpavg
  42 from input_data_files import download_and_unzip_data_file
  43
  44
  45 # Separator used in file names
  46 SEPARATOR = u"__"
  47
  48
  49 class ExecutionChecker(ResultVisitor):
  50     """Class to traverse through the test suite structure.
  51
  52     The functionality implemented in this class generates a json structure:
  53
  54     Performance tests:
  55
  56     {
  57         "metadata": {
  58             "generated": "Timestamp",
  59             "version": "SUT version",
  60             "job": "Jenkins job name",
  61             "build": "Information about the build"
  62         },
  63         "suites": {
  64             "Suite long name 1": {
  65                 "name": Suite name,
  66                 "doc": "Suite 1 documentation",
  67                 "parent": "Suite 1 parent",
  68                 "level": "Level of the suite in the suite hierarchy"
  69             }
  70             "Suite long name N": {
  71                 "name": Suite name,
  72                 "doc": "Suite N documentation",
  73                 "parent": "Suite 2 parent",
  74                 "level": "Level of the suite in the suite hierarchy"
  75             }
  76         }
  77         "tests": {
  78             # NDRPDR tests:
  79             "ID": {
  80                 "name": "Test name",
  81                 "parent": "Name of the parent of the test",
  82                 "doc": "Test documentation",
  83                 "msg": "Test message",
  84                 "conf-history": "DUT1 and DUT2 VAT History",
  85                 "show-run": "Show Run",
  86                 "tags": ["tag 1", "tag 2", "tag n"],
  87                 "type": "NDRPDR",
  88                 "status": "PASS" | "FAIL",
  89                 "throughput": {
  90                     "NDR": {
  91                         "LOWER": float,
  92                         "UPPER": float
  93                     },
  94                     "PDR": {
  95                         "LOWER": float,
  96                         "UPPER": float
  97                     }
  98                 },
  99                 "latency": {
 100                     "NDR": {
 101                         "direction1": {
 102                             "min": float,
 103                             "avg": float,
 104                             "max": float,
 105                             "hdrh": str
 106                         },
 107                         "direction2": {
 108                             "min": float,
 109                             "avg": float,
 110                             "max": float,
 111                             "hdrh": str
 112                         }
 113                     },
 114                     "PDR": {
 115                         "direction1": {
 116                             "min": float,
 117                             "avg": float,
 118                             "max": float,
 119                             "hdrh": str
 120                         },
 121                         "direction2": {
 122                             "min": float,
 123                             "avg": float,
 124                             "max": float,
 125                             "hdrh": str
 126                         }
 127                     }
 128                 }
 129             }
 130
 131             # TCP tests:
 132             "ID": {
 133                 "name": "Test name",
 134                 "parent": "Name of the parent of the test",
 135                 "doc": "Test documentation",
 136                 "msg": "Test message",
 137                 "tags": ["tag 1", "tag 2", "tag n"],
 138                 "type": "TCP",
 139                 "status": "PASS" | "FAIL",
 140                 "result": int
 141             }
 142
 143             # MRR, BMRR tests:
 144             "ID": {
 145                 "name": "Test name",
 146                 "parent": "Name of the parent of the test",
 147                 "doc": "Test documentation",
 148                 "msg": "Test message",
 149                 "tags": ["tag 1", "tag 2", "tag n"],
 150                 "type": "MRR" | "BMRR",
 151                 "status": "PASS" | "FAIL",
 152                 "result": {
 153                     "receive-rate": float,
 154                     # Average of a list, computed using AvgStdevStats.
 155                     # In CSIT-1180, replace with List[float].
 156                 }
 157             }
 158
 159             "ID" {
 160                 # next test
 161             }
 162         }
 163     }
 164
 165
 166     Functional tests:
 167
 168     {
 169         "metadata": {  # Optional
 170             "version": "VPP version",
 171             "job": "Jenkins job name",
 172             "build": "Information about the build"
 173         },
 174         "suites": {
 175             "Suite name 1": {
 176                 "doc": "Suite 1 documentation",
 177                 "parent": "Suite 1 parent",
 178                 "level": "Level of the suite in the suite hierarchy"
 179             }
 180             "Suite name N": {
 181                 "doc": "Suite N documentation",
 182                 "parent": "Suite 2 parent",
 183                 "level": "Level of the suite in the suite hierarchy"
 184             }
 185         }
 186         "tests": {
 187             "ID": {
 188                 "name": "Test name",
 189                 "parent": "Name of the parent of the test",
 190                 "doc": "Test documentation"
 191                 "msg": "Test message"
 192                 "tags": ["tag 1", "tag 2", "tag n"],
 193                 "conf-history": "DUT1 and DUT2 VAT History"
 194                 "show-run": "Show Run"
 195                 "status": "PASS" | "FAIL"
 196             },
 197             "ID" {
 198                 # next test
 199             }
 200         }
 201     }
 202
 203     .. note:: ID is the lowercase full path to the test.
 204     """
 205
 206     REGEX_PLR_RATE = re.compile(
 207         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 208         r'PLRsearch upper bound::?\s(\d+.\d+)'
 209     )
 210     REGEX_NDRPDR_RATE = re.compile(
 211         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 212         r'NDR_UPPER:\s(\d+.\d+).*\n'
 213         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 214         r'PDR_UPPER:\s(\d+.\d+)'
 215     )
 216     REGEX_PERF_MSG_INFO = re.compile(
 217         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 218         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 219         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 220         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 221         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 222     )
 223     REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
 224
 225     # TODO: Remove when not needed
 226     REGEX_NDRPDR_LAT_BASE = re.compile(
 227         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 228         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 229     )
 230     REGEX_NDRPDR_LAT = re.compile(
 231         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 232         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 233         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 234         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 235         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 236         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 237     )
 238     # TODO: Remove when not needed
 239     REGEX_NDRPDR_LAT_LONG = re.compile(
 240         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 241         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 242         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 243         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 244         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 245         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 246         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 247         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 248         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 249     )
 250     REGEX_VERSION_VPP = re.compile(
 251         r"(return STDOUT Version:\s*|"
 252         r"VPP Version:\s*|VPP version:\s*)(.*)"
 253     )
 254     REGEX_VERSION_DPDK = re.compile(
 255         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 256     )
 257     REGEX_TCP = re.compile(
 258         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 259     )
 260     REGEX_MRR = re.compile(
 261         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 262         r'tx\s(\d*),\srx\s(\d*)'
 263     )
 264     REGEX_BMRR = re.compile(
 265         r'Maximum Receive Rate trial results'
 266         r' in packets per second: \[(.*)\]'
 267     )
 268     REGEX_RECONF_LOSS = re.compile(
 269         r'Packets lost due to reconfig: (\d*)'
 270     )
 271     REGEX_RECONF_TIME = re.compile(
 272         r'Implied time lost: (\d*.[\de-]*)'
 273     )
 274     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 275
 276     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 277
 278     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 279
 280     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 281
 282     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 283
 284     def __init__(self, metadata, mapping, ignore):
 285         """Initialisation.
 286
 287         :param metadata: Key-value pairs to be included in "metadata" part of
 288             JSON structure.
 289         :param mapping: Mapping of the old names of test cases to the new
 290             (actual) one.
 291         :param ignore: List of TCs to be ignored.
 292         :type metadata: dict
 293         :type mapping: dict
 294         :type ignore: list
 295         """
 296
 297         # Type of message to parse out from the test messages
 298         self._msg_type = None
 299
 300         # VPP version
 301         self._version = None
 302
 303         # Timestamp
 304         self._timestamp = None
 305
 306         # Testbed. The testbed is identified by TG node IP address.
 307         self._testbed = None
 308
 309         # Mapping of TCs long names
 310         self._mapping = mapping
 311
 312         # Ignore list
 313         self._ignore = ignore
 314
 315         # Number of PAPI History messages found:
 316         # 0 - no message
 317         # 1 - PAPI History of DUT1
 318         # 2 - PAPI History of DUT2
 319         self._conf_history_lookup_nr = 0
 320
 321         self._sh_run_counter = 0
 322
 323         # Test ID of currently processed test- the lowercase full path to the
 324         # test
 325         self._test_id = None
 326
 327         # The main data structure
 328         self._data = {
 329             u"metadata": OrderedDict(),
 330             u"suites": OrderedDict(),
 331             u"tests": OrderedDict()
 332         }
 333
 334         # Save the provided metadata
 335         for key, val in metadata.items():
 336             self._data[u"metadata"][key] = val
 337
 338         # Dictionary defining the methods used to parse different types of
 339         # messages
 340         self.parse_msg = {
 341             u"timestamp": self._get_timestamp,
 342             u"vpp-version": self._get_vpp_version,
 343             u"dpdk-version": self._get_dpdk_version,
 344             # TODO: Remove when not needed:
 345             u"teardown-vat-history": self._get_vat_history,
 346             u"teardown-papi-history": self._get_papi_history,
 347             u"test-show-runtime": self._get_show_run,
 348             u"testbed": self._get_testbed
 349         }
 350
 351     @property
 352     def data(self):
 353         """Getter - Data parsed from the XML file.
 354
 355         :returns: Data parsed from the XML file.
 356         :rtype: dict
 357         """
 358         return self._data
 359
 360     def _get_data_from_mrr_test_msg(self, msg):
 361         """Get info from message of MRR performance tests.
 362
 363         :param msg: Message to be processed.
 364         :type msg: str
 365         :returns: Processed message or original message if a problem occurs.
 366         :rtype: str
 367         """
 368
 369         groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
 370         if not groups or groups.lastindex != 1:
 371             return u"Test Failed."
 372
 373         try:
 374             data = groups.group(1).split(u", ")
 375         except (AttributeError, IndexError, ValueError, KeyError):
 376             return u"Test Failed."
 377
 378         out_str = u"["
 379         try:
 380             for item in data:
 381                 out_str += f"{(float(item) / 1e6):.2f}, "
 382             return out_str[:-2] + u"]"
 383         except (AttributeError, IndexError, ValueError, KeyError):
 384             return u"Test Failed."
 385
 386     def _get_data_from_perf_test_msg(self, msg):
 387         """Get info from message of NDRPDR performance tests.
 388
 389         :param msg: Message to be processed.
 390         :type msg: str
 391         :returns: Processed message or original message if a problem occurs.
 392         :rtype: str
 393         """
 394
 395         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 396         if not groups or groups.lastindex != 10:
 397             return u"Test Failed."
 398
 399         try:
 400             data = {
 401                 u"ndr_low": float(groups.group(1)),
 402                 u"ndr_low_b": float(groups.group(2)),
 403                 u"pdr_low": float(groups.group(3)),
 404                 u"pdr_low_b": float(groups.group(4)),
 405                 u"pdr_lat_90_1": groups.group(5),
 406                 u"pdr_lat_90_2": groups.group(6),
 407                 u"pdr_lat_50_1": groups.group(7),
 408                 u"pdr_lat_50_2": groups.group(8),
 409                 u"pdr_lat_10_1": groups.group(9),
 410                 u"pdr_lat_10_2": groups.group(10),
 411             }
 412         except (AttributeError, IndexError, ValueError, KeyError):
 413             return u"Test Failed."
 414
 415         def _process_lat(in_str_1, in_str_2):
 416             """Extract min, avg, max values from latency string.
 417
 418             :param in_str_1: Latency string for one direction produced by robot
 419                 framework.
 420             :param in_str_2: Latency string for second direction produced by
 421                 robot framework.
 422             :type in_str_1: str
 423             :type in_str_2: str
 424             :returns: Processed latency string or None if a problem occurs.
 425             :rtype: tuple
 426             """
 427             in_list_1 = in_str_1.split('/', 3)
 428             in_list_2 = in_str_2.split('/', 3)
 429
 430             if len(in_list_1) != 4 and len(in_list_2) != 4:
 431                 return None
 432
 433             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 434             try:
 435                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 436             except hdrh.codec.HdrLengthException:
 437                 return None
 438
 439             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 440             try:
 441                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 442             except hdrh.codec.HdrLengthException:
 443                 return None
 444
 445             if hdr_lat_1 and hdr_lat_2:
 446                 hdr_lat = (
 447                     hdr_lat_1.get_value_at_percentile(50.0),
 448                     hdr_lat_1.get_value_at_percentile(90.0),
 449                     hdr_lat_1.get_value_at_percentile(99.0),
 450                     hdr_lat_2.get_value_at_percentile(50.0),
 451                     hdr_lat_2.get_value_at_percentile(90.0),
 452                     hdr_lat_2.get_value_at_percentile(99.0)
 453                 )
 454
 455                 if all(hdr_lat):
 456                     return hdr_lat
 457
 458             return None
 459
 460         try:
 461             out_msg = (
 462                 f"1. {(data[u'ndr_low'] / 1e6):5.2f}      "
 463                 f"{data[u'ndr_low_b']:5.2f}"
 464                 f"\n2. {(data[u'pdr_low'] / 1e6):5.2f}      "
 465                 f"{data[u'pdr_low_b']:5.2f}"
 466             )
 467             latency = (
 468                 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
 469                 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
 470                 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
 471             )
 472             if all(latency):
 473                 max_len = len(str(max((max(item) for item in latency))))
 474                 max_len = 4 if max_len < 4 else max_len
 475
 476                 for idx, lat in enumerate(latency):
 477                     if not idx:
 478                         out_msg += u"\n"
 479                     out_msg += (
 480                         f"\n{idx + 3}. "
 481                         f"{lat[0]:{max_len}d} "
 482                         f"{lat[1]:{max_len}d} "
 483                         f"{lat[2]:{max_len}d}      "
 484                         f"{lat[3]:{max_len}d} "
 485                         f"{lat[4]:{max_len}d} "
 486                         f"{lat[5]:{max_len}d} "
 487                     )
 488
 489             return out_msg
 490
 491         except (AttributeError, IndexError, ValueError, KeyError):
 492             return u"Test Failed."
 493
 494     def _get_testbed(self, msg):
 495         """Called when extraction of testbed IP is required.
 496         The testbed is identified by TG node IP address.
 497
 498         :param msg: Message to process.
 499         :type msg: Message
 500         :returns: Nothing.
 501         """
 502
 503         if msg.message.count(u"Setup of TG node") or \
 504                 msg.message.count(u"Setup of node TG host"):
 505             reg_tg_ip = re.compile(
 506                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 507             try:
 508                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 509             except (KeyError, ValueError, IndexError, AttributeError):
 510                 pass
 511             finally:
 512                 self._data[u"metadata"][u"testbed"] = self._testbed
 513                 self._msg_type = None
 514
 515     def _get_vpp_version(self, msg):
 516         """Called when extraction of VPP version is required.
 517
 518         :param msg: Message to process.
 519         :type msg: Message
 520         :returns: Nothing.
 521         """
 522
 523         if msg.message.count(u"return STDOUT Version:") or \
 524             msg.message.count(u"VPP Version:") or \
 525             msg.message.count(u"VPP version:"):
 526             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 527                                 group(2))
 528             self._data[u"metadata"][u"version"] = self._version
 529             self._msg_type = None
 530
 531     def _get_dpdk_version(self, msg):
 532         """Called when extraction of DPDK version is required.
 533
 534         :param msg: Message to process.
 535         :type msg: Message
 536         :returns: Nothing.
 537         """
 538
 539         if msg.message.count(u"DPDK Version:"):
 540             try:
 541                 self._version = str(re.search(
 542                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 543                 self._data[u"metadata"][u"version"] = self._version
 544             except IndexError:
 545                 pass
 546             finally:
 547                 self._msg_type = None
 548
 549     def _get_timestamp(self, msg):
 550         """Called when extraction of timestamp is required.
 551
 552         :param msg: Message to process.
 553         :type msg: Message
 554         :returns: Nothing.
 555         """
 556
 557         self._timestamp = msg.timestamp[:14]
 558         self._data[u"metadata"][u"generated"] = self._timestamp
 559         self._msg_type = None
 560
 561     def _get_vat_history(self, msg):
 562         """Called when extraction of VAT command history is required.
 563
 564         TODO: Remove when not needed.
 565
 566         :param msg: Message to process.
 567         :type msg: Message
 568         :returns: Nothing.
 569         """
 570         if msg.message.count(u"VAT command history:"):
 571             self._conf_history_lookup_nr += 1
 572             if self._conf_history_lookup_nr == 1:
 573                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 574             else:
 575                 self._msg_type = None
 576             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 577                           r"VAT command history:", u"",
 578                           msg.message, count=1).replace(u'\n', u' |br| ').\
 579                 replace(u'"', u"'")
 580
 581             self._data[u"tests"][self._test_id][u"conf-history"] += (
 582                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 583             )
 584
 585     def _get_papi_history(self, msg):
 586         """Called when extraction of PAPI command history is required.
 587
 588         :param msg: Message to process.
 589         :type msg: Message
 590         :returns: Nothing.
 591         """
 592         if msg.message.count(u"PAPI command history:"):
 593             self._conf_history_lookup_nr += 1
 594             if self._conf_history_lookup_nr == 1:
 595                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 596             else:
 597                 self._msg_type = None
 598             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 599                           r"PAPI command history:", u"",
 600                           msg.message, count=1).replace(u'\n', u' |br| ').\
 601                 replace(u'"', u"'")
 602             self._data[u"tests"][self._test_id][u"conf-history"] += (
 603                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 604             )
 605
 606     def _get_show_run(self, msg):
 607         """Called when extraction of VPP operational data (output of CLI command
 608         Show Runtime) is required.
 609
 610         :param msg: Message to process.
 611         :type msg: Message
 612         :returns: Nothing.
 613         """
 614
 615         if not msg.message.count(u"stats runtime"):
 616             return
 617
 618         # Temporary solution
 619         if self._sh_run_counter > 1:
 620             return
 621
 622         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 623             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 624
 625         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 626         if not groups:
 627             return
 628         try:
 629             host = groups.group(1)
 630         except (AttributeError, IndexError):
 631             host = u""
 632         try:
 633             sock = groups.group(2)
 634         except (AttributeError, IndexError):
 635             sock = u""
 636
 637         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 638                         replace(u"'", u'"').replace(u'b"', u'"').
 639                         replace(u'u"', u'"').split(u":", 1)[1])
 640
 641         try:
 642             threads_nr = len(runtime[0][u"clocks"])
 643         except (IndexError, KeyError):
 644             return
 645
 646         dut = u"DUT{nr}".format(
 647             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 648
 649         oper = {
 650             u"host": host,
 651             u"socket": sock,
 652             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 653         }
 654
 655         for item in runtime:
 656             for idx in range(threads_nr):
 657                 if item[u"vectors"][idx] > 0:
 658                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 659                 elif item[u"calls"][idx] > 0:
 660                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 661                 elif item[u"suspends"][idx] > 0:
 662                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 663                 else:
 664                     clocks = 0.0
 665
 666                 if item[u"calls"][idx] > 0:
 667                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 668                 else:
 669                     vectors_call = 0.0
 670
 671                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 672                         int(item[u"suspends"][idx]):
 673                     oper[u"threads"][idx].append([
 674                         item[u"name"],
 675                         item[u"calls"][idx],
 676                         item[u"vectors"][idx],
 677                         item[u"suspends"][idx],
 678                         clocks,
 679                         vectors_call
 680                     ])
 681
 682         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 683
 684     def _get_ndrpdr_throughput(self, msg):
 685         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 686         message.
 687
 688         :param msg: The test message to be parsed.
 689         :type msg: str
 690         :returns: Parsed data as a dict and the status (PASS/FAIL).
 691         :rtype: tuple(dict, str)
 692         """
 693
 694         throughput = {
 695             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 696             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 697         }
 698         status = u"FAIL"
 699         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 700
 701         if groups is not None:
 702             try:
 703                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 704                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 705                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 706                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 707                 status = u"PASS"
 708             except (IndexError, ValueError):
 709                 pass
 710
 711         return throughput, status
 712
 713     def _get_plr_throughput(self, msg):
 714         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 715         message.
 716
 717         :param msg: The test message to be parsed.
 718         :type msg: str
 719         :returns: Parsed data as a dict and the status (PASS/FAIL).
 720         :rtype: tuple(dict, str)
 721         """
 722
 723         throughput = {
 724             u"LOWER": -1.0,
 725             u"UPPER": -1.0
 726         }
 727         status = u"FAIL"
 728         groups = re.search(self.REGEX_PLR_RATE, msg)
 729
 730         if groups is not None:
 731             try:
 732                 throughput[u"LOWER"] = float(groups.group(1))
 733                 throughput[u"UPPER"] = float(groups.group(2))
 734                 status = u"PASS"
 735             except (IndexError, ValueError):
 736                 pass
 737
 738         return throughput, status
 739
 740     def _get_ndrpdr_latency(self, msg):
 741         """Get LATENCY from the test message.
 742
 743         :param msg: The test message to be parsed.
 744         :type msg: str
 745         :returns: Parsed data as a dict and the status (PASS/FAIL).
 746         :rtype: tuple(dict, str)
 747         """
 748         latency_default = {
 749             u"min": -1.0,
 750             u"avg": -1.0,
 751             u"max": -1.0,
 752             u"hdrh": u""
 753         }
 754         latency = {
 755             u"NDR": {
 756                 u"direction1": copy.copy(latency_default),
 757                 u"direction2": copy.copy(latency_default)
 758             },
 759             u"PDR": {
 760                 u"direction1": copy.copy(latency_default),
 761                 u"direction2": copy.copy(latency_default)
 762             },
 763             u"LAT0": {
 764                 u"direction1": copy.copy(latency_default),
 765                 u"direction2": copy.copy(latency_default)
 766             },
 767             u"PDR10": {
 768                 u"direction1": copy.copy(latency_default),
 769                 u"direction2": copy.copy(latency_default)
 770             },
 771             u"PDR50": {
 772                 u"direction1": copy.copy(latency_default),
 773                 u"direction2": copy.copy(latency_default)
 774             },
 775             u"PDR90": {
 776                 u"direction1": copy.copy(latency_default),
 777                 u"direction2": copy.copy(latency_default)
 778             },
 779         }
 780
 781         # TODO: Rewrite when long and base are not needed
 782         groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
 783         if groups is None:
 784             groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 785         if groups is None:
 786             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 787         if groups is None:
 788             return latency, u"FAIL"
 789
 790         def process_latency(in_str):
 791             """Return object with parsed latency values.
 792
 793             TODO: Define class for the return type.
 794
 795             :param in_str: Input string, min/avg/max/hdrh format.
 796             :type in_str: str
 797             :returns: Dict with corresponding keys, except hdrh float values.
 798             :rtype dict:
 799             :throws IndexError: If in_str does not have enough substrings.
 800             :throws ValueError: If a substring does not convert to float.
 801             """
 802             in_list = in_str.split('/', 3)
 803
 804             rval = {
 805                 u"min": float(in_list[0]),
 806                 u"avg": float(in_list[1]),
 807                 u"max": float(in_list[2]),
 808                 u"hdrh": u""
 809             }
 810
 811             if len(in_list) == 4:
 812                 rval[u"hdrh"] = str(in_list[3])
 813
 814             return rval
 815
 816         try:
 817             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 818             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 819             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 820             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 821             if groups.lastindex == 4:
 822                 return latency, u"PASS"
 823         except (IndexError, ValueError):
 824             pass
 825
 826         try:
 827             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 828             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 829             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 830             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 831             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 832             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 833             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 834             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 835             if groups.lastindex == 12:
 836                 return latency, u"PASS"
 837         except (IndexError, ValueError):
 838             pass
 839
 840         # TODO: Remove when not needed
 841         latency[u"NDR10"] = {
 842             u"direction1": copy.copy(latency_default),
 843             u"direction2": copy.copy(latency_default)
 844         }
 845         latency[u"NDR50"] = {
 846             u"direction1": copy.copy(latency_default),
 847             u"direction2": copy.copy(latency_default)
 848         }
 849         latency[u"NDR90"] = {
 850             u"direction1": copy.copy(latency_default),
 851             u"direction2": copy.copy(latency_default)
 852         }
 853         try:
 854             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
 855             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
 856             latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
 857             latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
 858             latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
 859             latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
 860             latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
 861             latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
 862             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
 863             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
 864             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
 865             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
 866             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
 867             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
 868             return latency, u"PASS"
 869         except (IndexError, ValueError):
 870             pass
 871
 872         return latency, u"FAIL"
 873
 874     def visit_suite(self, suite):
 875         """Implements traversing through the suite and its direct children.
 876
 877         :param suite: Suite to process.
 878         :type suite: Suite
 879         :returns: Nothing.
 880         """
 881         if self.start_suite(suite) is not False:
 882             suite.suites.visit(self)
 883             suite.tests.visit(self)
 884             self.end_suite(suite)
 885
 886     def start_suite(self, suite):
 887         """Called when suite starts.
 888
 889         :param suite: Suite to process.
 890         :type suite: Suite
 891         :returns: Nothing.
 892         """
 893
 894         try:
 895             parent_name = suite.parent.name
 896         except AttributeError:
 897             return
 898
 899         doc_str = suite.doc.\
 900             replace(u'"', u"'").\
 901             replace(u'\n', u' ').\
 902             replace(u'\r', u'').\
 903             replace(u'*[', u' |br| *[').\
 904             replace(u"*", u"**").\
 905             replace(u' |br| *[', u'*[', 1)
 906
 907         self._data[u"suites"][suite.longname.lower().
 908                               replace(u'"', u"'").
 909                               replace(u" ", u"_")] = {
 910                                   u"name": suite.name.lower(),
 911                                   u"doc": doc_str,
 912                                   u"parent": parent_name,
 913                                   u"level": len(suite.longname.split(u"."))
 914                               }
 915
 916         suite.keywords.visit(self)
 917
 918     def end_suite(self, suite):
 919         """Called when suite ends.
 920
 921         :param suite: Suite to process.
 922         :type suite: Suite
 923         :returns: Nothing.
 924         """
 925
 926     def visit_test(self, test):
 927         """Implements traversing through the test.
 928
 929         :param test: Test to process.
 930         :type test: Test
 931         :returns: Nothing.
 932         """
 933         if self.start_test(test) is not False:
 934             test.keywords.visit(self)
 935             self.end_test(test)
 936
 937     def start_test(self, test):
 938         """Called when test starts.
 939
 940         :param test: Test to process.
 941         :type test: Test
 942         :returns: Nothing.
 943         """
 944
 945         self._sh_run_counter = 0
 946
 947         longname_orig = test.longname.lower()
 948
 949         # Check the ignore list
 950         if longname_orig in self._ignore:
 951             return
 952
 953         tags = [str(tag) for tag in test.tags]
 954         test_result = dict()
 955
 956         # Change the TC long name and name if defined in the mapping table
 957         longname = self._mapping.get(longname_orig, None)
 958         if longname is not None:
 959             name = longname.split(u'.')[-1]
 960             logging.debug(
 961                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
 962                 f"{name}"
 963             )
 964         else:
 965             longname = longname_orig
 966             name = test.name.lower()
 967
 968         # Remove TC number from the TC long name (backward compatibility):
 969         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
 970         # Remove TC number from the TC name (not needed):
 971         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
 972
 973         test_result[u"parent"] = test.parent.name.lower()
 974         test_result[u"tags"] = tags
 975         test_result["doc"] = test.doc.\
 976             replace(u'"', u"'").\
 977             replace(u'\n', u' ').\
 978             replace(u'\r', u'').\
 979             replace(u'[', u' |br| [').\
 980             replace(u' |br| [', u'[', 1)
 981         test_result[u"type"] = u"FUNC"
 982         test_result[u"status"] = test.status
 983
 984         if test.status == u"PASS":
 985             if u"NDRPDR" in tags:
 986                 test_result[u"msg"] = self._get_data_from_perf_test_msg(
 987                     test.message).replace(u'\n', u' |br| ').\
 988                     replace(u'\r', u'').replace(u'"', u"'")
 989             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
 990                 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
 991                     test.message).replace(u'\n', u' |br| ').\
 992                     replace(u'\r', u'').replace(u'"', u"'")
 993             else:
 994                 test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\
 995                     replace(u'\r', u'').replace(u'"', u"'")
 996         else:
 997             test_result[u"msg"] = u"Test Failed."
 998
 999         if u"PERFTEST" in tags:
1000             # Replace info about cores (e.g. -1c-) with the info about threads
1001             # and cores (e.g. -1t1c-) in the long test case names and in the
1002             # test case names if necessary.
1003             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1004             if not groups:
1005                 tag_count = 0
1006                 tag_tc = str()
1007                 for tag in test_result[u"tags"]:
1008                     groups = re.search(self.REGEX_TC_TAG, tag)
1009                     if groups:
1010                         tag_count += 1
1011                         tag_tc = tag
1012
1013                 if tag_count == 1:
1014                     self._test_id = re.sub(
1015                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1016                         self._test_id, count=1
1017                     )
1018                     test_result[u"name"] = re.sub(
1019                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1020                         test_result["name"], count=1
1021                     )
1022                 else:
1023                     test_result[u"status"] = u"FAIL"
1024                     self._data[u"tests"][self._test_id] = test_result
1025                     logging.debug(
1026                         f"The test {self._test_id} has no or more than one "
1027                         f"multi-threading tags.\n"
1028                         f"Tags: {test_result[u'tags']}"
1029                     )
1030                     return
1031
1032         if test.status == u"PASS":
1033             if u"NDRPDR" in tags:
1034                 test_result[u"type"] = u"NDRPDR"
1035                 test_result[u"throughput"], test_result[u"status"] = \
1036                     self._get_ndrpdr_throughput(test.message)
1037                 test_result[u"latency"], test_result[u"status"] = \
1038                     self._get_ndrpdr_latency(test.message)
1039             elif u"SOAK" in tags:
1040                 test_result[u"type"] = u"SOAK"
1041                 test_result[u"throughput"], test_result[u"status"] = \
1042                     self._get_plr_throughput(test.message)
1043             elif u"TCP" in tags:
1044                 test_result[u"type"] = u"TCP"
1045                 groups = re.search(self.REGEX_TCP, test.message)
1046                 test_result[u"result"] = int(groups.group(2))
1047             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1048                 if u"MRR" in tags:
1049                     test_result[u"type"] = u"MRR"
1050                 else:
1051                     test_result[u"type"] = u"BMRR"
1052
1053                 test_result[u"result"] = dict()
1054                 groups = re.search(self.REGEX_BMRR, test.message)
1055                 if groups is not None:
1056                     items_str = groups.group(1)
1057                     items_float = [float(item.strip()) for item
1058                                    in items_str.split(",")]
1059                     # Use whole list in CSIT-1180.
1060                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1061                     test_result[u"result"][u"receive-rate"] = stats.avg
1062                 else:
1063                     groups = re.search(self.REGEX_MRR, test.message)
1064                     test_result[u"result"][u"receive-rate"] = \
1065                         float(groups.group(3)) / float(groups.group(1))
1066             elif u"RECONF" in tags:
1067                 test_result[u"type"] = u"RECONF"
1068                 test_result[u"result"] = None
1069                 try:
1070                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1071                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1072                     test_result[u"result"] = {
1073                         u"loss": int(grps_loss.group(1)),
1074                         u"time": float(grps_time.group(1))
1075                     }
1076                 except (AttributeError, IndexError, ValueError, TypeError):
1077                     test_result[u"status"] = u"FAIL"
1078             elif u"DEVICETEST" in tags:
1079                 test_result[u"type"] = u"DEVICETEST"
1080             else:
1081                 test_result[u"status"] = u"FAIL"
1082                 self._data[u"tests"][self._test_id] = test_result
1083                 return
1084
1085         self._data[u"tests"][self._test_id] = test_result
1086
1087     def end_test(self, test):
1088         """Called when test ends.
1089
1090         :param test: Test to process.
1091         :type test: Test
1092         :returns: Nothing.
1093         """
1094
1095     def visit_keyword(self, keyword):
1096         """Implements traversing through the keyword and its child keywords.
1097
1098         :param keyword: Keyword to process.
1099         :type keyword: Keyword
1100         :returns: Nothing.
1101         """
1102         if self.start_keyword(keyword) is not False:
1103             self.end_keyword(keyword)
1104
1105     def start_keyword(self, keyword):
1106         """Called when keyword starts. Default implementation does nothing.
1107
1108         :param keyword: Keyword to process.
1109         :type keyword: Keyword
1110         :returns: Nothing.
1111         """
1112         try:
1113             if keyword.type == u"setup":
1114                 self.visit_setup_kw(keyword)
1115             elif keyword.type == u"teardown":
1116                 self.visit_teardown_kw(keyword)
1117             else:
1118                 self.visit_test_kw(keyword)
1119         except AttributeError:
1120             pass
1121
1122     def end_keyword(self, keyword):
1123         """Called when keyword ends. Default implementation does nothing.
1124
1125         :param keyword: Keyword to process.
1126         :type keyword: Keyword
1127         :returns: Nothing.
1128         """
1129
1130     def visit_test_kw(self, test_kw):
1131         """Implements traversing through the test keyword and its child
1132         keywords.
1133
1134         :param test_kw: Keyword to process.
1135         :type test_kw: Keyword
1136         :returns: Nothing.
1137         """
1138         for keyword in test_kw.keywords:
1139             if self.start_test_kw(keyword) is not False:
1140                 self.visit_test_kw(keyword)
1141                 self.end_test_kw(keyword)
1142
1143     def start_test_kw(self, test_kw):
1144         """Called when test keyword starts. Default implementation does
1145         nothing.
1146
1147         :param test_kw: Keyword to process.
1148         :type test_kw: Keyword
1149         :returns: Nothing.
1150         """
1151         if test_kw.name.count(u"Show Runtime On All Duts") or \
1152                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
1153             self._msg_type = u"test-show-runtime"
1154             self._sh_run_counter += 1
1155         elif test_kw.name.count(u"Install Dpdk Test") and not self._version:
1156             self._msg_type = u"dpdk-version"
1157         else:
1158             return
1159         test_kw.messages.visit(self)
1160
1161     def end_test_kw(self, test_kw):
1162         """Called when keyword ends. Default implementation does nothing.
1163
1164         :param test_kw: Keyword to process.
1165         :type test_kw: Keyword
1166         :returns: Nothing.
1167         """
1168
1169     def visit_setup_kw(self, setup_kw):
1170         """Implements traversing through the teardown keyword and its child
1171         keywords.
1172
1173         :param setup_kw: Keyword to process.
1174         :type setup_kw: Keyword
1175         :returns: Nothing.
1176         """
1177         for keyword in setup_kw.keywords:
1178             if self.start_setup_kw(keyword) is not False:
1179                 self.visit_setup_kw(keyword)
1180                 self.end_setup_kw(keyword)
1181
1182     def start_setup_kw(self, setup_kw):
1183         """Called when teardown keyword starts. Default implementation does
1184         nothing.
1185
1186         :param setup_kw: Keyword to process.
1187         :type setup_kw: Keyword
1188         :returns: Nothing.
1189         """
1190         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1191                 and not self._version:
1192             self._msg_type = u"vpp-version"
1193         elif setup_kw.name.count(u"Set Global Variable") \
1194                 and not self._timestamp:
1195             self._msg_type = u"timestamp"
1196         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1197             self._msg_type = u"testbed"
1198         else:
1199             return
1200         setup_kw.messages.visit(self)
1201
1202     def end_setup_kw(self, setup_kw):
1203         """Called when keyword ends. Default implementation does nothing.
1204
1205         :param setup_kw: Keyword to process.
1206         :type setup_kw: Keyword
1207         :returns: Nothing.
1208         """
1209
1210     def visit_teardown_kw(self, teardown_kw):
1211         """Implements traversing through the teardown keyword and its child
1212         keywords.
1213
1214         :param teardown_kw: Keyword to process.
1215         :type teardown_kw: Keyword
1216         :returns: Nothing.
1217         """
1218         for keyword in teardown_kw.keywords:
1219             if self.start_teardown_kw(keyword) is not False:
1220                 self.visit_teardown_kw(keyword)
1221                 self.end_teardown_kw(keyword)
1222
1223     def start_teardown_kw(self, teardown_kw):
1224         """Called when teardown keyword starts
1225
1226         :param teardown_kw: Keyword to process.
1227         :type teardown_kw: Keyword
1228         :returns: Nothing.
1229         """
1230
1231         if teardown_kw.name.count(u"Show Vat History On All Duts"):
1232             # TODO: Remove when not needed:
1233             self._conf_history_lookup_nr = 0
1234             self._msg_type = u"teardown-vat-history"
1235             teardown_kw.messages.visit(self)
1236         elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1237             self._conf_history_lookup_nr = 0
1238             self._msg_type = u"teardown-papi-history"
1239             teardown_kw.messages.visit(self)
1240
1241     def end_teardown_kw(self, teardown_kw):
1242         """Called when keyword ends. Default implementation does nothing.
1243
1244         :param teardown_kw: Keyword to process.
1245         :type teardown_kw: Keyword
1246         :returns: Nothing.
1247         """
1248
1249     def visit_message(self, msg):
1250         """Implements visiting the message.
1251
1252         :param msg: Message to process.
1253         :type msg: Message
1254         :returns: Nothing.
1255         """
1256         if self.start_message(msg) is not False:
1257             self.end_message(msg)
1258
1259     def start_message(self, msg):
1260         """Called when message starts. Get required information from messages:
1261         - VPP version.
1262
1263         :param msg: Message to process.
1264         :type msg: Message
1265         :returns: Nothing.
1266         """
1267
1268         if self._msg_type:
1269             self.parse_msg[self._msg_type](msg)
1270
1271     def end_message(self, msg):
1272         """Called when message ends. Default implementation does nothing.
1273
1274         :param msg: Message to process.
1275         :type msg: Message
1276         :returns: Nothing.
1277         """
1278
1279
1280 class InputData:
1281     """Input data
1282
1283     The data is extracted from output.xml files generated by Jenkins jobs and
1284     stored in pandas' DataFrames.
1285
1286     The data structure:
1287     - job name
1288       - build number
1289         - metadata
1290           (as described in ExecutionChecker documentation)
1291         - suites
1292           (as described in ExecutionChecker documentation)
1293         - tests
1294           (as described in ExecutionChecker documentation)
1295     """
1296
1297     def __init__(self, spec):
1298         """Initialization.
1299
1300         :param spec: Specification.
1301         :type spec: Specification
1302         """
1303
1304         # Specification:
1305         self._cfg = spec
1306
1307         # Data store:
1308         self._input_data = pd.Series()
1309
1310     @property
1311     def data(self):
1312         """Getter - Input data.
1313
1314         :returns: Input data
1315         :rtype: pandas.Series
1316         """
1317         return self._input_data
1318
1319     def metadata(self, job, build):
1320         """Getter - metadata
1321
1322         :param job: Job which metadata we want.
1323         :param build: Build which metadata we want.
1324         :type job: str
1325         :type build: str
1326         :returns: Metadata
1327         :rtype: pandas.Series
1328         """
1329
1330         return self.data[job][build][u"metadata"]
1331
1332     def suites(self, job, build):
1333         """Getter - suites
1334
1335         :param job: Job which suites we want.
1336         :param build: Build which suites we want.
1337         :type job: str
1338         :type build: str
1339         :returns: Suites.
1340         :rtype: pandas.Series
1341         """
1342
1343         return self.data[job][str(build)][u"suites"]
1344
1345     def tests(self, job, build):
1346         """Getter - tests
1347
1348         :param job: Job which tests we want.
1349         :param build: Build which tests we want.
1350         :type job: str
1351         :type build: str
1352         :returns: Tests.
1353         :rtype: pandas.Series
1354         """
1355
1356         return self.data[job][build][u"tests"]
1357
1358     def _parse_tests(self, job, build, log):
1359         """Process data from robot output.xml file and return JSON structured
1360         data.
1361
1362         :param job: The name of job which build output data will be processed.
1363         :param build: The build which output data will be processed.
1364         :param log: List of log messages.
1365         :type job: str
1366         :type build: dict
1367         :type log: list of tuples (severity, msg)
1368         :returns: JSON data structure.
1369         :rtype: dict
1370         """
1371
1372         metadata = {
1373             u"job": job,
1374             u"build": build
1375         }
1376
1377         with open(build[u"file-name"], u'r') as data_file:
1378             try:
1379                 result = ExecutionResult(data_file)
1380             except errors.DataError as err:
1381                 log.append(
1382                     (u"ERROR", f"Error occurred while parsing output.xml: "
1383                                f"{repr(err)}")
1384                 )
1385                 return None
1386         checker = ExecutionChecker(metadata, self._cfg.mapping,
1387                                    self._cfg.ignore)
1388         result.visit(checker)
1389
1390         return checker.data
1391
1392     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1393         """Download and parse the input data file.
1394
1395         :param pid: PID of the process executing this method.
1396         :param job: Name of the Jenkins job which generated the processed input
1397             file.
1398         :param build: Information about the Jenkins build which generated the
1399             processed input file.
1400         :param repeat: Repeat the download specified number of times if not
1401             successful.
1402         :type pid: int
1403         :type job: str
1404         :type build: dict
1405         :type repeat: int
1406         """
1407
1408         logs = list()
1409
1410         logs.append(
1411             (u"INFO", f"  Processing the job/build: {job}: {build[u'build']}")
1412         )
1413
1414         state = u"failed"
1415         success = False
1416         data = None
1417         do_repeat = repeat
1418         while do_repeat:
1419             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1420                                                    logs)
1421             if success:
1422                 break
1423             do_repeat -= 1
1424         if not success:
1425             logs.append(
1426                 (u"ERROR",
1427                  f"It is not possible to download the input data file from the "
1428                  f"job {job}, build {build[u'build']}, or it is damaged. "
1429                  f"Skipped.")
1430             )
1431         if success:
1432             logs.append(
1433                 (u"INFO",
1434                  f"    Processing data from the build {build[u'build']} ...")
1435             )
1436             data = self._parse_tests(job, build, logs)
1437             if data is None:
1438                 logs.append(
1439                     (u"ERROR",
1440                      f"Input data file from the job {job}, build "
1441                      f"{build[u'build']} is damaged. Skipped.")
1442                 )
1443             else:
1444                 state = u"processed"
1445
1446             try:
1447                 remove(build[u"file-name"])
1448             except OSError as err:
1449                 logs.append(
1450                     ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
1451                               f"{repr(err)}")
1452                 )
1453
1454         # If the time-period is defined in the specification file, remove all
1455         # files which are outside the time period.
1456         timeperiod = self._cfg.input.get(u"time-period", None)
1457         if timeperiod and data:
1458             now = dt.utcnow()
1459             timeperiod = timedelta(int(timeperiod))
1460             metadata = data.get(u"metadata", None)
1461             if metadata:
1462                 generated = metadata.get(u"generated", None)
1463                 if generated:
1464                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1465                     if (now - generated) > timeperiod:
1466                         # Remove the data and the file:
1467                         state = u"removed"
1468                         data = None
1469                         logs.append(
1470                             (u"INFO",
1471                              f"    The build {job}/{build[u'build']} is "
1472                              f"outdated, will be removed.")
1473                         )
1474         logs.append((u"INFO", u"  Done."))
1475
1476         for level, line in logs:
1477             if level == u"INFO":
1478                 logging.info(line)
1479             elif level == u"ERROR":
1480                 logging.error(line)
1481             elif level == u"DEBUG":
1482                 logging.debug(line)
1483             elif level == u"CRITICAL":
1484                 logging.critical(line)
1485             elif level == u"WARNING":
1486                 logging.warning(line)
1487
1488         return {u"data": data, u"state": state, u"job": job, u"build": build}
1489
1490     def download_and_parse_data(self, repeat=1):
1491         """Download the input data files, parse input data from input files and
1492         store in pandas' Series.
1493
1494         :param repeat: Repeat the download specified number of times if not
1495             successful.
1496         :type repeat: int
1497         """
1498
1499         logging.info(u"Downloading and parsing input files ...")
1500
1501         for job, builds in self._cfg.builds.items():
1502             for build in builds:
1503
1504                 result = self._download_and_parse_build(job, build, repeat)
1505                 build_nr = result[u"build"][u"build"]
1506
1507                 if result[u"data"]:
1508                     data = result[u"data"]
1509                     build_data = pd.Series({
1510                         u"metadata": pd.Series(
1511                             list(data[u"metadata"].values()),
1512                             index=list(data[u"metadata"].keys())
1513                         ),
1514                         u"suites": pd.Series(
1515                             list(data[u"suites"].values()),
1516                             index=list(data[u"suites"].keys())
1517                         ),
1518                         u"tests": pd.Series(
1519                             list(data[u"tests"].values()),
1520                             index=list(data[u"tests"].keys())
1521                         )
1522                     })
1523
1524                     if self._input_data.get(job, None) is None:
1525                         self._input_data[job] = pd.Series()
1526                     self._input_data[job][str(build_nr)] = build_data
1527
1528                     self._cfg.set_input_file_name(
1529                         job, build_nr, result[u"build"][u"file-name"])
1530
1531                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1532
1533                 mem_alloc = \
1534                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1535                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1536
1537         logging.info(u"Done.")
1538
1539     @staticmethod
1540     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1541         """Return the index of character in the string which is the end of tag.
1542
1543         :param tag_filter: The string where the end of tag is being searched.
1544         :param start: The index where the searching is stated.
1545         :param closer: The character which is the tag closer.
1546         :type tag_filter: str
1547         :type start: int
1548         :type closer: str
1549         :returns: The index of the tag closer.
1550         :rtype: int
1551         """
1552
1553         try:
1554             idx_opener = tag_filter.index(closer, start)
1555             return tag_filter.index(closer, idx_opener + 1)
1556         except ValueError:
1557             return None
1558
1559     @staticmethod
1560     def _condition(tag_filter):
1561         """Create a conditional statement from the given tag filter.
1562
1563         :param tag_filter: Filter based on tags from the element specification.
1564         :type tag_filter: str
1565         :returns: Conditional statement which can be evaluated.
1566         :rtype: str
1567         """
1568
1569         index = 0
1570         while True:
1571             index = InputData._end_of_tag(tag_filter, index)
1572             if index is None:
1573                 return tag_filter
1574             index += 1
1575             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1576
1577     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1578                     continue_on_error=False):
1579         """Filter required data from the given jobs and builds.
1580
1581         The output data structure is:
1582
1583         - job 1
1584           - build 1
1585             - test (or suite) 1 ID:
1586               - param 1
1587               - param 2
1588               ...
1589               - param n
1590             ...
1591             - test (or suite) n ID:
1592             ...
1593           ...
1594           - build n
1595         ...
1596         - job n
1597
1598         :param element: Element which will use the filtered data.
1599         :param params: Parameters which will be included in the output. If None,
1600             all parameters are included.
1601         :param data: If not None, this data is used instead of data specified
1602             in the element.
1603         :param data_set: The set of data to be filtered: tests, suites,
1604             metadata.
1605         :param continue_on_error: Continue if there is error while reading the
1606             data. The Item will be empty then
1607         :type element: pandas.Series
1608         :type params: list
1609         :type data: dict
1610         :type data_set: str
1611         :type continue_on_error: bool
1612         :returns: Filtered data.
1613         :rtype pandas.Series
1614         """
1615
1616         try:
1617             if data_set == "suites":
1618                 cond = u"True"
1619             elif element[u"filter"] in (u"all", u"template"):
1620                 cond = u"True"
1621             else:
1622                 cond = InputData._condition(element[u"filter"])
1623             logging.debug(f"   Filter: {cond}")
1624         except KeyError:
1625             logging.error(u"  No filter defined.")
1626             return None
1627
1628         if params is None:
1629             params = element.get(u"parameters", None)
1630             if params:
1631                 params.append(u"type")
1632
1633         data_to_filter = data if data else element[u"data"]
1634         data = pd.Series()
1635         try:
1636             for job, builds in data_to_filter.items():
1637                 data[job] = pd.Series()
1638                 for build in builds:
1639                     data[job][str(build)] = pd.Series()
1640                     try:
1641                         data_dict = dict(
1642                             self.data[job][str(build)][data_set].items())
1643                     except KeyError:
1644                         if continue_on_error:
1645                             continue
1646                         return None
1647
1648                     for test_id, test_data in data_dict.items():
1649                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1650                             data[job][str(build)][test_id] = pd.Series()
1651                             if params is None:
1652                                 for param, val in test_data.items():
1653                                     data[job][str(build)][test_id][param] = val
1654                             else:
1655                                 for param in params:
1656                                     try:
1657                                         data[job][str(build)][test_id][param] =\
1658                                             test_data[param]
1659                                     except KeyError:
1660                                         data[job][str(build)][test_id][param] =\
1661                                             u"No Data"
1662             return data
1663
1664         except (KeyError, IndexError, ValueError) as err:
1665             logging.error(
1666                 f"Missing mandatory parameter in the element specification: "
1667                 f"{repr(err)}"
1668             )
1669             return None
1670         except AttributeError as err:
1671             logging.error(repr(err))
1672             return None
1673         except SyntaxError as err:
1674             logging.error(
1675                 f"The filter {cond} is not correct. Check if all tags are "
1676                 f"enclosed by apostrophes.\n{repr(err)}"
1677             )
1678             return None
1679
1680     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1681                              continue_on_error=False):
1682         """Filter required data from the given jobs and builds.
1683
1684         The output data structure is:
1685
1686         - job 1
1687           - build 1
1688             - test (or suite) 1 ID:
1689               - param 1
1690               - param 2
1691               ...
1692               - param n
1693             ...
1694             - test (or suite) n ID:
1695             ...
1696           ...
1697           - build n
1698         ...
1699         - job n
1700
1701         :param element: Element which will use the filtered data.
1702         :param params: Parameters which will be included in the output. If None,
1703         all parameters are included.
1704         :param data_set: The set of data to be filtered: tests, suites,
1705         metadata.
1706         :param continue_on_error: Continue if there is error while reading the
1707         data. The Item will be empty then
1708         :type element: pandas.Series
1709         :type params: list
1710         :type data_set: str
1711         :type continue_on_error: bool
1712         :returns: Filtered data.
1713         :rtype pandas.Series
1714         """
1715
1716         include = element.get(u"include", None)
1717         if not include:
1718             logging.warning(u"No tests to include, skipping the element.")
1719             return None
1720
1721         if params is None:
1722             params = element.get(u"parameters", None)
1723             if params:
1724                 params.append(u"type")
1725
1726         data = pd.Series()
1727         try:
1728             for job, builds in element[u"data"].items():
1729                 data[job] = pd.Series()
1730                 for build in builds:
1731                     data[job][str(build)] = pd.Series()
1732                     for test in include:
1733                         try:
1734                             reg_ex = re.compile(str(test).lower())
1735                             for test_id in self.data[job][
1736                                     str(build)][data_set].keys():
1737                                 if re.match(reg_ex, str(test_id).lower()):
1738                                     test_data = self.data[job][
1739                                         str(build)][data_set][test_id]
1740                                     data[job][str(build)][test_id] = pd.Series()
1741                                     if params is None:
1742                                         for param, val in test_data.items():
1743                                             data[job][str(build)][test_id]\
1744                                                 [param] = val
1745                                     else:
1746                                         for param in params:
1747                                             try:
1748                                                 data[job][str(build)][
1749                                                     test_id][param] = \
1750                                                     test_data[param]
1751                                             except KeyError:
1752                                                 data[job][str(build)][
1753                                                     test_id][param] = u"No Data"
1754                         except KeyError as err:
1755                             logging.error(repr(err))
1756                             if continue_on_error:
1757                                 continue
1758                             return None
1759             return data
1760
1761         except (KeyError, IndexError, ValueError) as err:
1762             logging.error(
1763                 f"Missing mandatory parameter in the element "
1764                 f"specification: {repr(err)}"
1765             )
1766             return None
1767         except AttributeError as err:
1768             logging.error(repr(err))
1769             return None
1770
1771     @staticmethod
1772     def merge_data(data):
1773         """Merge data from more jobs and builds to a simple data structure.
1774
1775         The output data structure is:
1776
1777         - test (suite) 1 ID:
1778           - param 1
1779           - param 2
1780           ...
1781           - param n
1782         ...
1783         - test (suite) n ID:
1784         ...
1785
1786         :param data: Data to merge.
1787         :type data: pandas.Series
1788         :returns: Merged data.
1789         :rtype: pandas.Series
1790         """
1791
1792         logging.info(u"    Merging data ...")
1793
1794         merged_data = pd.Series()
1795         for builds in data.values:
1796             for item in builds.values:
1797                 for item_id, item_data in item.items():
1798                     merged_data[item_id] = item_data
1799
1800         return merged_data
1801
1802     def print_all_oper_data(self):
1803         """Print all operational data to console.
1804         """
1805
1806         tbl_hdr = (
1807             u"Name",
1808             u"Nr of Vectors",
1809             u"Nr of Packets",
1810             u"Suspends",
1811             u"Cycles per Packet",
1812             u"Average Vector Size"
1813         )
1814
1815         for job in self._input_data.values:
1816             for build in job.values:
1817                 for test_id, test_data in build[u"tests"].items():
1818                     print(f"{test_id}")
1819                     if test_data.get(u"show-run", None) is None:
1820                         continue
1821                     for dut_name, data in test_data[u"show-run"].items():
1822                         if data.get(u"threads", None) is None:
1823                             continue
1824                         print(f"Host IP: {data.get(u'host', '')}, "
1825                               f"Socket: {data.get(u'socket', '')}")
1826                         for thread_nr, thread in data[u"threads"].items():
1827                             txt_table = prettytable.PrettyTable(tbl_hdr)
1828                             avg = 0.0
1829                             for row in thread:
1830                                 txt_table.add_row(row)
1831                                 avg += row[-1]
1832                             if len(thread) == 0:
1833                                 avg = u""
1834                             else:
1835                                 avg = f", Average Vector Size per Node: " \
1836                                       f"{(avg / len(thread)):.2f}"
1837                             th_name = u"main" if thread_nr == 0 \
1838                                 else f"worker_{thread_nr}"
1839                             print(f"{dut_name}, {th_name}{avg}")
1840                             txt_table.float_format = u".2"
1841                             txt_table.align = u"r"
1842                             txt_table.align[u"Name"] = u"l"
1843                             print(f"{txt_table.get_string()}\n")