resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove
  29 from datetime import datetime as dt
  30 from datetime import timedelta
  31 from json import loads
  32
  33 import hdrh.histogram
  34 import hdrh.codec
  35 import prettytable
  36 import pandas as pd
  37
  38 from robot.api import ExecutionResult, ResultVisitor
  39 from robot import errors
  40
  41 from resources.libraries.python import jumpavg
  42 from input_data_files import download_and_unzip_data_file
  43
  44
  45 # Separator used in file names
  46 SEPARATOR = u"__"
  47
  48
  49 class ExecutionChecker(ResultVisitor):
  50     """Class to traverse through the test suite structure.
  51
  52     The functionality implemented in this class generates a json structure:
  53
  54     Performance tests:
  55
  56     {
  57         "metadata": {
  58             "generated": "Timestamp",
  59             "version": "SUT version",
  60             "job": "Jenkins job name",
  61             "build": "Information about the build"
  62         },
  63         "suites": {
  64             "Suite long name 1": {
  65                 "name": Suite name,
  66                 "doc": "Suite 1 documentation",
  67                 "parent": "Suite 1 parent",
  68                 "level": "Level of the suite in the suite hierarchy"
  69             }
  70             "Suite long name N": {
  71                 "name": Suite name,
  72                 "doc": "Suite N documentation",
  73                 "parent": "Suite 2 parent",
  74                 "level": "Level of the suite in the suite hierarchy"
  75             }
  76         }
  77         "tests": {
  78             # NDRPDR tests:
  79             "ID": {
  80                 "name": "Test name",
  81                 "parent": "Name of the parent of the test",
  82                 "doc": "Test documentation",
  83                 "msg": "Test message",
  84                 "conf-history": "DUT1 and DUT2 VAT History",
  85                 "show-run": "Show Run",
  86                 "tags": ["tag 1", "tag 2", "tag n"],
  87                 "type": "NDRPDR",
  88                 "status": "PASS" | "FAIL",
  89                 "throughput": {
  90                     "NDR": {
  91                         "LOWER": float,
  92                         "UPPER": float
  93                     },
  94                     "PDR": {
  95                         "LOWER": float,
  96                         "UPPER": float
  97                     }
  98                 },
  99                 "latency": {
 100                     "NDR": {
 101                         "direction1": {
 102                             "min": float,
 103                             "avg": float,
 104                             "max": float,
 105                             "hdrh": str
 106                         },
 107                         "direction2": {
 108                             "min": float,
 109                             "avg": float,
 110                             "max": float,
 111                             "hdrh": str
 112                         }
 113                     },
 114                     "PDR": {
 115                         "direction1": {
 116                             "min": float,
 117                             "avg": float,
 118                             "max": float,
 119                             "hdrh": str
 120                         },
 121                         "direction2": {
 122                             "min": float,
 123                             "avg": float,
 124                             "max": float,
 125                             "hdrh": str
 126                         }
 127                     }
 128                 }
 129             }
 130
 131             # TCP tests:
 132             "ID": {
 133                 "name": "Test name",
 134                 "parent": "Name of the parent of the test",
 135                 "doc": "Test documentation",
 136                 "msg": "Test message",
 137                 "tags": ["tag 1", "tag 2", "tag n"],
 138                 "type": "TCP",
 139                 "status": "PASS" | "FAIL",
 140                 "result": int
 141             }
 142
 143             # MRR, BMRR tests:
 144             "ID": {
 145                 "name": "Test name",
 146                 "parent": "Name of the parent of the test",
 147                 "doc": "Test documentation",
 148                 "msg": "Test message",
 149                 "tags": ["tag 1", "tag 2", "tag n"],
 150                 "type": "MRR" | "BMRR",
 151                 "status": "PASS" | "FAIL",
 152                 "result": {
 153                     "receive-rate": float,
 154                     # Average of a list, computed using AvgStdevStats.
 155                     # In CSIT-1180, replace with List[float].
 156                 }
 157             }
 158
 159             "ID" {
 160                 # next test
 161             }
 162         }
 163     }
 164
 165
 166     Functional tests:
 167
 168     {
 169         "metadata": {  # Optional
 170             "version": "VPP version",
 171             "job": "Jenkins job name",
 172             "build": "Information about the build"
 173         },
 174         "suites": {
 175             "Suite name 1": {
 176                 "doc": "Suite 1 documentation",
 177                 "parent": "Suite 1 parent",
 178                 "level": "Level of the suite in the suite hierarchy"
 179             }
 180             "Suite name N": {
 181                 "doc": "Suite N documentation",
 182                 "parent": "Suite 2 parent",
 183                 "level": "Level of the suite in the suite hierarchy"
 184             }
 185         }
 186         "tests": {
 187             "ID": {
 188                 "name": "Test name",
 189                 "parent": "Name of the parent of the test",
 190                 "doc": "Test documentation"
 191                 "msg": "Test message"
 192                 "tags": ["tag 1", "tag 2", "tag n"],
 193                 "conf-history": "DUT1 and DUT2 VAT History"
 194                 "show-run": "Show Run"
 195                 "status": "PASS" | "FAIL"
 196             },
 197             "ID" {
 198                 # next test
 199             }
 200         }
 201     }
 202
 203     .. note:: ID is the lowercase full path to the test.
 204     """
 205
 206     REGEX_PLR_RATE = re.compile(
 207         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 208         r'PLRsearch upper bound::?\s(\d+.\d+)'
 209     )
 210     REGEX_NDRPDR_RATE = re.compile(
 211         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 212         r'NDR_UPPER:\s(\d+.\d+).*\n'
 213         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 214         r'PDR_UPPER:\s(\d+.\d+)'
 215     )
 216     REGEX_PERF_MSG_INFO = re.compile(
 217         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 218         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 219         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 220         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 221         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 222     )
 223     # TODO: Remove when not needed
 224     REGEX_NDRPDR_LAT_BASE = re.compile(
 225         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 226         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 227     )
 228     REGEX_NDRPDR_LAT = re.compile(
 229         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 230         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 231         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 232         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 233         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 234         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 235     )
 236     # TODO: Remove when not needed
 237     REGEX_NDRPDR_LAT_LONG = re.compile(
 238         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 239         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 240         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 241         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 242         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 243         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 244         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 245         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 246         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 247     )
 248     REGEX_VERSION_VPP = re.compile(
 249         r"(return STDOUT Version:\s*|"
 250         r"VPP Version:\s*|VPP version:\s*)(.*)"
 251     )
 252     REGEX_VERSION_DPDK = re.compile(
 253         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 254     )
 255     REGEX_TCP = re.compile(
 256         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 257     )
 258     REGEX_MRR = re.compile(
 259         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 260         r'tx\s(\d*),\srx\s(\d*)'
 261     )
 262     REGEX_BMRR = re.compile(
 263         r'Maximum Receive Rate trial results'
 264         r' in packets per second: \[(.*)\]'
 265     )
 266     REGEX_RECONF_LOSS = re.compile(
 267         r'Packets lost due to reconfig: (\d*)'
 268     )
 269     REGEX_RECONF_TIME = re.compile(
 270         r'Implied time lost: (\d*.[\de-]*)'
 271     )
 272     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 273
 274     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 275
 276     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 277
 278     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 279
 280     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 281
 282     def __init__(self, metadata, mapping, ignore):
 283         """Initialisation.
 284
 285         :param metadata: Key-value pairs to be included in "metadata" part of
 286             JSON structure.
 287         :param mapping: Mapping of the old names of test cases to the new
 288             (actual) one.
 289         :param ignore: List of TCs to be ignored.
 290         :type metadata: dict
 291         :type mapping: dict
 292         :type ignore: list
 293         """
 294
 295         # Type of message to parse out from the test messages
 296         self._msg_type = None
 297
 298         # VPP version
 299         self._version = None
 300
 301         # Timestamp
 302         self._timestamp = None
 303
 304         # Testbed. The testbed is identified by TG node IP address.
 305         self._testbed = None
 306
 307         # Mapping of TCs long names
 308         self._mapping = mapping
 309
 310         # Ignore list
 311         self._ignore = ignore
 312
 313         # Number of PAPI History messages found:
 314         # 0 - no message
 315         # 1 - PAPI History of DUT1
 316         # 2 - PAPI History of DUT2
 317         self._conf_history_lookup_nr = 0
 318
 319         self._sh_run_counter = 0
 320
 321         # Test ID of currently processed test- the lowercase full path to the
 322         # test
 323         self._test_id = None
 324
 325         # The main data structure
 326         self._data = {
 327             u"metadata": OrderedDict(),
 328             u"suites": OrderedDict(),
 329             u"tests": OrderedDict()
 330         }
 331
 332         # Save the provided metadata
 333         for key, val in metadata.items():
 334             self._data[u"metadata"][key] = val
 335
 336         # Dictionary defining the methods used to parse different types of
 337         # messages
 338         self.parse_msg = {
 339             u"timestamp": self._get_timestamp,
 340             u"vpp-version": self._get_vpp_version,
 341             u"dpdk-version": self._get_dpdk_version,
 342             # TODO: Remove when not needed:
 343             u"teardown-vat-history": self._get_vat_history,
 344             u"teardown-papi-history": self._get_papi_history,
 345             u"test-show-runtime": self._get_show_run,
 346             u"testbed": self._get_testbed
 347         }
 348
 349     @property
 350     def data(self):
 351         """Getter - Data parsed from the XML file.
 352
 353         :returns: Data parsed from the XML file.
 354         :rtype: dict
 355         """
 356         return self._data
 357
 358     def _get_data_from_perf_test_msg(self, msg):
 359         """Get
 360             - NDR_LOWER
 361             - LATENCY
 362             - NDR_UPPER
 363             - PDR_LOWER
 364             - LATENCY
 365             - PDR_UPPER
 366         from message of NDRPDR performance tests.
 367
 368         :param msg: Message to be processed.
 369         :type msg: str
 370         :returns: Processed message or original message if a problem occurs.
 371         :rtype: str
 372         """
 373
 374         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 375         if not groups or groups.lastindex != 10:
 376             return msg
 377
 378         try:
 379             data = {
 380                 u"ndr_low": float(groups.group(1)),
 381                 u"ndr_low_b": float(groups.group(2)),
 382                 u"pdr_low": float(groups.group(3)),
 383                 u"pdr_low_b": float(groups.group(4)),
 384                 u"pdr_lat_90_1": groups.group(5),
 385                 u"pdr_lat_90_2": groups.group(6),
 386                 u"pdr_lat_50_1": groups.group(7),
 387                 u"pdr_lat_50_2": groups.group(8),
 388                 u"pdr_lat_10_1": groups.group(9),
 389                 u"pdr_lat_10_2": groups.group(10),
 390             }
 391         except (AttributeError, IndexError, ValueError, KeyError):
 392             return msg
 393
 394         def _process_lat(in_str_1, in_str_2):
 395             """Extract min, avg, max values from latency string.
 396
 397             :param in_str_1: Latency string for one direction produced by robot
 398                 framework.
 399             :param in_str_2: Latency string for second direction produced by
 400                 robot framework.
 401             :type in_str_1: str
 402             :type in_str_2: str
 403             :returns: Processed latency string or empty string if a problem
 404                 occurs.
 405             :rtype: tuple(str, str)
 406             """
 407             in_list_1 = in_str_1.split('/', 3)
 408             in_list_2 = in_str_2.split('/', 3)
 409
 410             if len(in_list_1) != 4 and len(in_list_2) != 4:
 411                 return u""
 412
 413             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 414             try:
 415                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 416             except hdrh.codec.HdrLengthException:
 417                 return u""
 418
 419             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 420             try:
 421                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 422             except hdrh.codec.HdrLengthException:
 423                 return u""
 424
 425             if hdr_lat_1 and hdr_lat_2:
 426                 return (
 427                     f"{hdr_lat_1.get_value_at_percentile(50.0)} "
 428                     f"{hdr_lat_1.get_value_at_percentile(90.0)} "
 429                     f"{hdr_lat_1.get_value_at_percentile(99.0)} , "
 430                     f"{hdr_lat_2.get_value_at_percentile(50.0)} "
 431                     f"{hdr_lat_2.get_value_at_percentile(90.0)} "
 432                     f"{hdr_lat_2.get_value_at_percentile(99.0)}"
 433                 )
 434
 435             return u""
 436
 437         try:
 438             pdr_lat_10 = _process_lat(data[u'pdr_lat_10_1'],
 439                                       data[u'pdr_lat_10_2'])
 440             pdr_lat_50 = _process_lat(data[u'pdr_lat_50_1'],
 441                                       data[u'pdr_lat_50_2'])
 442             pdr_lat_90 = _process_lat(data[u'pdr_lat_90_1'],
 443                                       data[u'pdr_lat_90_2'])
 444             pdr_lat_10 = f"\n3. {pdr_lat_10}" if pdr_lat_10 else u""
 445             pdr_lat_50 = f"\n4. {pdr_lat_50}" if pdr_lat_50 else u""
 446             pdr_lat_90 = f"\n5. {pdr_lat_90}" if pdr_lat_90 else u""
 447
 448             return (
 449                 u" |prein| "
 450                 f"1. {(data[u'ndr_low'] / 1e6):.2f} {data[u'ndr_low_b']:.2f}"
 451                 f"\n2. {(data[u'pdr_low'] / 1e6):.2f} {data[u'pdr_low_b']:.2f}"
 452                 f"{pdr_lat_10}"
 453                 f"{pdr_lat_50}"
 454                 f"{pdr_lat_90}"
 455                 u" |preout| "
 456             )
 457         except (AttributeError, IndexError, ValueError, KeyError):
 458             return msg
 459
 460     def _get_testbed(self, msg):
 461         """Called when extraction of testbed IP is required.
 462         The testbed is identified by TG node IP address.
 463
 464         :param msg: Message to process.
 465         :type msg: Message
 466         :returns: Nothing.
 467         """
 468
 469         if msg.message.count(u"Setup of TG node") or \
 470                 msg.message.count(u"Setup of node TG host"):
 471             reg_tg_ip = re.compile(
 472                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 473             try:
 474                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 475             except (KeyError, ValueError, IndexError, AttributeError):
 476                 pass
 477             finally:
 478                 self._data[u"metadata"][u"testbed"] = self._testbed
 479                 self._msg_type = None
 480
 481     def _get_vpp_version(self, msg):
 482         """Called when extraction of VPP version is required.
 483
 484         :param msg: Message to process.
 485         :type msg: Message
 486         :returns: Nothing.
 487         """
 488
 489         if msg.message.count(u"return STDOUT Version:") or \
 490             msg.message.count(u"VPP Version:") or \
 491             msg.message.count(u"VPP version:"):
 492             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 493                                 group(2))
 494             self._data[u"metadata"][u"version"] = self._version
 495             self._msg_type = None
 496
 497     def _get_dpdk_version(self, msg):
 498         """Called when extraction of DPDK version is required.
 499
 500         :param msg: Message to process.
 501         :type msg: Message
 502         :returns: Nothing.
 503         """
 504
 505         if msg.message.count(u"DPDK Version:"):
 506             try:
 507                 self._version = str(re.search(
 508                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 509                 self._data[u"metadata"][u"version"] = self._version
 510             except IndexError:
 511                 pass
 512             finally:
 513                 self._msg_type = None
 514
 515     def _get_timestamp(self, msg):
 516         """Called when extraction of timestamp is required.
 517
 518         :param msg: Message to process.
 519         :type msg: Message
 520         :returns: Nothing.
 521         """
 522
 523         self._timestamp = msg.timestamp[:14]
 524         self._data[u"metadata"][u"generated"] = self._timestamp
 525         self._msg_type = None
 526
 527     def _get_vat_history(self, msg):
 528         """Called when extraction of VAT command history is required.
 529
 530         TODO: Remove when not needed.
 531
 532         :param msg: Message to process.
 533         :type msg: Message
 534         :returns: Nothing.
 535         """
 536         if msg.message.count(u"VAT command history:"):
 537             self._conf_history_lookup_nr += 1
 538             if self._conf_history_lookup_nr == 1:
 539                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 540             else:
 541                 self._msg_type = None
 542             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 543                           r"VAT command history:", u"",
 544                           msg.message, count=1).replace(u'\n', u' |br| ').\
 545                 replace(u'"', u"'")
 546
 547             self._data[u"tests"][self._test_id][u"conf-history"] += (
 548                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 549             )
 550
 551     def _get_papi_history(self, msg):
 552         """Called when extraction of PAPI command history is required.
 553
 554         :param msg: Message to process.
 555         :type msg: Message
 556         :returns: Nothing.
 557         """
 558         if msg.message.count(u"PAPI command history:"):
 559             self._conf_history_lookup_nr += 1
 560             if self._conf_history_lookup_nr == 1:
 561                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 562             else:
 563                 self._msg_type = None
 564             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 565                           r"PAPI command history:", u"",
 566                           msg.message, count=1).replace(u'\n', u' |br| ').\
 567                 replace(u'"', u"'")
 568             self._data[u"tests"][self._test_id][u"conf-history"] += (
 569                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 570             )
 571
 572     def _get_show_run(self, msg):
 573         """Called when extraction of VPP operational data (output of CLI command
 574         Show Runtime) is required.
 575
 576         :param msg: Message to process.
 577         :type msg: Message
 578         :returns: Nothing.
 579         """
 580
 581         if not msg.message.count(u"stats runtime"):
 582             return
 583
 584         # Temporary solution
 585         if self._sh_run_counter > 1:
 586             return
 587
 588         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 589             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 590
 591         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 592         if not groups:
 593             return
 594         try:
 595             host = groups.group(1)
 596         except (AttributeError, IndexError):
 597             host = u""
 598         try:
 599             sock = groups.group(2)
 600         except (AttributeError, IndexError):
 601             sock = u""
 602
 603         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 604                         replace(u"'", u'"').replace(u'b"', u'"').
 605                         replace(u'u"', u'"').split(u":", 1)[1])
 606
 607         try:
 608             threads_nr = len(runtime[0][u"clocks"])
 609         except (IndexError, KeyError):
 610             return
 611
 612         dut = u"DUT{nr}".format(
 613             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 614
 615         oper = {
 616             u"host": host,
 617             u"socket": sock,
 618             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 619         }
 620
 621         for item in runtime:
 622             for idx in range(threads_nr):
 623                 if item[u"vectors"][idx] > 0:
 624                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 625                 elif item[u"calls"][idx] > 0:
 626                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 627                 elif item[u"suspends"][idx] > 0:
 628                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 629                 else:
 630                     clocks = 0.0
 631
 632                 if item[u"calls"][idx] > 0:
 633                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 634                 else:
 635                     vectors_call = 0.0
 636
 637                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 638                         int(item[u"suspends"][idx]):
 639                     oper[u"threads"][idx].append([
 640                         item[u"name"],
 641                         item[u"calls"][idx],
 642                         item[u"vectors"][idx],
 643                         item[u"suspends"][idx],
 644                         clocks,
 645                         vectors_call
 646                     ])
 647
 648         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 649
 650     def _get_ndrpdr_throughput(self, msg):
 651         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 652         message.
 653
 654         :param msg: The test message to be parsed.
 655         :type msg: str
 656         :returns: Parsed data as a dict and the status (PASS/FAIL).
 657         :rtype: tuple(dict, str)
 658         """
 659
 660         throughput = {
 661             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 662             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 663         }
 664         status = u"FAIL"
 665         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 666
 667         if groups is not None:
 668             try:
 669                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 670                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 671                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 672                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 673                 status = u"PASS"
 674             except (IndexError, ValueError):
 675                 pass
 676
 677         return throughput, status
 678
 679     def _get_plr_throughput(self, msg):
 680         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 681         message.
 682
 683         :param msg: The test message to be parsed.
 684         :type msg: str
 685         :returns: Parsed data as a dict and the status (PASS/FAIL).
 686         :rtype: tuple(dict, str)
 687         """
 688
 689         throughput = {
 690             u"LOWER": -1.0,
 691             u"UPPER": -1.0
 692         }
 693         status = u"FAIL"
 694         groups = re.search(self.REGEX_PLR_RATE, msg)
 695
 696         if groups is not None:
 697             try:
 698                 throughput[u"LOWER"] = float(groups.group(1))
 699                 throughput[u"UPPER"] = float(groups.group(2))
 700                 status = u"PASS"
 701             except (IndexError, ValueError):
 702                 pass
 703
 704         return throughput, status
 705
 706     def _get_ndrpdr_latency(self, msg):
 707         """Get LATENCY from the test message.
 708
 709         :param msg: The test message to be parsed.
 710         :type msg: str
 711         :returns: Parsed data as a dict and the status (PASS/FAIL).
 712         :rtype: tuple(dict, str)
 713         """
 714         latency_default = {
 715             u"min": -1.0,
 716             u"avg": -1.0,
 717             u"max": -1.0,
 718             u"hdrh": u""
 719         }
 720         latency = {
 721             u"NDR": {
 722                 u"direction1": copy.copy(latency_default),
 723                 u"direction2": copy.copy(latency_default)
 724             },
 725             u"PDR": {
 726                 u"direction1": copy.copy(latency_default),
 727                 u"direction2": copy.copy(latency_default)
 728             },
 729             u"LAT0": {
 730                 u"direction1": copy.copy(latency_default),
 731                 u"direction2": copy.copy(latency_default)
 732             },
 733             u"PDR10": {
 734                 u"direction1": copy.copy(latency_default),
 735                 u"direction2": copy.copy(latency_default)
 736             },
 737             u"PDR50": {
 738                 u"direction1": copy.copy(latency_default),
 739                 u"direction2": copy.copy(latency_default)
 740             },
 741             u"PDR90": {
 742                 u"direction1": copy.copy(latency_default),
 743                 u"direction2": copy.copy(latency_default)
 744             },
 745         }
 746
 747         # TODO: Rewrite when long and base are not needed
 748         groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
 749         if groups is None:
 750             groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 751         if groups is None:
 752             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 753         if groups is None:
 754             return latency, u"FAIL"
 755
 756         def process_latency(in_str):
 757             """Return object with parsed latency values.
 758
 759             TODO: Define class for the return type.
 760
 761             :param in_str: Input string, min/avg/max/hdrh format.
 762             :type in_str: str
 763             :returns: Dict with corresponding keys, except hdrh float values.
 764             :rtype dict:
 765             :throws IndexError: If in_str does not have enough substrings.
 766             :throws ValueError: If a substring does not convert to float.
 767             """
 768             in_list = in_str.split('/', 3)
 769
 770             rval = {
 771                 u"min": float(in_list[0]),
 772                 u"avg": float(in_list[1]),
 773                 u"max": float(in_list[2]),
 774                 u"hdrh": u""
 775             }
 776
 777             if len(in_list) == 4:
 778                 rval[u"hdrh"] = str(in_list[3])
 779
 780             return rval
 781
 782         try:
 783             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 784             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 785             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 786             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 787             if groups.lastindex == 4:
 788                 return latency, u"PASS"
 789         except (IndexError, ValueError):
 790             pass
 791
 792         try:
 793             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 794             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 795             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 796             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 797             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 798             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 799             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 800             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 801             if groups.lastindex == 12:
 802                 return latency, u"PASS"
 803         except (IndexError, ValueError):
 804             pass
 805
 806         # TODO: Remove when not needed
 807         latency[u"NDR10"] = {
 808             u"direction1": copy.copy(latency_default),
 809             u"direction2": copy.copy(latency_default)
 810         }
 811         latency[u"NDR50"] = {
 812             u"direction1": copy.copy(latency_default),
 813             u"direction2": copy.copy(latency_default)
 814         }
 815         latency[u"NDR90"] = {
 816             u"direction1": copy.copy(latency_default),
 817             u"direction2": copy.copy(latency_default)
 818         }
 819         try:
 820             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
 821             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
 822             latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
 823             latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
 824             latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
 825             latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
 826             latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
 827             latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
 828             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
 829             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
 830             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
 831             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
 832             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
 833             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
 834             return latency, u"PASS"
 835         except (IndexError, ValueError):
 836             pass
 837
 838         return latency, u"FAIL"
 839
 840     def visit_suite(self, suite):
 841         """Implements traversing through the suite and its direct children.
 842
 843         :param suite: Suite to process.
 844         :type suite: Suite
 845         :returns: Nothing.
 846         """
 847         if self.start_suite(suite) is not False:
 848             suite.suites.visit(self)
 849             suite.tests.visit(self)
 850             self.end_suite(suite)
 851
 852     def start_suite(self, suite):
 853         """Called when suite starts.
 854
 855         :param suite: Suite to process.
 856         :type suite: Suite
 857         :returns: Nothing.
 858         """
 859
 860         try:
 861             parent_name = suite.parent.name
 862         except AttributeError:
 863             return
 864
 865         doc_str = suite.doc.\
 866             replace(u'"', u"'").\
 867             replace(u'\n', u' ').\
 868             replace(u'\r', u'').\
 869             replace(u'*[', u' |br| *[').\
 870             replace(u"*", u"**").\
 871             replace(u' |br| *[', u'*[', 1)
 872
 873         self._data[u"suites"][suite.longname.lower().
 874                               replace(u'"', u"'").
 875                               replace(u" ", u"_")] = {
 876                                   u"name": suite.name.lower(),
 877                                   u"doc": doc_str,
 878                                   u"parent": parent_name,
 879                                   u"level": len(suite.longname.split(u"."))
 880                               }
 881
 882         suite.keywords.visit(self)
 883
 884     def end_suite(self, suite):
 885         """Called when suite ends.
 886
 887         :param suite: Suite to process.
 888         :type suite: Suite
 889         :returns: Nothing.
 890         """
 891
 892     def visit_test(self, test):
 893         """Implements traversing through the test.
 894
 895         :param test: Test to process.
 896         :type test: Test
 897         :returns: Nothing.
 898         """
 899         if self.start_test(test) is not False:
 900             test.keywords.visit(self)
 901             self.end_test(test)
 902
 903     def start_test(self, test):
 904         """Called when test starts.
 905
 906         :param test: Test to process.
 907         :type test: Test
 908         :returns: Nothing.
 909         """
 910
 911         self._sh_run_counter = 0
 912
 913         longname_orig = test.longname.lower()
 914
 915         # Check the ignore list
 916         if longname_orig in self._ignore:
 917             return
 918
 919         tags = [str(tag) for tag in test.tags]
 920         test_result = dict()
 921
 922         # Change the TC long name and name if defined in the mapping table
 923         longname = self._mapping.get(longname_orig, None)
 924         if longname is not None:
 925             name = longname.split(u'.')[-1]
 926             logging.debug(
 927                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
 928                 f"{name}"
 929             )
 930         else:
 931             longname = longname_orig
 932             name = test.name.lower()
 933
 934         # Remove TC number from the TC long name (backward compatibility):
 935         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
 936         # Remove TC number from the TC name (not needed):
 937         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
 938
 939         test_result[u"parent"] = test.parent.name.lower()
 940         test_result[u"tags"] = tags
 941         test_result["doc"] = test.doc.\
 942             replace(u'"', u"'").\
 943             replace(u'\n', u' ').\
 944             replace(u'\r', u'').\
 945             replace(u'[', u' |br| [').\
 946             replace(u' |br| [', u'[', 1)
 947         test_result[u"msg"] = self._get_data_from_perf_test_msg(test.message).\
 948             replace(u'\n', u' |br| ').\
 949             replace(u'\r', u'').\
 950             replace(u'"', u"'")
 951         test_result[u"type"] = u"FUNC"
 952         test_result[u"status"] = test.status
 953
 954         if u"PERFTEST" in tags:
 955             # Replace info about cores (e.g. -1c-) with the info about threads
 956             # and cores (e.g. -1t1c-) in the long test case names and in the
 957             # test case names if necessary.
 958             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
 959             if not groups:
 960                 tag_count = 0
 961                 tag_tc = str()
 962                 for tag in test_result[u"tags"]:
 963                     groups = re.search(self.REGEX_TC_TAG, tag)
 964                     if groups:
 965                         tag_count += 1
 966                         tag_tc = tag
 967
 968                 if tag_count == 1:
 969                     self._test_id = re.sub(self.REGEX_TC_NAME_NEW,
 970                                            f"-{tag_tc.lower()}-",
 971                                            self._test_id,
 972                                            count=1)
 973                     test_result[u"name"] = re.sub(self.REGEX_TC_NAME_NEW,
 974                                                   f"-{tag_tc.lower()}-",
 975                                                   test_result["name"],
 976                                                   count=1)
 977                 else:
 978                     test_result[u"status"] = u"FAIL"
 979                     self._data[u"tests"][self._test_id] = test_result
 980                     logging.debug(
 981                         f"The test {self._test_id} has no or more than one "
 982                         f"multi-threading tags.\n"
 983                         f"Tags: {test_result[u'tags']}"
 984                     )
 985                     return
 986
 987         if test.status == u"PASS":
 988             if u"NDRPDR" in tags:
 989                 test_result[u"type"] = u"NDRPDR"
 990                 test_result[u"throughput"], test_result[u"status"] = \
 991                     self._get_ndrpdr_throughput(test.message)
 992                 test_result[u"latency"], test_result[u"status"] = \
 993                     self._get_ndrpdr_latency(test.message)
 994             elif u"SOAK" in tags:
 995                 test_result[u"type"] = u"SOAK"
 996                 test_result[u"throughput"], test_result[u"status"] = \
 997                     self._get_plr_throughput(test.message)
 998             elif u"TCP" in tags:
 999                 test_result[u"type"] = u"TCP"
1000                 groups = re.search(self.REGEX_TCP, test.message)
1001                 test_result[u"result"] = int(groups.group(2))
1002             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1003                 if u"MRR" in tags:
1004                     test_result[u"type"] = u"MRR"
1005                 else:
1006                     test_result[u"type"] = u"BMRR"
1007
1008                 test_result[u"result"] = dict()
1009                 groups = re.search(self.REGEX_BMRR, test.message)
1010                 if groups is not None:
1011                     items_str = groups.group(1)
1012                     items_float = [float(item.strip()) for item
1013                                    in items_str.split(",")]
1014                     # Use whole list in CSIT-1180.
1015                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1016                     test_result[u"result"][u"receive-rate"] = stats.avg
1017                 else:
1018                     groups = re.search(self.REGEX_MRR, test.message)
1019                     test_result[u"result"][u"receive-rate"] = \
1020                         float(groups.group(3)) / float(groups.group(1))
1021             elif u"RECONF" in tags:
1022                 test_result[u"type"] = u"RECONF"
1023                 test_result[u"result"] = None
1024                 try:
1025                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1026                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1027                     test_result[u"result"] = {
1028                         u"loss": int(grps_loss.group(1)),
1029                         u"time": float(grps_time.group(1))
1030                     }
1031                 except (AttributeError, IndexError, ValueError, TypeError):
1032                     test_result[u"status"] = u"FAIL"
1033             else:
1034                 test_result[u"status"] = u"FAIL"
1035                 self._data[u"tests"][self._test_id] = test_result
1036                 return
1037
1038         self._data[u"tests"][self._test_id] = test_result
1039
1040     def end_test(self, test):
1041         """Called when test ends.
1042
1043         :param test: Test to process.
1044         :type test: Test
1045         :returns: Nothing.
1046         """
1047
1048     def visit_keyword(self, keyword):
1049         """Implements traversing through the keyword and its child keywords.
1050
1051         :param keyword: Keyword to process.
1052         :type keyword: Keyword
1053         :returns: Nothing.
1054         """
1055         if self.start_keyword(keyword) is not False:
1056             self.end_keyword(keyword)
1057
1058     def start_keyword(self, keyword):
1059         """Called when keyword starts. Default implementation does nothing.
1060
1061         :param keyword: Keyword to process.
1062         :type keyword: Keyword
1063         :returns: Nothing.
1064         """
1065         try:
1066             if keyword.type == u"setup":
1067                 self.visit_setup_kw(keyword)
1068             elif keyword.type == u"teardown":
1069                 self.visit_teardown_kw(keyword)
1070             else:
1071                 self.visit_test_kw(keyword)
1072         except AttributeError:
1073             pass
1074
1075     def end_keyword(self, keyword):
1076         """Called when keyword ends. Default implementation does nothing.
1077
1078         :param keyword: Keyword to process.
1079         :type keyword: Keyword
1080         :returns: Nothing.
1081         """
1082
1083     def visit_test_kw(self, test_kw):
1084         """Implements traversing through the test keyword and its child
1085         keywords.
1086
1087         :param test_kw: Keyword to process.
1088         :type test_kw: Keyword
1089         :returns: Nothing.
1090         """
1091         for keyword in test_kw.keywords:
1092             if self.start_test_kw(keyword) is not False:
1093                 self.visit_test_kw(keyword)
1094                 self.end_test_kw(keyword)
1095
1096     def start_test_kw(self, test_kw):
1097         """Called when test keyword starts. Default implementation does
1098         nothing.
1099
1100         :param test_kw: Keyword to process.
1101         :type test_kw: Keyword
1102         :returns: Nothing.
1103         """
1104         if test_kw.name.count(u"Show Runtime On All Duts") or \
1105                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
1106             self._msg_type = u"test-show-runtime"
1107             self._sh_run_counter += 1
1108         elif test_kw.name.count(u"Install Dpdk Test") and not self._version:
1109             self._msg_type = u"dpdk-version"
1110         else:
1111             return
1112         test_kw.messages.visit(self)
1113
1114     def end_test_kw(self, test_kw):
1115         """Called when keyword ends. Default implementation does nothing.
1116
1117         :param test_kw: Keyword to process.
1118         :type test_kw: Keyword
1119         :returns: Nothing.
1120         """
1121
1122     def visit_setup_kw(self, setup_kw):
1123         """Implements traversing through the teardown keyword and its child
1124         keywords.
1125
1126         :param setup_kw: Keyword to process.
1127         :type setup_kw: Keyword
1128         :returns: Nothing.
1129         """
1130         for keyword in setup_kw.keywords:
1131             if self.start_setup_kw(keyword) is not False:
1132                 self.visit_setup_kw(keyword)
1133                 self.end_setup_kw(keyword)
1134
1135     def start_setup_kw(self, setup_kw):
1136         """Called when teardown keyword starts. Default implementation does
1137         nothing.
1138
1139         :param setup_kw: Keyword to process.
1140         :type setup_kw: Keyword
1141         :returns: Nothing.
1142         """
1143         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1144                 and not self._version:
1145             self._msg_type = u"vpp-version"
1146         elif setup_kw.name.count(u"Set Global Variable") \
1147                 and not self._timestamp:
1148             self._msg_type = u"timestamp"
1149         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1150             self._msg_type = u"testbed"
1151         else:
1152             return
1153         setup_kw.messages.visit(self)
1154
1155     def end_setup_kw(self, setup_kw):
1156         """Called when keyword ends. Default implementation does nothing.
1157
1158         :param setup_kw: Keyword to process.
1159         :type setup_kw: Keyword
1160         :returns: Nothing.
1161         """
1162
1163     def visit_teardown_kw(self, teardown_kw):
1164         """Implements traversing through the teardown keyword and its child
1165         keywords.
1166
1167         :param teardown_kw: Keyword to process.
1168         :type teardown_kw: Keyword
1169         :returns: Nothing.
1170         """
1171         for keyword in teardown_kw.keywords:
1172             if self.start_teardown_kw(keyword) is not False:
1173                 self.visit_teardown_kw(keyword)
1174                 self.end_teardown_kw(keyword)
1175
1176     def start_teardown_kw(self, teardown_kw):
1177         """Called when teardown keyword starts
1178
1179         :param teardown_kw: Keyword to process.
1180         :type teardown_kw: Keyword
1181         :returns: Nothing.
1182         """
1183
1184         if teardown_kw.name.count(u"Show Vat History On All Duts"):
1185             # TODO: Remove when not needed:
1186             self._conf_history_lookup_nr = 0
1187             self._msg_type = u"teardown-vat-history"
1188             teardown_kw.messages.visit(self)
1189         elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1190             self._conf_history_lookup_nr = 0
1191             self._msg_type = u"teardown-papi-history"
1192             teardown_kw.messages.visit(self)
1193
1194     def end_teardown_kw(self, teardown_kw):
1195         """Called when keyword ends. Default implementation does nothing.
1196
1197         :param teardown_kw: Keyword to process.
1198         :type teardown_kw: Keyword
1199         :returns: Nothing.
1200         """
1201
1202     def visit_message(self, msg):
1203         """Implements visiting the message.
1204
1205         :param msg: Message to process.
1206         :type msg: Message
1207         :returns: Nothing.
1208         """
1209         if self.start_message(msg) is not False:
1210             self.end_message(msg)
1211
1212     def start_message(self, msg):
1213         """Called when message starts. Get required information from messages:
1214         - VPP version.
1215
1216         :param msg: Message to process.
1217         :type msg: Message
1218         :returns: Nothing.
1219         """
1220
1221         if self._msg_type:
1222             self.parse_msg[self._msg_type](msg)
1223
1224     def end_message(self, msg):
1225         """Called when message ends. Default implementation does nothing.
1226
1227         :param msg: Message to process.
1228         :type msg: Message
1229         :returns: Nothing.
1230         """
1231
1232
1233 class InputData:
1234     """Input data
1235
1236     The data is extracted from output.xml files generated by Jenkins jobs and
1237     stored in pandas' DataFrames.
1238
1239     The data structure:
1240     - job name
1241       - build number
1242         - metadata
1243           (as described in ExecutionChecker documentation)
1244         - suites
1245           (as described in ExecutionChecker documentation)
1246         - tests
1247           (as described in ExecutionChecker documentation)
1248     """
1249
1250     def __init__(self, spec):
1251         """Initialization.
1252
1253         :param spec: Specification.
1254         :type spec: Specification
1255         """
1256
1257         # Specification:
1258         self._cfg = spec
1259
1260         # Data store:
1261         self._input_data = pd.Series()
1262
1263     @property
1264     def data(self):
1265         """Getter - Input data.
1266
1267         :returns: Input data
1268         :rtype: pandas.Series
1269         """
1270         return self._input_data
1271
1272     def metadata(self, job, build):
1273         """Getter - metadata
1274
1275         :param job: Job which metadata we want.
1276         :param build: Build which metadata we want.
1277         :type job: str
1278         :type build: str
1279         :returns: Metadata
1280         :rtype: pandas.Series
1281         """
1282
1283         return self.data[job][build][u"metadata"]
1284
1285     def suites(self, job, build):
1286         """Getter - suites
1287
1288         :param job: Job which suites we want.
1289         :param build: Build which suites we want.
1290         :type job: str
1291         :type build: str
1292         :returns: Suites.
1293         :rtype: pandas.Series
1294         """
1295
1296         return self.data[job][str(build)][u"suites"]
1297
1298     def tests(self, job, build):
1299         """Getter - tests
1300
1301         :param job: Job which tests we want.
1302         :param build: Build which tests we want.
1303         :type job: str
1304         :type build: str
1305         :returns: Tests.
1306         :rtype: pandas.Series
1307         """
1308
1309         return self.data[job][build][u"tests"]
1310
1311     def _parse_tests(self, job, build, log):
1312         """Process data from robot output.xml file and return JSON structured
1313         data.
1314
1315         :param job: The name of job which build output data will be processed.
1316         :param build: The build which output data will be processed.
1317         :param log: List of log messages.
1318         :type job: str
1319         :type build: dict
1320         :type log: list of tuples (severity, msg)
1321         :returns: JSON data structure.
1322         :rtype: dict
1323         """
1324
1325         metadata = {
1326             u"job": job,
1327             u"build": build
1328         }
1329
1330         with open(build[u"file-name"], u'r') as data_file:
1331             try:
1332                 result = ExecutionResult(data_file)
1333             except errors.DataError as err:
1334                 log.append(
1335                     (u"ERROR", f"Error occurred while parsing output.xml: "
1336                                f"{repr(err)}")
1337                 )
1338                 return None
1339         checker = ExecutionChecker(metadata, self._cfg.mapping,
1340                                    self._cfg.ignore)
1341         result.visit(checker)
1342
1343         return checker.data
1344
1345     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1346         """Download and parse the input data file.
1347
1348         :param pid: PID of the process executing this method.
1349         :param job: Name of the Jenkins job which generated the processed input
1350             file.
1351         :param build: Information about the Jenkins build which generated the
1352             processed input file.
1353         :param repeat: Repeat the download specified number of times if not
1354             successful.
1355         :type pid: int
1356         :type job: str
1357         :type build: dict
1358         :type repeat: int
1359         """
1360
1361         logs = list()
1362
1363         logs.append(
1364             (u"INFO", f"  Processing the job/build: {job}: {build[u'build']}")
1365         )
1366
1367         state = u"failed"
1368         success = False
1369         data = None
1370         do_repeat = repeat
1371         while do_repeat:
1372             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1373                                                    logs)
1374             if success:
1375                 break
1376             do_repeat -= 1
1377         if not success:
1378             logs.append(
1379                 (u"ERROR",
1380                  f"It is not possible to download the input data file from the "
1381                  f"job {job}, build {build[u'build']}, or it is damaged. "
1382                  f"Skipped.")
1383             )
1384         if success:
1385             logs.append(
1386                 (u"INFO",
1387                  f"    Processing data from the build {build[u'build']} ...")
1388             )
1389             data = self._parse_tests(job, build, logs)
1390             if data is None:
1391                 logs.append(
1392                     (u"ERROR",
1393                      f"Input data file from the job {job}, build "
1394                      f"{build[u'build']} is damaged. Skipped.")
1395                 )
1396             else:
1397                 state = u"processed"
1398
1399             try:
1400                 remove(build[u"file-name"])
1401             except OSError as err:
1402                 logs.append(
1403                     ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
1404                               f"{repr(err)}")
1405                 )
1406
1407         # If the time-period is defined in the specification file, remove all
1408         # files which are outside the time period.
1409         timeperiod = self._cfg.input.get(u"time-period", None)
1410         if timeperiod and data:
1411             now = dt.utcnow()
1412             timeperiod = timedelta(int(timeperiod))
1413             metadata = data.get(u"metadata", None)
1414             if metadata:
1415                 generated = metadata.get(u"generated", None)
1416                 if generated:
1417                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1418                     if (now - generated) > timeperiod:
1419                         # Remove the data and the file:
1420                         state = u"removed"
1421                         data = None
1422                         logs.append(
1423                             (u"INFO",
1424                              f"    The build {job}/{build[u'build']} is "
1425                              f"outdated, will be removed.")
1426                         )
1427         logs.append((u"INFO", u"  Done."))
1428
1429         for level, line in logs:
1430             if level == u"INFO":
1431                 logging.info(line)
1432             elif level == u"ERROR":
1433                 logging.error(line)
1434             elif level == u"DEBUG":
1435                 logging.debug(line)
1436             elif level == u"CRITICAL":
1437                 logging.critical(line)
1438             elif level == u"WARNING":
1439                 logging.warning(line)
1440
1441         return {u"data": data, u"state": state, u"job": job, u"build": build}
1442
1443     def download_and_parse_data(self, repeat=1):
1444         """Download the input data files, parse input data from input files and
1445         store in pandas' Series.
1446
1447         :param repeat: Repeat the download specified number of times if not
1448             successful.
1449         :type repeat: int
1450         """
1451
1452         logging.info(u"Downloading and parsing input files ...")
1453
1454         for job, builds in self._cfg.builds.items():
1455             for build in builds:
1456
1457                 result = self._download_and_parse_build(job, build, repeat)
1458                 build_nr = result[u"build"][u"build"]
1459
1460                 if result[u"data"]:
1461                     data = result[u"data"]
1462                     build_data = pd.Series({
1463                         u"metadata": pd.Series(
1464                             list(data[u"metadata"].values()),
1465                             index=list(data[u"metadata"].keys())
1466                         ),
1467                         u"suites": pd.Series(
1468                             list(data[u"suites"].values()),
1469                             index=list(data[u"suites"].keys())
1470                         ),
1471                         u"tests": pd.Series(
1472                             list(data[u"tests"].values()),
1473                             index=list(data[u"tests"].keys())
1474                         )
1475                     })
1476
1477                     if self._input_data.get(job, None) is None:
1478                         self._input_data[job] = pd.Series()
1479                     self._input_data[job][str(build_nr)] = build_data
1480
1481                     self._cfg.set_input_file_name(
1482                         job, build_nr, result[u"build"][u"file-name"])
1483
1484                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1485
1486                 mem_alloc = \
1487                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1488                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1489
1490         logging.info(u"Done.")
1491
1492     @staticmethod
1493     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1494         """Return the index of character in the string which is the end of tag.
1495
1496         :param tag_filter: The string where the end of tag is being searched.
1497         :param start: The index where the searching is stated.
1498         :param closer: The character which is the tag closer.
1499         :type tag_filter: str
1500         :type start: int
1501         :type closer: str
1502         :returns: The index of the tag closer.
1503         :rtype: int
1504         """
1505
1506         try:
1507             idx_opener = tag_filter.index(closer, start)
1508             return tag_filter.index(closer, idx_opener + 1)
1509         except ValueError:
1510             return None
1511
1512     @staticmethod
1513     def _condition(tag_filter):
1514         """Create a conditional statement from the given tag filter.
1515
1516         :param tag_filter: Filter based on tags from the element specification.
1517         :type tag_filter: str
1518         :returns: Conditional statement which can be evaluated.
1519         :rtype: str
1520         """
1521
1522         index = 0
1523         while True:
1524             index = InputData._end_of_tag(tag_filter, index)
1525             if index is None:
1526                 return tag_filter
1527             index += 1
1528             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1529
1530     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1531                     continue_on_error=False):
1532         """Filter required data from the given jobs and builds.
1533
1534         The output data structure is:
1535
1536         - job 1
1537           - build 1
1538             - test (or suite) 1 ID:
1539               - param 1
1540               - param 2
1541               ...
1542               - param n
1543             ...
1544             - test (or suite) n ID:
1545             ...
1546           ...
1547           - build n
1548         ...
1549         - job n
1550
1551         :param element: Element which will use the filtered data.
1552         :param params: Parameters which will be included in the output. If None,
1553             all parameters are included.
1554         :param data: If not None, this data is used instead of data specified
1555             in the element.
1556         :param data_set: The set of data to be filtered: tests, suites,
1557             metadata.
1558         :param continue_on_error: Continue if there is error while reading the
1559             data. The Item will be empty then
1560         :type element: pandas.Series
1561         :type params: list
1562         :type data: dict
1563         :type data_set: str
1564         :type continue_on_error: bool
1565         :returns: Filtered data.
1566         :rtype pandas.Series
1567         """
1568
1569         try:
1570             if data_set == "suites":
1571                 cond = u"True"
1572             elif element[u"filter"] in (u"all", u"template"):
1573                 cond = u"True"
1574             else:
1575                 cond = InputData._condition(element[u"filter"])
1576             logging.debug(f"   Filter: {cond}")
1577         except KeyError:
1578             logging.error(u"  No filter defined.")
1579             return None
1580
1581         if params is None:
1582             params = element.get(u"parameters", None)
1583             if params:
1584                 params.append(u"type")
1585
1586         data_to_filter = data if data else element[u"data"]
1587         data = pd.Series()
1588         try:
1589             for job, builds in data_to_filter.items():
1590                 data[job] = pd.Series()
1591                 for build in builds:
1592                     data[job][str(build)] = pd.Series()
1593                     try:
1594                         data_dict = dict(
1595                             self.data[job][str(build)][data_set].items())
1596                     except KeyError:
1597                         if continue_on_error:
1598                             continue
1599                         return None
1600
1601                     for test_id, test_data in data_dict.items():
1602                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1603                             data[job][str(build)][test_id] = pd.Series()
1604                             if params is None:
1605                                 for param, val in test_data.items():
1606                                     data[job][str(build)][test_id][param] = val
1607                             else:
1608                                 for param in params:
1609                                     try:
1610                                         data[job][str(build)][test_id][param] =\
1611                                             test_data[param]
1612                                     except KeyError:
1613                                         data[job][str(build)][test_id][param] =\
1614                                             u"No Data"
1615             return data
1616
1617         except (KeyError, IndexError, ValueError) as err:
1618             logging.error(
1619                 f"Missing mandatory parameter in the element specification: "
1620                 f"{repr(err)}"
1621             )
1622             return None
1623         except AttributeError as err:
1624             logging.error(repr(err))
1625             return None
1626         except SyntaxError as err:
1627             logging.error(
1628                 f"The filter {cond} is not correct. Check if all tags are "
1629                 f"enclosed by apostrophes.\n{repr(err)}"
1630             )
1631             return None
1632
1633     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1634                              continue_on_error=False):
1635         """Filter required data from the given jobs and builds.
1636
1637         The output data structure is:
1638
1639         - job 1
1640           - build 1
1641             - test (or suite) 1 ID:
1642               - param 1
1643               - param 2
1644               ...
1645               - param n
1646             ...
1647             - test (or suite) n ID:
1648             ...
1649           ...
1650           - build n
1651         ...
1652         - job n
1653
1654         :param element: Element which will use the filtered data.
1655         :param params: Parameters which will be included in the output. If None,
1656         all parameters are included.
1657         :param data_set: The set of data to be filtered: tests, suites,
1658         metadata.
1659         :param continue_on_error: Continue if there is error while reading the
1660         data. The Item will be empty then
1661         :type element: pandas.Series
1662         :type params: list
1663         :type data_set: str
1664         :type continue_on_error: bool
1665         :returns: Filtered data.
1666         :rtype pandas.Series
1667         """
1668
1669         include = element.get(u"include", None)
1670         if not include:
1671             logging.warning(u"No tests to include, skipping the element.")
1672             return None
1673
1674         if params is None:
1675             params = element.get(u"parameters", None)
1676             if params:
1677                 params.append(u"type")
1678
1679         data = pd.Series()
1680         try:
1681             for job, builds in element[u"data"].items():
1682                 data[job] = pd.Series()
1683                 for build in builds:
1684                     data[job][str(build)] = pd.Series()
1685                     for test in include:
1686                         try:
1687                             reg_ex = re.compile(str(test).lower())
1688                             for test_id in self.data[job][
1689                                     str(build)][data_set].keys():
1690                                 if re.match(reg_ex, str(test_id).lower()):
1691                                     test_data = self.data[job][
1692                                         str(build)][data_set][test_id]
1693                                     data[job][str(build)][test_id] = pd.Series()
1694                                     if params is None:
1695                                         for param, val in test_data.items():
1696                                             data[job][str(build)][test_id]\
1697                                                 [param] = val
1698                                     else:
1699                                         for param in params:
1700                                             try:
1701                                                 data[job][str(build)][
1702                                                     test_id][param] = \
1703                                                     test_data[param]
1704                                             except KeyError:
1705                                                 data[job][str(build)][
1706                                                     test_id][param] = u"No Data"
1707                         except KeyError as err:
1708                             logging.error(repr(err))
1709                             if continue_on_error:
1710                                 continue
1711                             return None
1712             return data
1713
1714         except (KeyError, IndexError, ValueError) as err:
1715             logging.error(
1716                 f"Missing mandatory parameter in the element "
1717                 f"specification: {repr(err)}"
1718             )
1719             return None
1720         except AttributeError as err:
1721             logging.error(repr(err))
1722             return None
1723
1724     @staticmethod
1725     def merge_data(data):
1726         """Merge data from more jobs and builds to a simple data structure.
1727
1728         The output data structure is:
1729
1730         - test (suite) 1 ID:
1731           - param 1
1732           - param 2
1733           ...
1734           - param n
1735         ...
1736         - test (suite) n ID:
1737         ...
1738
1739         :param data: Data to merge.
1740         :type data: pandas.Series
1741         :returns: Merged data.
1742         :rtype: pandas.Series
1743         """
1744
1745         logging.info(u"    Merging data ...")
1746
1747         merged_data = pd.Series()
1748         for builds in data.values:
1749             for item in builds.values:
1750                 for item_id, item_data in item.items():
1751                     merged_data[item_id] = item_data
1752
1753         return merged_data
1754
1755     def print_all_oper_data(self):
1756         """Print all operational data to console.
1757         """
1758
1759         tbl_hdr = (
1760             u"Name",
1761             u"Nr of Vectors",
1762             u"Nr of Packets",
1763             u"Suspends",
1764             u"Cycles per Packet",
1765             u"Average Vector Size"
1766         )
1767
1768         for job in self._input_data.values:
1769             for build in job.values:
1770                 for test_id, test_data in build[u"tests"].items():
1771                     print(f"{test_id}")
1772                     if test_data.get(u"show-run", None) is None:
1773                         continue
1774                     for dut_name, data in test_data[u"show-run"].items():
1775                         if data.get(u"threads", None) is None:
1776                             continue
1777                         print(f"Host IP: {data.get(u'host', '')}, "
1778                               f"Socket: {data.get(u'socket', '')}")
1779                         for thread_nr, thread in data[u"threads"].items():
1780                             txt_table = prettytable.PrettyTable(tbl_hdr)
1781                             avg = 0.0
1782                             for row in thread:
1783                                 txt_table.add_row(row)
1784                                 avg += row[-1]
1785                             if len(thread) == 0:
1786                                 avg = u""
1787                             else:
1788                                 avg = f", Average Vector Size per Node: " \
1789                                       f"{(avg / len(thread)):.2f}"
1790                             th_name = u"main" if thread_nr == 0 \
1791                                 else f"worker_{thread_nr}"
1792                             print(f"{dut_name}, {th_name}{avg}")
1793                             txt_table.float_format = u".2"
1794                             txt_table.align = u"r"
1795                             txt_table.align[u"Name"] = u"l"
1796                             print(f"{txt_table.get_string()}\n")