resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove
  29 from datetime import datetime as dt
  30 from datetime import timedelta
  31 from json import loads
  32
  33 import hdrh.histogram
  34 import hdrh.codec
  35 import prettytable
  36 import pandas as pd
  37
  38 from robot.api import ExecutionResult, ResultVisitor
  39 from robot import errors
  40
  41 from resources.libraries.python import jumpavg
  42 from input_data_files import download_and_unzip_data_file
  43
  44
  45 # Separator used in file names
  46 SEPARATOR = u"__"
  47
  48
  49 class ExecutionChecker(ResultVisitor):
  50     """Class to traverse through the test suite structure.
  51
  52     The functionality implemented in this class generates a json structure:
  53
  54     Performance tests:
  55
  56     {
  57         "metadata": {
  58             "generated": "Timestamp",
  59             "version": "SUT version",
  60             "job": "Jenkins job name",
  61             "build": "Information about the build"
  62         },
  63         "suites": {
  64             "Suite long name 1": {
  65                 "name": Suite name,
  66                 "doc": "Suite 1 documentation",
  67                 "parent": "Suite 1 parent",
  68                 "level": "Level of the suite in the suite hierarchy"
  69             }
  70             "Suite long name N": {
  71                 "name": Suite name,
  72                 "doc": "Suite N documentation",
  73                 "parent": "Suite 2 parent",
  74                 "level": "Level of the suite in the suite hierarchy"
  75             }
  76         }
  77         "tests": {
  78             # NDRPDR tests:
  79             "ID": {
  80                 "name": "Test name",
  81                 "parent": "Name of the parent of the test",
  82                 "doc": "Test documentation",
  83                 "msg": "Test message",
  84                 "conf-history": "DUT1 and DUT2 VAT History",
  85                 "show-run": "Show Run",
  86                 "tags": ["tag 1", "tag 2", "tag n"],
  87                 "type": "NDRPDR",
  88                 "status": "PASS" | "FAIL",
  89                 "throughput": {
  90                     "NDR": {
  91                         "LOWER": float,
  92                         "UPPER": float
  93                     },
  94                     "PDR": {
  95                         "LOWER": float,
  96                         "UPPER": float
  97                     }
  98                 },
  99                 "latency": {
 100                     "NDR": {
 101                         "direction1": {
 102                             "min": float,
 103                             "avg": float,
 104                             "max": float,
 105                             "hdrh": str
 106                         },
 107                         "direction2": {
 108                             "min": float,
 109                             "avg": float,
 110                             "max": float,
 111                             "hdrh": str
 112                         }
 113                     },
 114                     "PDR": {
 115                         "direction1": {
 116                             "min": float,
 117                             "avg": float,
 118                             "max": float,
 119                             "hdrh": str
 120                         },
 121                         "direction2": {
 122                             "min": float,
 123                             "avg": float,
 124                             "max": float,
 125                             "hdrh": str
 126                         }
 127                     }
 128                 }
 129             }
 130
 131             # TCP tests:
 132             "ID": {
 133                 "name": "Test name",
 134                 "parent": "Name of the parent of the test",
 135                 "doc": "Test documentation",
 136                 "msg": "Test message",
 137                 "tags": ["tag 1", "tag 2", "tag n"],
 138                 "type": "TCP",
 139                 "status": "PASS" | "FAIL",
 140                 "result": int
 141             }
 142
 143             # MRR, BMRR tests:
 144             "ID": {
 145                 "name": "Test name",
 146                 "parent": "Name of the parent of the test",
 147                 "doc": "Test documentation",
 148                 "msg": "Test message",
 149                 "tags": ["tag 1", "tag 2", "tag n"],
 150                 "type": "MRR" | "BMRR",
 151                 "status": "PASS" | "FAIL",
 152                 "result": {
 153                     "receive-rate": float,
 154                     # Average of a list, computed using AvgStdevStats.
 155                     # In CSIT-1180, replace with List[float].
 156                 }
 157             }
 158
 159             "ID" {
 160                 # next test
 161             }
 162         }
 163     }
 164
 165
 166     Functional tests:
 167
 168     {
 169         "metadata": {  # Optional
 170             "version": "VPP version",
 171             "job": "Jenkins job name",
 172             "build": "Information about the build"
 173         },
 174         "suites": {
 175             "Suite name 1": {
 176                 "doc": "Suite 1 documentation",
 177                 "parent": "Suite 1 parent",
 178                 "level": "Level of the suite in the suite hierarchy"
 179             }
 180             "Suite name N": {
 181                 "doc": "Suite N documentation",
 182                 "parent": "Suite 2 parent",
 183                 "level": "Level of the suite in the suite hierarchy"
 184             }
 185         }
 186         "tests": {
 187             "ID": {
 188                 "name": "Test name",
 189                 "parent": "Name of the parent of the test",
 190                 "doc": "Test documentation"
 191                 "msg": "Test message"
 192                 "tags": ["tag 1", "tag 2", "tag n"],
 193                 "conf-history": "DUT1 and DUT2 VAT History"
 194                 "show-run": "Show Run"
 195                 "status": "PASS" | "FAIL"
 196             },
 197             "ID" {
 198                 # next test
 199             }
 200         }
 201     }
 202
 203     .. note:: ID is the lowercase full path to the test.
 204     """
 205
 206     REGEX_PLR_RATE = re.compile(
 207         r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 208         r'PLRsearch upper bound::?\s(\d+.\d+)'
 209     )
 210     REGEX_NDRPDR_RATE = re.compile(
 211         r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 212         r'NDR_UPPER:\s(\d+.\d+).*\n'
 213         r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 214         r'PDR_UPPER:\s(\d+.\d+)'
 215     )
 216     REGEX_PERF_MSG_INFO = re.compile(
 217         r'NDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 218         r'PDR_LOWER:\s(\d+.\d+)\s.*\s(\d+.\d+)\s.*\n.*\n.*\n'
 219         r'Latency at 90% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 220         r'Latency at 50% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 221         r'Latency at 10% PDR:.*\[\'(.*)\', \'(.*)\'\].*\n'
 222     )
 223     REGEX_MRR_MSG_INFO = re.compile(r'.*\[(.*)\]')
 224
 225     # TODO: Remove when not needed
 226     REGEX_NDRPDR_LAT_BASE = re.compile(
 227         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 228         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]'
 229     )
 230     REGEX_NDRPDR_LAT = re.compile(
 231         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 232         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 233         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 234         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 235         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 236         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 237     )
 238     # TODO: Remove when not needed
 239     REGEX_NDRPDR_LAT_LONG = re.compile(
 240         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 241         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 242         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 243         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 244         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 245         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 246         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 247         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 248         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 249     )
 250     REGEX_VERSION_VPP = re.compile(
 251         r"(return STDOUT Version:\s*|"
 252         r"VPP Version:\s*|VPP version:\s*)(.*)"
 253     )
 254     REGEX_VERSION_DPDK = re.compile(
 255         r"(DPDK version:\s*|DPDK Version:\s*)(.*)"
 256     )
 257     REGEX_TCP = re.compile(
 258         r'Total\s(rps|cps|throughput):\s(\d*).*$'
 259     )
 260     REGEX_MRR = re.compile(
 261         r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 262         r'tx\s(\d*),\srx\s(\d*)'
 263     )
 264     REGEX_BMRR = re.compile(
 265         r'Maximum Receive Rate trial results'
 266         r' in packets per second: \[(.*)\]'
 267     )
 268     REGEX_RECONF_LOSS = re.compile(
 269         r'Packets lost due to reconfig: (\d*)'
 270     )
 271     REGEX_RECONF_TIME = re.compile(
 272         r'Implied time lost: (\d*.[\de-]*)'
 273     )
 274     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 275
 276     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 277
 278     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 279
 280     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 281
 282     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 283
 284     def __init__(self, metadata, mapping, ignore):
 285         """Initialisation.
 286
 287         :param metadata: Key-value pairs to be included in "metadata" part of
 288             JSON structure.
 289         :param mapping: Mapping of the old names of test cases to the new
 290             (actual) one.
 291         :param ignore: List of TCs to be ignored.
 292         :type metadata: dict
 293         :type mapping: dict
 294         :type ignore: list
 295         """
 296
 297         # Type of message to parse out from the test messages
 298         self._msg_type = None
 299
 300         # VPP version
 301         self._version = None
 302
 303         # Timestamp
 304         self._timestamp = None
 305
 306         # Testbed. The testbed is identified by TG node IP address.
 307         self._testbed = None
 308
 309         # Mapping of TCs long names
 310         self._mapping = mapping
 311
 312         # Ignore list
 313         self._ignore = ignore
 314
 315         # Number of PAPI History messages found:
 316         # 0 - no message
 317         # 1 - PAPI History of DUT1
 318         # 2 - PAPI History of DUT2
 319         self._conf_history_lookup_nr = 0
 320
 321         self._sh_run_counter = 0
 322
 323         # Test ID of currently processed test- the lowercase full path to the
 324         # test
 325         self._test_id = None
 326
 327         # The main data structure
 328         self._data = {
 329             u"metadata": OrderedDict(),
 330             u"suites": OrderedDict(),
 331             u"tests": OrderedDict()
 332         }
 333
 334         # Save the provided metadata
 335         for key, val in metadata.items():
 336             self._data[u"metadata"][key] = val
 337
 338         # Dictionary defining the methods used to parse different types of
 339         # messages
 340         self.parse_msg = {
 341             u"timestamp": self._get_timestamp,
 342             u"vpp-version": self._get_vpp_version,
 343             u"dpdk-version": self._get_dpdk_version,
 344             # TODO: Remove when not needed:
 345             u"teardown-vat-history": self._get_vat_history,
 346             u"teardown-papi-history": self._get_papi_history,
 347             u"test-show-runtime": self._get_show_run,
 348             u"testbed": self._get_testbed
 349         }
 350
 351     @property
 352     def data(self):
 353         """Getter - Data parsed from the XML file.
 354
 355         :returns: Data parsed from the XML file.
 356         :rtype: dict
 357         """
 358         return self._data
 359
 360     def _get_data_from_mrr_test_msg(self, msg):
 361         """Get info from message of MRR performance tests.
 362
 363         :param msg: Message to be processed.
 364         :type msg: str
 365         :returns: Processed message or original message if a problem occurs.
 366         :rtype: str
 367         """
 368
 369         groups = re.search(self.REGEX_MRR_MSG_INFO, msg)
 370         if not groups or groups.lastindex != 1:
 371             return u"Test Failed."
 372
 373         try:
 374             data = groups.group(1).split(u", ")
 375         except (AttributeError, IndexError, ValueError, KeyError):
 376             return u"Test Failed."
 377
 378         out_str = u"["
 379         try:
 380             for item in data:
 381                 out_str += f"{(float(item) / 1e6):.2f}, "
 382             return out_str[:-2] + u"]"
 383         except (AttributeError, IndexError, ValueError, KeyError):
 384             return u"Test Failed."
 385
 386     def _get_data_from_perf_test_msg(self, msg):
 387         """Get info from message of NDRPDR performance tests.
 388
 389         :param msg: Message to be processed.
 390         :type msg: str
 391         :returns: Processed message or original message if a problem occurs.
 392         :rtype: str
 393         """
 394
 395         groups = re.search(self.REGEX_PERF_MSG_INFO, msg)
 396         if not groups or groups.lastindex != 10:
 397             return u"Test Failed."
 398
 399         try:
 400             data = {
 401                 u"ndr_low": float(groups.group(1)),
 402                 u"ndr_low_b": float(groups.group(2)),
 403                 u"pdr_low": float(groups.group(3)),
 404                 u"pdr_low_b": float(groups.group(4)),
 405                 u"pdr_lat_90_1": groups.group(5),
 406                 u"pdr_lat_90_2": groups.group(6),
 407                 u"pdr_lat_50_1": groups.group(7),
 408                 u"pdr_lat_50_2": groups.group(8),
 409                 u"pdr_lat_10_1": groups.group(9),
 410                 u"pdr_lat_10_2": groups.group(10),
 411             }
 412         except (AttributeError, IndexError, ValueError, KeyError):
 413             return u"Test Failed."
 414
 415         def _process_lat(in_str_1, in_str_2):
 416             """Extract min, avg, max values from latency string.
 417
 418             :param in_str_1: Latency string for one direction produced by robot
 419                 framework.
 420             :param in_str_2: Latency string for second direction produced by
 421                 robot framework.
 422             :type in_str_1: str
 423             :type in_str_2: str
 424             :returns: Processed latency string or None if a problem occurs.
 425             :rtype: tuple
 426             """
 427             in_list_1 = in_str_1.split('/', 3)
 428             in_list_2 = in_str_2.split('/', 3)
 429
 430             if len(in_list_1) != 4 and len(in_list_2) != 4:
 431                 return None
 432
 433             in_list_1[3] += u"=" * (len(in_list_1[3]) % 4)
 434             try:
 435                 hdr_lat_1 = hdrh.histogram.HdrHistogram.decode(in_list_1[3])
 436             except hdrh.codec.HdrLengthException:
 437                 return None
 438
 439             in_list_2[3] += u"=" * (len(in_list_2[3]) % 4)
 440             try:
 441                 hdr_lat_2 = hdrh.histogram.HdrHistogram.decode(in_list_2[3])
 442             except hdrh.codec.HdrLengthException:
 443                 return None
 444
 445             if hdr_lat_1 and hdr_lat_2:
 446                 hdr_lat = (
 447                     hdr_lat_1.get_value_at_percentile(50.0),
 448                     hdr_lat_1.get_value_at_percentile(90.0),
 449                     hdr_lat_1.get_value_at_percentile(99.0),
 450                     hdr_lat_2.get_value_at_percentile(50.0),
 451                     hdr_lat_2.get_value_at_percentile(90.0),
 452                     hdr_lat_2.get_value_at_percentile(99.0)
 453                 )
 454
 455                 if all(hdr_lat):
 456                     return hdr_lat
 457
 458             return None
 459
 460         try:
 461             out_msg = (
 462                 f"1. {(data[u'ndr_low'] / 1e6):.2f}      "
 463                 f"{data[u'ndr_low_b']:.2f}"
 464                 f"\n2. {(data[u'pdr_low'] / 1e6):.2f}      "
 465                 f"{data[u'pdr_low_b']:.2f}"
 466             )
 467             latency = (
 468                 _process_lat(data[u'pdr_lat_10_1'], data[u'pdr_lat_10_2']),
 469                 _process_lat(data[u'pdr_lat_50_1'], data[u'pdr_lat_50_2']),
 470                 _process_lat(data[u'pdr_lat_90_1'], data[u'pdr_lat_90_2'])
 471             )
 472             if all(latency):
 473                 max_len = len(str(max((max(item) for item in latency))))
 474
 475                 for idx, lat in enumerate(latency):
 476                     if not idx:
 477                         out_msg += u"\n"
 478                     out_msg += f"\n{idx + 3}. "
 479                     for count, itm in enumerate(lat):
 480                         if count == 3:
 481                             out_msg += u" " * 6
 482                         out_msg += u" " * (max_len - len(str(itm)) + 1)
 483                         out_msg += str(itm)
 484
 485             return out_msg
 486
 487         except (AttributeError, IndexError, ValueError, KeyError):
 488             return u"Test Failed."
 489
 490     def _get_testbed(self, msg):
 491         """Called when extraction of testbed IP is required.
 492         The testbed is identified by TG node IP address.
 493
 494         :param msg: Message to process.
 495         :type msg: Message
 496         :returns: Nothing.
 497         """
 498
 499         if msg.message.count(u"Setup of TG node") or \
 500                 msg.message.count(u"Setup of node TG host"):
 501             reg_tg_ip = re.compile(
 502                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 503             try:
 504                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 505             except (KeyError, ValueError, IndexError, AttributeError):
 506                 pass
 507             finally:
 508                 self._data[u"metadata"][u"testbed"] = self._testbed
 509                 self._msg_type = None
 510
 511     def _get_vpp_version(self, msg):
 512         """Called when extraction of VPP version is required.
 513
 514         :param msg: Message to process.
 515         :type msg: Message
 516         :returns: Nothing.
 517         """
 518
 519         if msg.message.count(u"return STDOUT Version:") or \
 520             msg.message.count(u"VPP Version:") or \
 521             msg.message.count(u"VPP version:"):
 522             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 523                                 group(2))
 524             self._data[u"metadata"][u"version"] = self._version
 525             self._msg_type = None
 526
 527     def _get_dpdk_version(self, msg):
 528         """Called when extraction of DPDK version is required.
 529
 530         :param msg: Message to process.
 531         :type msg: Message
 532         :returns: Nothing.
 533         """
 534
 535         if msg.message.count(u"DPDK Version:"):
 536             try:
 537                 self._version = str(re.search(
 538                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 539                 self._data[u"metadata"][u"version"] = self._version
 540             except IndexError:
 541                 pass
 542             finally:
 543                 self._msg_type = None
 544
 545     def _get_timestamp(self, msg):
 546         """Called when extraction of timestamp is required.
 547
 548         :param msg: Message to process.
 549         :type msg: Message
 550         :returns: Nothing.
 551         """
 552
 553         self._timestamp = msg.timestamp[:14]
 554         self._data[u"metadata"][u"generated"] = self._timestamp
 555         self._msg_type = None
 556
 557     def _get_vat_history(self, msg):
 558         """Called when extraction of VAT command history is required.
 559
 560         TODO: Remove when not needed.
 561
 562         :param msg: Message to process.
 563         :type msg: Message
 564         :returns: Nothing.
 565         """
 566         if msg.message.count(u"VAT command history:"):
 567             self._conf_history_lookup_nr += 1
 568             if self._conf_history_lookup_nr == 1:
 569                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 570             else:
 571                 self._msg_type = None
 572             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 573                           r"VAT command history:", u"",
 574                           msg.message, count=1).replace(u'\n', u' |br| ').\
 575                 replace(u'"', u"'")
 576
 577             self._data[u"tests"][self._test_id][u"conf-history"] += (
 578                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 579             )
 580
 581     def _get_papi_history(self, msg):
 582         """Called when extraction of PAPI command history is required.
 583
 584         :param msg: Message to process.
 585         :type msg: Message
 586         :returns: Nothing.
 587         """
 588         if msg.message.count(u"PAPI command history:"):
 589             self._conf_history_lookup_nr += 1
 590             if self._conf_history_lookup_nr == 1:
 591                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 592             else:
 593                 self._msg_type = None
 594             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 595                           r"PAPI command history:", u"",
 596                           msg.message, count=1).replace(u'\n', u' |br| ').\
 597                 replace(u'"', u"'")
 598             self._data[u"tests"][self._test_id][u"conf-history"] += (
 599                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 600             )
 601
 602     def _get_show_run(self, msg):
 603         """Called when extraction of VPP operational data (output of CLI command
 604         Show Runtime) is required.
 605
 606         :param msg: Message to process.
 607         :type msg: Message
 608         :returns: Nothing.
 609         """
 610
 611         if not msg.message.count(u"stats runtime"):
 612             return
 613
 614         # Temporary solution
 615         if self._sh_run_counter > 1:
 616             return
 617
 618         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 619             self._data[u"tests"][self._test_id][u"show-run"] = dict()
 620
 621         groups = re.search(self.REGEX_TC_PAPI_CLI, msg.message)
 622         if not groups:
 623             return
 624         try:
 625             host = groups.group(1)
 626         except (AttributeError, IndexError):
 627             host = u""
 628         try:
 629             sock = groups.group(2)
 630         except (AttributeError, IndexError):
 631             sock = u""
 632
 633         runtime = loads(str(msg.message).replace(u' ', u'').replace(u'\n', u'').
 634                         replace(u"'", u'"').replace(u'b"', u'"').
 635                         replace(u'u"', u'"').split(u":", 1)[1])
 636
 637         try:
 638             threads_nr = len(runtime[0][u"clocks"])
 639         except (IndexError, KeyError):
 640             return
 641
 642         dut = u"DUT{nr}".format(
 643             nr=len(self._data[u'tests'][self._test_id][u'show-run'].keys()) + 1)
 644
 645         oper = {
 646             u"host": host,
 647             u"socket": sock,
 648             u"threads": OrderedDict({idx: list() for idx in range(threads_nr)})
 649         }
 650
 651         for item in runtime:
 652             for idx in range(threads_nr):
 653                 if item[u"vectors"][idx] > 0:
 654                     clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
 655                 elif item[u"calls"][idx] > 0:
 656                     clocks = item[u"clocks"][idx] / item[u"calls"][idx]
 657                 elif item[u"suspends"][idx] > 0:
 658                     clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
 659                 else:
 660                     clocks = 0.0
 661
 662                 if item[u"calls"][idx] > 0:
 663                     vectors_call = item[u"vectors"][idx] / item[u"calls"][idx]
 664                 else:
 665                     vectors_call = 0.0
 666
 667                 if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
 668                         int(item[u"suspends"][idx]):
 669                     oper[u"threads"][idx].append([
 670                         item[u"name"],
 671                         item[u"calls"][idx],
 672                         item[u"vectors"][idx],
 673                         item[u"suspends"][idx],
 674                         clocks,
 675                         vectors_call
 676                     ])
 677
 678         self._data[u'tests'][self._test_id][u'show-run'][dut] = copy.copy(oper)
 679
 680     def _get_ndrpdr_throughput(self, msg):
 681         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 682         message.
 683
 684         :param msg: The test message to be parsed.
 685         :type msg: str
 686         :returns: Parsed data as a dict and the status (PASS/FAIL).
 687         :rtype: tuple(dict, str)
 688         """
 689
 690         throughput = {
 691             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 692             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 693         }
 694         status = u"FAIL"
 695         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 696
 697         if groups is not None:
 698             try:
 699                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 700                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 701                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 702                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 703                 status = u"PASS"
 704             except (IndexError, ValueError):
 705                 pass
 706
 707         return throughput, status
 708
 709     def _get_plr_throughput(self, msg):
 710         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 711         message.
 712
 713         :param msg: The test message to be parsed.
 714         :type msg: str
 715         :returns: Parsed data as a dict and the status (PASS/FAIL).
 716         :rtype: tuple(dict, str)
 717         """
 718
 719         throughput = {
 720             u"LOWER": -1.0,
 721             u"UPPER": -1.0
 722         }
 723         status = u"FAIL"
 724         groups = re.search(self.REGEX_PLR_RATE, msg)
 725
 726         if groups is not None:
 727             try:
 728                 throughput[u"LOWER"] = float(groups.group(1))
 729                 throughput[u"UPPER"] = float(groups.group(2))
 730                 status = u"PASS"
 731             except (IndexError, ValueError):
 732                 pass
 733
 734         return throughput, status
 735
 736     def _get_ndrpdr_latency(self, msg):
 737         """Get LATENCY from the test message.
 738
 739         :param msg: The test message to be parsed.
 740         :type msg: str
 741         :returns: Parsed data as a dict and the status (PASS/FAIL).
 742         :rtype: tuple(dict, str)
 743         """
 744         latency_default = {
 745             u"min": -1.0,
 746             u"avg": -1.0,
 747             u"max": -1.0,
 748             u"hdrh": u""
 749         }
 750         latency = {
 751             u"NDR": {
 752                 u"direction1": copy.copy(latency_default),
 753                 u"direction2": copy.copy(latency_default)
 754             },
 755             u"PDR": {
 756                 u"direction1": copy.copy(latency_default),
 757                 u"direction2": copy.copy(latency_default)
 758             },
 759             u"LAT0": {
 760                 u"direction1": copy.copy(latency_default),
 761                 u"direction2": copy.copy(latency_default)
 762             },
 763             u"PDR10": {
 764                 u"direction1": copy.copy(latency_default),
 765                 u"direction2": copy.copy(latency_default)
 766             },
 767             u"PDR50": {
 768                 u"direction1": copy.copy(latency_default),
 769                 u"direction2": copy.copy(latency_default)
 770             },
 771             u"PDR90": {
 772                 u"direction1": copy.copy(latency_default),
 773                 u"direction2": copy.copy(latency_default)
 774             },
 775         }
 776
 777         # TODO: Rewrite when long and base are not needed
 778         groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
 779         if groups is None:
 780             groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 781         if groups is None:
 782             groups = re.search(self.REGEX_NDRPDR_LAT_BASE, msg)
 783         if groups is None:
 784             return latency, u"FAIL"
 785
 786         def process_latency(in_str):
 787             """Return object with parsed latency values.
 788
 789             TODO: Define class for the return type.
 790
 791             :param in_str: Input string, min/avg/max/hdrh format.
 792             :type in_str: str
 793             :returns: Dict with corresponding keys, except hdrh float values.
 794             :rtype dict:
 795             :throws IndexError: If in_str does not have enough substrings.
 796             :throws ValueError: If a substring does not convert to float.
 797             """
 798             in_list = in_str.split('/', 3)
 799
 800             rval = {
 801                 u"min": float(in_list[0]),
 802                 u"avg": float(in_list[1]),
 803                 u"max": float(in_list[2]),
 804                 u"hdrh": u""
 805             }
 806
 807             if len(in_list) == 4:
 808                 rval[u"hdrh"] = str(in_list[3])
 809
 810             return rval
 811
 812         try:
 813             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 814             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 815             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 816             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 817             if groups.lastindex == 4:
 818                 return latency, u"PASS"
 819         except (IndexError, ValueError):
 820             pass
 821
 822         try:
 823             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(5))
 824             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(6))
 825             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(7))
 826             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(8))
 827             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(9))
 828             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(10))
 829             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(11))
 830             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(12))
 831             if groups.lastindex == 12:
 832                 return latency, u"PASS"
 833         except (IndexError, ValueError):
 834             pass
 835
 836         # TODO: Remove when not needed
 837         latency[u"NDR10"] = {
 838             u"direction1": copy.copy(latency_default),
 839             u"direction2": copy.copy(latency_default)
 840         }
 841         latency[u"NDR50"] = {
 842             u"direction1": copy.copy(latency_default),
 843             u"direction2": copy.copy(latency_default)
 844         }
 845         latency[u"NDR90"] = {
 846             u"direction1": copy.copy(latency_default),
 847             u"direction2": copy.copy(latency_default)
 848         }
 849         try:
 850             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
 851             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
 852             latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
 853             latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
 854             latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
 855             latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
 856             latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
 857             latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
 858             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
 859             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
 860             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
 861             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
 862             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
 863             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
 864             return latency, u"PASS"
 865         except (IndexError, ValueError):
 866             pass
 867
 868         return latency, u"FAIL"
 869
 870     def visit_suite(self, suite):
 871         """Implements traversing through the suite and its direct children.
 872
 873         :param suite: Suite to process.
 874         :type suite: Suite
 875         :returns: Nothing.
 876         """
 877         if self.start_suite(suite) is not False:
 878             suite.suites.visit(self)
 879             suite.tests.visit(self)
 880             self.end_suite(suite)
 881
 882     def start_suite(self, suite):
 883         """Called when suite starts.
 884
 885         :param suite: Suite to process.
 886         :type suite: Suite
 887         :returns: Nothing.
 888         """
 889
 890         try:
 891             parent_name = suite.parent.name
 892         except AttributeError:
 893             return
 894
 895         doc_str = suite.doc.\
 896             replace(u'"', u"'").\
 897             replace(u'\n', u' ').\
 898             replace(u'\r', u'').\
 899             replace(u'*[', u' |br| *[').\
 900             replace(u"*", u"**").\
 901             replace(u' |br| *[', u'*[', 1)
 902
 903         self._data[u"suites"][suite.longname.lower().
 904                               replace(u'"', u"'").
 905                               replace(u" ", u"_")] = {
 906                                   u"name": suite.name.lower(),
 907                                   u"doc": doc_str,
 908                                   u"parent": parent_name,
 909                                   u"level": len(suite.longname.split(u"."))
 910                               }
 911
 912         suite.keywords.visit(self)
 913
 914     def end_suite(self, suite):
 915         """Called when suite ends.
 916
 917         :param suite: Suite to process.
 918         :type suite: Suite
 919         :returns: Nothing.
 920         """
 921
 922     def visit_test(self, test):
 923         """Implements traversing through the test.
 924
 925         :param test: Test to process.
 926         :type test: Test
 927         :returns: Nothing.
 928         """
 929         if self.start_test(test) is not False:
 930             test.keywords.visit(self)
 931             self.end_test(test)
 932
 933     def start_test(self, test):
 934         """Called when test starts.
 935
 936         :param test: Test to process.
 937         :type test: Test
 938         :returns: Nothing.
 939         """
 940
 941         self._sh_run_counter = 0
 942
 943         longname_orig = test.longname.lower()
 944
 945         # Check the ignore list
 946         if longname_orig in self._ignore:
 947             return
 948
 949         tags = [str(tag) for tag in test.tags]
 950         test_result = dict()
 951
 952         # Change the TC long name and name if defined in the mapping table
 953         longname = self._mapping.get(longname_orig, None)
 954         if longname is not None:
 955             name = longname.split(u'.')[-1]
 956             logging.debug(
 957                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
 958                 f"{name}"
 959             )
 960         else:
 961             longname = longname_orig
 962             name = test.name.lower()
 963
 964         # Remove TC number from the TC long name (backward compatibility):
 965         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
 966         # Remove TC number from the TC name (not needed):
 967         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
 968
 969         test_result[u"parent"] = test.parent.name.lower()
 970         test_result[u"tags"] = tags
 971         test_result["doc"] = test.doc.\
 972             replace(u'"', u"'").\
 973             replace(u'\n', u' ').\
 974             replace(u'\r', u'').\
 975             replace(u'[', u' |br| [').\
 976             replace(u' |br| [', u'[', 1)
 977         test_result[u"type"] = u"FUNC"
 978         test_result[u"status"] = test.status
 979
 980         if test.status == u"PASS":
 981             if u"NDRPDR" in tags:
 982                 test_result[u"msg"] = self._get_data_from_perf_test_msg(
 983                     test.message).replace(u'\n', u' |br| ').\
 984                     replace(u'\r', u'').replace(u'"', u"'")
 985             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
 986                 test_result[u"msg"] = self._get_data_from_mrr_test_msg(
 987                     test.message).replace(u'\n', u' |br| ').\
 988                     replace(u'\r', u'').replace(u'"', u"'")
 989             else:
 990                 test_result[u"msg"] = test.message.replace(u'\n', u' |br| ').\
 991                     replace(u'\r', u'').replace(u'"', u"'")
 992         else:
 993             test_result[u"msg"] = u"Test Failed."
 994
 995         if u"PERFTEST" in tags:
 996             # Replace info about cores (e.g. -1c-) with the info about threads
 997             # and cores (e.g. -1t1c-) in the long test case names and in the
 998             # test case names if necessary.
 999             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
1000             if not groups:
1001                 tag_count = 0
1002                 tag_tc = str()
1003                 for tag in test_result[u"tags"]:
1004                     groups = re.search(self.REGEX_TC_TAG, tag)
1005                     if groups:
1006                         tag_count += 1
1007                         tag_tc = tag
1008
1009                 if tag_count == 1:
1010                     self._test_id = re.sub(
1011                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1012                         self._test_id, count=1
1013                     )
1014                     test_result[u"name"] = re.sub(
1015                         self.REGEX_TC_NAME_NEW, f"-{tag_tc.lower()}-",
1016                         test_result["name"], count=1
1017                     )
1018                 else:
1019                     test_result[u"status"] = u"FAIL"
1020                     self._data[u"tests"][self._test_id] = test_result
1021                     logging.debug(
1022                         f"The test {self._test_id} has no or more than one "
1023                         f"multi-threading tags.\n"
1024                         f"Tags: {test_result[u'tags']}"
1025                     )
1026                     return
1027
1028         if test.status == u"PASS":
1029             if u"NDRPDR" in tags:
1030                 test_result[u"type"] = u"NDRPDR"
1031                 test_result[u"throughput"], test_result[u"status"] = \
1032                     self._get_ndrpdr_throughput(test.message)
1033                 test_result[u"latency"], test_result[u"status"] = \
1034                     self._get_ndrpdr_latency(test.message)
1035             elif u"SOAK" in tags:
1036                 test_result[u"type"] = u"SOAK"
1037                 test_result[u"throughput"], test_result[u"status"] = \
1038                     self._get_plr_throughput(test.message)
1039             elif u"TCP" in tags:
1040                 test_result[u"type"] = u"TCP"
1041                 groups = re.search(self.REGEX_TCP, test.message)
1042                 test_result[u"result"] = int(groups.group(2))
1043             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
1044                 if u"MRR" in tags:
1045                     test_result[u"type"] = u"MRR"
1046                 else:
1047                     test_result[u"type"] = u"BMRR"
1048
1049                 test_result[u"result"] = dict()
1050                 groups = re.search(self.REGEX_BMRR, test.message)
1051                 if groups is not None:
1052                     items_str = groups.group(1)
1053                     items_float = [float(item.strip()) for item
1054                                    in items_str.split(",")]
1055                     # Use whole list in CSIT-1180.
1056                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
1057                     test_result[u"result"][u"receive-rate"] = stats.avg
1058                 else:
1059                     groups = re.search(self.REGEX_MRR, test.message)
1060                     test_result[u"result"][u"receive-rate"] = \
1061                         float(groups.group(3)) / float(groups.group(1))
1062             elif u"RECONF" in tags:
1063                 test_result[u"type"] = u"RECONF"
1064                 test_result[u"result"] = None
1065                 try:
1066                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
1067                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
1068                     test_result[u"result"] = {
1069                         u"loss": int(grps_loss.group(1)),
1070                         u"time": float(grps_time.group(1))
1071                     }
1072                 except (AttributeError, IndexError, ValueError, TypeError):
1073                     test_result[u"status"] = u"FAIL"
1074             elif u"DEVICETEST" in tags:
1075                 test_result[u"type"] = u"DEVICETEST"
1076             else:
1077                 test_result[u"status"] = u"FAIL"
1078                 self._data[u"tests"][self._test_id] = test_result
1079                 return
1080
1081         self._data[u"tests"][self._test_id] = test_result
1082
1083     def end_test(self, test):
1084         """Called when test ends.
1085
1086         :param test: Test to process.
1087         :type test: Test
1088         :returns: Nothing.
1089         """
1090
1091     def visit_keyword(self, keyword):
1092         """Implements traversing through the keyword and its child keywords.
1093
1094         :param keyword: Keyword to process.
1095         :type keyword: Keyword
1096         :returns: Nothing.
1097         """
1098         if self.start_keyword(keyword) is not False:
1099             self.end_keyword(keyword)
1100
1101     def start_keyword(self, keyword):
1102         """Called when keyword starts. Default implementation does nothing.
1103
1104         :param keyword: Keyword to process.
1105         :type keyword: Keyword
1106         :returns: Nothing.
1107         """
1108         try:
1109             if keyword.type == u"setup":
1110                 self.visit_setup_kw(keyword)
1111             elif keyword.type == u"teardown":
1112                 self.visit_teardown_kw(keyword)
1113             else:
1114                 self.visit_test_kw(keyword)
1115         except AttributeError:
1116             pass
1117
1118     def end_keyword(self, keyword):
1119         """Called when keyword ends. Default implementation does nothing.
1120
1121         :param keyword: Keyword to process.
1122         :type keyword: Keyword
1123         :returns: Nothing.
1124         """
1125
1126     def visit_test_kw(self, test_kw):
1127         """Implements traversing through the test keyword and its child
1128         keywords.
1129
1130         :param test_kw: Keyword to process.
1131         :type test_kw: Keyword
1132         :returns: Nothing.
1133         """
1134         for keyword in test_kw.keywords:
1135             if self.start_test_kw(keyword) is not False:
1136                 self.visit_test_kw(keyword)
1137                 self.end_test_kw(keyword)
1138
1139     def start_test_kw(self, test_kw):
1140         """Called when test keyword starts. Default implementation does
1141         nothing.
1142
1143         :param test_kw: Keyword to process.
1144         :type test_kw: Keyword
1145         :returns: Nothing.
1146         """
1147         if test_kw.name.count(u"Show Runtime On All Duts") or \
1148                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
1149             self._msg_type = u"test-show-runtime"
1150             self._sh_run_counter += 1
1151         elif test_kw.name.count(u"Install Dpdk Test") and not self._version:
1152             self._msg_type = u"dpdk-version"
1153         else:
1154             return
1155         test_kw.messages.visit(self)
1156
1157     def end_test_kw(self, test_kw):
1158         """Called when keyword ends. Default implementation does nothing.
1159
1160         :param test_kw: Keyword to process.
1161         :type test_kw: Keyword
1162         :returns: Nothing.
1163         """
1164
1165     def visit_setup_kw(self, setup_kw):
1166         """Implements traversing through the teardown keyword and its child
1167         keywords.
1168
1169         :param setup_kw: Keyword to process.
1170         :type setup_kw: Keyword
1171         :returns: Nothing.
1172         """
1173         for keyword in setup_kw.keywords:
1174             if self.start_setup_kw(keyword) is not False:
1175                 self.visit_setup_kw(keyword)
1176                 self.end_setup_kw(keyword)
1177
1178     def start_setup_kw(self, setup_kw):
1179         """Called when teardown keyword starts. Default implementation does
1180         nothing.
1181
1182         :param setup_kw: Keyword to process.
1183         :type setup_kw: Keyword
1184         :returns: Nothing.
1185         """
1186         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1187                 and not self._version:
1188             self._msg_type = u"vpp-version"
1189         elif setup_kw.name.count(u"Set Global Variable") \
1190                 and not self._timestamp:
1191             self._msg_type = u"timestamp"
1192         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1193             self._msg_type = u"testbed"
1194         else:
1195             return
1196         setup_kw.messages.visit(self)
1197
1198     def end_setup_kw(self, setup_kw):
1199         """Called when keyword ends. Default implementation does nothing.
1200
1201         :param setup_kw: Keyword to process.
1202         :type setup_kw: Keyword
1203         :returns: Nothing.
1204         """
1205
1206     def visit_teardown_kw(self, teardown_kw):
1207         """Implements traversing through the teardown keyword and its child
1208         keywords.
1209
1210         :param teardown_kw: Keyword to process.
1211         :type teardown_kw: Keyword
1212         :returns: Nothing.
1213         """
1214         for keyword in teardown_kw.keywords:
1215             if self.start_teardown_kw(keyword) is not False:
1216                 self.visit_teardown_kw(keyword)
1217                 self.end_teardown_kw(keyword)
1218
1219     def start_teardown_kw(self, teardown_kw):
1220         """Called when teardown keyword starts
1221
1222         :param teardown_kw: Keyword to process.
1223         :type teardown_kw: Keyword
1224         :returns: Nothing.
1225         """
1226
1227         if teardown_kw.name.count(u"Show Vat History On All Duts"):
1228             # TODO: Remove when not needed:
1229             self._conf_history_lookup_nr = 0
1230             self._msg_type = u"teardown-vat-history"
1231             teardown_kw.messages.visit(self)
1232         elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1233             self._conf_history_lookup_nr = 0
1234             self._msg_type = u"teardown-papi-history"
1235             teardown_kw.messages.visit(self)
1236
1237     def end_teardown_kw(self, teardown_kw):
1238         """Called when keyword ends. Default implementation does nothing.
1239
1240         :param teardown_kw: Keyword to process.
1241         :type teardown_kw: Keyword
1242         :returns: Nothing.
1243         """
1244
1245     def visit_message(self, msg):
1246         """Implements visiting the message.
1247
1248         :param msg: Message to process.
1249         :type msg: Message
1250         :returns: Nothing.
1251         """
1252         if self.start_message(msg) is not False:
1253             self.end_message(msg)
1254
1255     def start_message(self, msg):
1256         """Called when message starts. Get required information from messages:
1257         - VPP version.
1258
1259         :param msg: Message to process.
1260         :type msg: Message
1261         :returns: Nothing.
1262         """
1263
1264         if self._msg_type:
1265             self.parse_msg[self._msg_type](msg)
1266
1267     def end_message(self, msg):
1268         """Called when message ends. Default implementation does nothing.
1269
1270         :param msg: Message to process.
1271         :type msg: Message
1272         :returns: Nothing.
1273         """
1274
1275
1276 class InputData:
1277     """Input data
1278
1279     The data is extracted from output.xml files generated by Jenkins jobs and
1280     stored in pandas' DataFrames.
1281
1282     The data structure:
1283     - job name
1284       - build number
1285         - metadata
1286           (as described in ExecutionChecker documentation)
1287         - suites
1288           (as described in ExecutionChecker documentation)
1289         - tests
1290           (as described in ExecutionChecker documentation)
1291     """
1292
1293     def __init__(self, spec):
1294         """Initialization.
1295
1296         :param spec: Specification.
1297         :type spec: Specification
1298         """
1299
1300         # Specification:
1301         self._cfg = spec
1302
1303         # Data store:
1304         self._input_data = pd.Series()
1305
1306     @property
1307     def data(self):
1308         """Getter - Input data.
1309
1310         :returns: Input data
1311         :rtype: pandas.Series
1312         """
1313         return self._input_data
1314
1315     def metadata(self, job, build):
1316         """Getter - metadata
1317
1318         :param job: Job which metadata we want.
1319         :param build: Build which metadata we want.
1320         :type job: str
1321         :type build: str
1322         :returns: Metadata
1323         :rtype: pandas.Series
1324         """
1325
1326         return self.data[job][build][u"metadata"]
1327
1328     def suites(self, job, build):
1329         """Getter - suites
1330
1331         :param job: Job which suites we want.
1332         :param build: Build which suites we want.
1333         :type job: str
1334         :type build: str
1335         :returns: Suites.
1336         :rtype: pandas.Series
1337         """
1338
1339         return self.data[job][str(build)][u"suites"]
1340
1341     def tests(self, job, build):
1342         """Getter - tests
1343
1344         :param job: Job which tests we want.
1345         :param build: Build which tests we want.
1346         :type job: str
1347         :type build: str
1348         :returns: Tests.
1349         :rtype: pandas.Series
1350         """
1351
1352         return self.data[job][build][u"tests"]
1353
1354     def _parse_tests(self, job, build, log):
1355         """Process data from robot output.xml file and return JSON structured
1356         data.
1357
1358         :param job: The name of job which build output data will be processed.
1359         :param build: The build which output data will be processed.
1360         :param log: List of log messages.
1361         :type job: str
1362         :type build: dict
1363         :type log: list of tuples (severity, msg)
1364         :returns: JSON data structure.
1365         :rtype: dict
1366         """
1367
1368         metadata = {
1369             u"job": job,
1370             u"build": build
1371         }
1372
1373         with open(build[u"file-name"], u'r') as data_file:
1374             try:
1375                 result = ExecutionResult(data_file)
1376             except errors.DataError as err:
1377                 log.append(
1378                     (u"ERROR", f"Error occurred while parsing output.xml: "
1379                                f"{repr(err)}")
1380                 )
1381                 return None
1382         checker = ExecutionChecker(metadata, self._cfg.mapping,
1383                                    self._cfg.ignore)
1384         result.visit(checker)
1385
1386         return checker.data
1387
1388     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1389         """Download and parse the input data file.
1390
1391         :param pid: PID of the process executing this method.
1392         :param job: Name of the Jenkins job which generated the processed input
1393             file.
1394         :param build: Information about the Jenkins build which generated the
1395             processed input file.
1396         :param repeat: Repeat the download specified number of times if not
1397             successful.
1398         :type pid: int
1399         :type job: str
1400         :type build: dict
1401         :type repeat: int
1402         """
1403
1404         logs = list()
1405
1406         logs.append(
1407             (u"INFO", f"  Processing the job/build: {job}: {build[u'build']}")
1408         )
1409
1410         state = u"failed"
1411         success = False
1412         data = None
1413         do_repeat = repeat
1414         while do_repeat:
1415             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1416                                                    logs)
1417             if success:
1418                 break
1419             do_repeat -= 1
1420         if not success:
1421             logs.append(
1422                 (u"ERROR",
1423                  f"It is not possible to download the input data file from the "
1424                  f"job {job}, build {build[u'build']}, or it is damaged. "
1425                  f"Skipped.")
1426             )
1427         if success:
1428             logs.append(
1429                 (u"INFO",
1430                  f"    Processing data from the build {build[u'build']} ...")
1431             )
1432             data = self._parse_tests(job, build, logs)
1433             if data is None:
1434                 logs.append(
1435                     (u"ERROR",
1436                      f"Input data file from the job {job}, build "
1437                      f"{build[u'build']} is damaged. Skipped.")
1438                 )
1439             else:
1440                 state = u"processed"
1441
1442             try:
1443                 remove(build[u"file-name"])
1444             except OSError as err:
1445                 logs.append(
1446                     ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
1447                               f"{repr(err)}")
1448                 )
1449
1450         # If the time-period is defined in the specification file, remove all
1451         # files which are outside the time period.
1452         timeperiod = self._cfg.input.get(u"time-period", None)
1453         if timeperiod and data:
1454             now = dt.utcnow()
1455             timeperiod = timedelta(int(timeperiod))
1456             metadata = data.get(u"metadata", None)
1457             if metadata:
1458                 generated = metadata.get(u"generated", None)
1459                 if generated:
1460                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1461                     if (now - generated) > timeperiod:
1462                         # Remove the data and the file:
1463                         state = u"removed"
1464                         data = None
1465                         logs.append(
1466                             (u"INFO",
1467                              f"    The build {job}/{build[u'build']} is "
1468                              f"outdated, will be removed.")
1469                         )
1470         logs.append((u"INFO", u"  Done."))
1471
1472         for level, line in logs:
1473             if level == u"INFO":
1474                 logging.info(line)
1475             elif level == u"ERROR":
1476                 logging.error(line)
1477             elif level == u"DEBUG":
1478                 logging.debug(line)
1479             elif level == u"CRITICAL":
1480                 logging.critical(line)
1481             elif level == u"WARNING":
1482                 logging.warning(line)
1483
1484         return {u"data": data, u"state": state, u"job": job, u"build": build}
1485
1486     def download_and_parse_data(self, repeat=1):
1487         """Download the input data files, parse input data from input files and
1488         store in pandas' Series.
1489
1490         :param repeat: Repeat the download specified number of times if not
1491             successful.
1492         :type repeat: int
1493         """
1494
1495         logging.info(u"Downloading and parsing input files ...")
1496
1497         for job, builds in self._cfg.builds.items():
1498             for build in builds:
1499
1500                 result = self._download_and_parse_build(job, build, repeat)
1501                 build_nr = result[u"build"][u"build"]
1502
1503                 if result[u"data"]:
1504                     data = result[u"data"]
1505                     build_data = pd.Series({
1506                         u"metadata": pd.Series(
1507                             list(data[u"metadata"].values()),
1508                             index=list(data[u"metadata"].keys())
1509                         ),
1510                         u"suites": pd.Series(
1511                             list(data[u"suites"].values()),
1512                             index=list(data[u"suites"].keys())
1513                         ),
1514                         u"tests": pd.Series(
1515                             list(data[u"tests"].values()),
1516                             index=list(data[u"tests"].keys())
1517                         )
1518                     })
1519
1520                     if self._input_data.get(job, None) is None:
1521                         self._input_data[job] = pd.Series()
1522                     self._input_data[job][str(build_nr)] = build_data
1523
1524                     self._cfg.set_input_file_name(
1525                         job, build_nr, result[u"build"][u"file-name"])
1526
1527                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1528
1529                 mem_alloc = \
1530                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1531                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1532
1533         logging.info(u"Done.")
1534
1535     @staticmethod
1536     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1537         """Return the index of character in the string which is the end of tag.
1538
1539         :param tag_filter: The string where the end of tag is being searched.
1540         :param start: The index where the searching is stated.
1541         :param closer: The character which is the tag closer.
1542         :type tag_filter: str
1543         :type start: int
1544         :type closer: str
1545         :returns: The index of the tag closer.
1546         :rtype: int
1547         """
1548
1549         try:
1550             idx_opener = tag_filter.index(closer, start)
1551             return tag_filter.index(closer, idx_opener + 1)
1552         except ValueError:
1553             return None
1554
1555     @staticmethod
1556     def _condition(tag_filter):
1557         """Create a conditional statement from the given tag filter.
1558
1559         :param tag_filter: Filter based on tags from the element specification.
1560         :type tag_filter: str
1561         :returns: Conditional statement which can be evaluated.
1562         :rtype: str
1563         """
1564
1565         index = 0
1566         while True:
1567             index = InputData._end_of_tag(tag_filter, index)
1568             if index is None:
1569                 return tag_filter
1570             index += 1
1571             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1572
1573     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1574                     continue_on_error=False):
1575         """Filter required data from the given jobs and builds.
1576
1577         The output data structure is:
1578
1579         - job 1
1580           - build 1
1581             - test (or suite) 1 ID:
1582               - param 1
1583               - param 2
1584               ...
1585               - param n
1586             ...
1587             - test (or suite) n ID:
1588             ...
1589           ...
1590           - build n
1591         ...
1592         - job n
1593
1594         :param element: Element which will use the filtered data.
1595         :param params: Parameters which will be included in the output. If None,
1596             all parameters are included.
1597         :param data: If not None, this data is used instead of data specified
1598             in the element.
1599         :param data_set: The set of data to be filtered: tests, suites,
1600             metadata.
1601         :param continue_on_error: Continue if there is error while reading the
1602             data. The Item will be empty then
1603         :type element: pandas.Series
1604         :type params: list
1605         :type data: dict
1606         :type data_set: str
1607         :type continue_on_error: bool
1608         :returns: Filtered data.
1609         :rtype pandas.Series
1610         """
1611
1612         try:
1613             if data_set == "suites":
1614                 cond = u"True"
1615             elif element[u"filter"] in (u"all", u"template"):
1616                 cond = u"True"
1617             else:
1618                 cond = InputData._condition(element[u"filter"])
1619             logging.debug(f"   Filter: {cond}")
1620         except KeyError:
1621             logging.error(u"  No filter defined.")
1622             return None
1623
1624         if params is None:
1625             params = element.get(u"parameters", None)
1626             if params:
1627                 params.append(u"type")
1628
1629         data_to_filter = data if data else element[u"data"]
1630         data = pd.Series()
1631         try:
1632             for job, builds in data_to_filter.items():
1633                 data[job] = pd.Series()
1634                 for build in builds:
1635                     data[job][str(build)] = pd.Series()
1636                     try:
1637                         data_dict = dict(
1638                             self.data[job][str(build)][data_set].items())
1639                     except KeyError:
1640                         if continue_on_error:
1641                             continue
1642                         return None
1643
1644                     for test_id, test_data in data_dict.items():
1645                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1646                             data[job][str(build)][test_id] = pd.Series()
1647                             if params is None:
1648                                 for param, val in test_data.items():
1649                                     data[job][str(build)][test_id][param] = val
1650                             else:
1651                                 for param in params:
1652                                     try:
1653                                         data[job][str(build)][test_id][param] =\
1654                                             test_data[param]
1655                                     except KeyError:
1656                                         data[job][str(build)][test_id][param] =\
1657                                             u"No Data"
1658             return data
1659
1660         except (KeyError, IndexError, ValueError) as err:
1661             logging.error(
1662                 f"Missing mandatory parameter in the element specification: "
1663                 f"{repr(err)}"
1664             )
1665             return None
1666         except AttributeError as err:
1667             logging.error(repr(err))
1668             return None
1669         except SyntaxError as err:
1670             logging.error(
1671                 f"The filter {cond} is not correct. Check if all tags are "
1672                 f"enclosed by apostrophes.\n{repr(err)}"
1673             )
1674             return None
1675
1676     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1677                              continue_on_error=False):
1678         """Filter required data from the given jobs and builds.
1679
1680         The output data structure is:
1681
1682         - job 1
1683           - build 1
1684             - test (or suite) 1 ID:
1685               - param 1
1686               - param 2
1687               ...
1688               - param n
1689             ...
1690             - test (or suite) n ID:
1691             ...
1692           ...
1693           - build n
1694         ...
1695         - job n
1696
1697         :param element: Element which will use the filtered data.
1698         :param params: Parameters which will be included in the output. If None,
1699         all parameters are included.
1700         :param data_set: The set of data to be filtered: tests, suites,
1701         metadata.
1702         :param continue_on_error: Continue if there is error while reading the
1703         data. The Item will be empty then
1704         :type element: pandas.Series
1705         :type params: list
1706         :type data_set: str
1707         :type continue_on_error: bool
1708         :returns: Filtered data.
1709         :rtype pandas.Series
1710         """
1711
1712         include = element.get(u"include", None)
1713         if not include:
1714             logging.warning(u"No tests to include, skipping the element.")
1715             return None
1716
1717         if params is None:
1718             params = element.get(u"parameters", None)
1719             if params:
1720                 params.append(u"type")
1721
1722         data = pd.Series()
1723         try:
1724             for job, builds in element[u"data"].items():
1725                 data[job] = pd.Series()
1726                 for build in builds:
1727                     data[job][str(build)] = pd.Series()
1728                     for test in include:
1729                         try:
1730                             reg_ex = re.compile(str(test).lower())
1731                             for test_id in self.data[job][
1732                                     str(build)][data_set].keys():
1733                                 if re.match(reg_ex, str(test_id).lower()):
1734                                     test_data = self.data[job][
1735                                         str(build)][data_set][test_id]
1736                                     data[job][str(build)][test_id] = pd.Series()
1737                                     if params is None:
1738                                         for param, val in test_data.items():
1739                                             data[job][str(build)][test_id]\
1740                                                 [param] = val
1741                                     else:
1742                                         for param in params:
1743                                             try:
1744                                                 data[job][str(build)][
1745                                                     test_id][param] = \
1746                                                     test_data[param]
1747                                             except KeyError:
1748                                                 data[job][str(build)][
1749                                                     test_id][param] = u"No Data"
1750                         except KeyError as err:
1751                             logging.error(repr(err))
1752                             if continue_on_error:
1753                                 continue
1754                             return None
1755             return data
1756
1757         except (KeyError, IndexError, ValueError) as err:
1758             logging.error(
1759                 f"Missing mandatory parameter in the element "
1760                 f"specification: {repr(err)}"
1761             )
1762             return None
1763         except AttributeError as err:
1764             logging.error(repr(err))
1765             return None
1766
1767     @staticmethod
1768     def merge_data(data):
1769         """Merge data from more jobs and builds to a simple data structure.
1770
1771         The output data structure is:
1772
1773         - test (suite) 1 ID:
1774           - param 1
1775           - param 2
1776           ...
1777           - param n
1778         ...
1779         - test (suite) n ID:
1780         ...
1781
1782         :param data: Data to merge.
1783         :type data: pandas.Series
1784         :returns: Merged data.
1785         :rtype: pandas.Series
1786         """
1787
1788         logging.info(u"    Merging data ...")
1789
1790         merged_data = pd.Series()
1791         for builds in data.values:
1792             for item in builds.values:
1793                 for item_id, item_data in item.items():
1794                     merged_data[item_id] = item_data
1795
1796         return merged_data
1797
1798     def print_all_oper_data(self):
1799         """Print all operational data to console.
1800         """
1801
1802         tbl_hdr = (
1803             u"Name",
1804             u"Nr of Vectors",
1805             u"Nr of Packets",
1806             u"Suspends",
1807             u"Cycles per Packet",
1808             u"Average Vector Size"
1809         )
1810
1811         for job in self._input_data.values:
1812             for build in job.values:
1813                 for test_id, test_data in build[u"tests"].items():
1814                     print(f"{test_id}")
1815                     if test_data.get(u"show-run", None) is None:
1816                         continue
1817                     for dut_name, data in test_data[u"show-run"].items():
1818                         if data.get(u"threads", None) is None:
1819                             continue
1820                         print(f"Host IP: {data.get(u'host', '')}, "
1821                               f"Socket: {data.get(u'socket', '')}")
1822                         for thread_nr, thread in data[u"threads"].items():
1823                             txt_table = prettytable.PrettyTable(tbl_hdr)
1824                             avg = 0.0
1825                             for row in thread:
1826                                 txt_table.add_row(row)
1827                                 avg += row[-1]
1828                             if len(thread) == 0:
1829                                 avg = u""
1830                             else:
1831                                 avg = f", Average Vector Size per Node: " \
1832                                       f"{(avg / len(thread)):.2f}"
1833                             th_name = u"main" if thread_nr == 0 \
1834                                 else f"worker_{thread_nr}"
1835                             print(f"{dut_name}, {th_name}{avg}")
1836                             txt_table.float_format = u".2"
1837                             txt_table.align = u"r"
1838                             txt_table.align[u"Name"] = u"l"
1839                             print(f"{txt_table.get_string()}\n")