resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import copy
  24 import resource
  25 import logging
  26
  27 from collections import OrderedDict
  28 from os import remove
  29 from datetime import datetime as dt
  30 from datetime import timedelta
  31 from json import loads
  32
  33 import prettytable
  34 import pandas as pd
  35
  36 from robot.api import ExecutionResult, ResultVisitor
  37 from robot import errors
  38
  39 from resources.libraries.python import jumpavg
  40 from input_data_files import download_and_unzip_data_file
  41
  42
  43 # Separator used in file names
  44 SEPARATOR = u"__"
  45
  46
  47 class ExecutionChecker(ResultVisitor):
  48     """Class to traverse through the test suite structure.
  49
  50     The functionality implemented in this class generates a json structure:
  51
  52     Performance tests:
  53
  54     {
  55         "metadata": {
  56             "generated": "Timestamp",
  57             "version": "SUT version",
  58             "job": "Jenkins job name",
  59             "build": "Information about the build"
  60         },
  61         "suites": {
  62             "Suite long name 1": {
  63                 "name": Suite name,
  64                 "doc": "Suite 1 documentation",
  65                 "parent": "Suite 1 parent",
  66                 "level": "Level of the suite in the suite hierarchy"
  67             }
  68             "Suite long name N": {
  69                 "name": Suite name,
  70                 "doc": "Suite N documentation",
  71                 "parent": "Suite 2 parent",
  72                 "level": "Level of the suite in the suite hierarchy"
  73             }
  74         }
  75         "tests": {
  76             # NDRPDR tests:
  77             "ID": {
  78                 "name": "Test name",
  79                 "parent": "Name of the parent of the test",
  80                 "doc": "Test documentation",
  81                 "msg": "Test message",
  82                 "conf-history": "DUT1 and DUT2 VAT History",
  83                 "show-run": "Show Run",
  84                 "tags": ["tag 1", "tag 2", "tag n"],
  85                 "type": "NDRPDR",
  86                 "status": "PASS" | "FAIL",
  87                 "throughput": {
  88                     "NDR": {
  89                         "LOWER": float,
  90                         "UPPER": float
  91                     },
  92                     "PDR": {
  93                         "LOWER": float,
  94                         "UPPER": float
  95                     }
  96                 },
  97                 "latency": {
  98                     "NDR": {
  99                         "direction1": {
 100                             "min": float,
 101                             "avg": float,
 102                             "max": float,
 103                             "hdrh": str
 104                         },
 105                         "direction2": {
 106                             "min": float,
 107                             "avg": float,
 108                             "max": float,
 109                             "hdrh": str
 110                         }
 111                     },
 112                     "PDR": {
 113                         "direction1": {
 114                             "min": float,
 115                             "avg": float,
 116                             "max": float,
 117                             "hdrh": str
 118                         },
 119                         "direction2": {
 120                             "min": float,
 121                             "avg": float,
 122                             "max": float,
 123                             "hdrh": str
 124                         }
 125                     }
 126                 }
 127             }
 128
 129             # TCP tests:
 130             "ID": {
 131                 "name": "Test name",
 132                 "parent": "Name of the parent of the test",
 133                 "doc": "Test documentation",
 134                 "msg": "Test message",
 135                 "tags": ["tag 1", "tag 2", "tag n"],
 136                 "type": "TCP",
 137                 "status": "PASS" | "FAIL",
 138                 "result": int
 139             }
 140
 141             # MRR, BMRR tests:
 142             "ID": {
 143                 "name": "Test name",
 144                 "parent": "Name of the parent of the test",
 145                 "doc": "Test documentation",
 146                 "msg": "Test message",
 147                 "tags": ["tag 1", "tag 2", "tag n"],
 148                 "type": "MRR" | "BMRR",
 149                 "status": "PASS" | "FAIL",
 150                 "result": {
 151                     "receive-rate": float,
 152                     # Average of a list, computed using AvgStdevStats.
 153                     # In CSIT-1180, replace with List[float].
 154                 }
 155             }
 156
 157             "ID" {
 158                 # next test
 159             }
 160         }
 161     }
 162
 163
 164     Functional tests:
 165
 166     {
 167         "metadata": {  # Optional
 168             "version": "VPP version",
 169             "job": "Jenkins job name",
 170             "build": "Information about the build"
 171         },
 172         "suites": {
 173             "Suite name 1": {
 174                 "doc": "Suite 1 documentation",
 175                 "parent": "Suite 1 parent",
 176                 "level": "Level of the suite in the suite hierarchy"
 177             }
 178             "Suite name N": {
 179                 "doc": "Suite N documentation",
 180                 "parent": "Suite 2 parent",
 181                 "level": "Level of the suite in the suite hierarchy"
 182             }
 183         }
 184         "tests": {
 185             "ID": {
 186                 "name": "Test name",
 187                 "parent": "Name of the parent of the test",
 188                 "doc": "Test documentation"
 189                 "msg": "Test message"
 190                 "tags": ["tag 1", "tag 2", "tag n"],
 191                 "conf-history": "DUT1 and DUT2 VAT History"
 192                 "show-run": "Show Run"
 193                 "status": "PASS" | "FAIL"
 194             },
 195             "ID" {
 196                 # next test
 197             }
 198         }
 199     }
 200
 201     .. note:: ID is the lowercase full path to the test.
 202     """
 203
 204     REGEX_PLR_RATE = re.compile(r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 205                                 r'PLRsearch upper bound::?\s(\d+.\d+)')
 206
 207     REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 208                                    r'NDR_UPPER:\s(\d+.\d+).*\n'
 209                                    r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 210                                    r'PDR_UPPER:\s(\d+.\d+)')
 211
 212     REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 213                                   r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
 214
 215     REGEX_NDRPDR_LAT_LONG = re.compile(
 216         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 217         r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n'
 218         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 219         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 220         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 221         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 222         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 223         r'Latency.*\[\'(.*)\', \'(.*)\'\]\s\n'
 224         r'Latency.*\[\'(.*)\', \'(.*)\'\]'
 225     )
 226
 227     REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
 228                                  r'[\D\d]*')
 229
 230     REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|"
 231                                    r"VPP Version:\s*|VPP version:\s*)(.*)")
 232
 233     REGEX_VERSION_DPDK = re.compile(r"(DPDK version:\s*|DPDK Version:\s*)(.*)")
 234
 235     REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s(\d*).*$')
 236
 237     REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 238                            r'tx\s(\d*),\srx\s(\d*)')
 239
 240     REGEX_BMRR = re.compile(r'Maximum Receive Rate trial results'
 241                             r' in packets per second: \[(.*)\]')
 242
 243     REGEX_RECONF_LOSS = re.compile(r'Packets lost due to reconfig: (\d*)')
 244     REGEX_RECONF_TIME = re.compile(r'Implied time lost: (\d*.[\de-]*)')
 245
 246     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 247
 248     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 249
 250     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 251
 252     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 253
 254     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 255
 256     def __init__(self, metadata, mapping, ignore):
 257         """Initialisation.
 258
 259         :param metadata: Key-value pairs to be included in "metadata" part of
 260             JSON structure.
 261         :param mapping: Mapping of the old names of test cases to the new
 262             (actual) one.
 263         :param ignore: List of TCs to be ignored.
 264         :type metadata: dict
 265         :type mapping: dict
 266         :type ignore: list
 267         """
 268
 269         # Type of message to parse out from the test messages
 270         self._msg_type = None
 271
 272         # VPP version
 273         self._version = None
 274
 275         # Timestamp
 276         self._timestamp = None
 277
 278         # Testbed. The testbed is identified by TG node IP address.
 279         self._testbed = None
 280
 281         # Mapping of TCs long names
 282         self._mapping = mapping
 283
 284         # Ignore list
 285         self._ignore = ignore
 286
 287         # Number of PAPI History messages found:
 288         # 0 - no message
 289         # 1 - PAPI History of DUT1
 290         # 2 - PAPI History of DUT2
 291         self._lookup_kw_nr = 0
 292         self._conf_history_lookup_nr = 0
 293
 294         # Number of Show Running messages found
 295         # 0 - no message
 296         # 1 - Show run message found
 297         self._show_run_lookup_nr = 0
 298
 299         # Test ID of currently processed test- the lowercase full path to the
 300         # test
 301         self._test_id = None
 302
 303         # The main data structure
 304         self._data = {
 305             u"metadata": OrderedDict(),
 306             u"suites": OrderedDict(),
 307             u"tests": OrderedDict()
 308         }
 309
 310         # Save the provided metadata
 311         for key, val in metadata.items():
 312             self._data[u"metadata"][key] = val
 313
 314         # Dictionary defining the methods used to parse different types of
 315         # messages
 316         self.parse_msg = {
 317             u"timestamp": self._get_timestamp,
 318             u"vpp-version": self._get_vpp_version,
 319             u"dpdk-version": self._get_dpdk_version,
 320             # TODO: Remove when not needed:
 321             u"teardown-vat-history": self._get_vat_history,
 322             u"teardown-papi-history": self._get_papi_history,
 323             u"test-show-runtime": self._get_show_run,
 324             u"testbed": self._get_testbed
 325         }
 326
 327     @property
 328     def data(self):
 329         """Getter - Data parsed from the XML file.
 330
 331         :returns: Data parsed from the XML file.
 332         :rtype: dict
 333         """
 334         return self._data
 335
 336     def _get_testbed(self, msg):
 337         """Called when extraction of testbed IP is required.
 338         The testbed is identified by TG node IP address.
 339
 340         :param msg: Message to process.
 341         :type msg: Message
 342         :returns: Nothing.
 343         """
 344
 345         if msg.message.count(u"Setup of TG node") or \
 346                 msg.message.count(u"Setup of node TG host"):
 347             reg_tg_ip = re.compile(
 348                 r'.*TG .* (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}).*')
 349             try:
 350                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 351             except (KeyError, ValueError, IndexError, AttributeError):
 352                 pass
 353             finally:
 354                 self._data[u"metadata"][u"testbed"] = self._testbed
 355                 self._msg_type = None
 356
 357     def _get_vpp_version(self, msg):
 358         """Called when extraction of VPP version is required.
 359
 360         :param msg: Message to process.
 361         :type msg: Message
 362         :returns: Nothing.
 363         """
 364
 365         if msg.message.count(u"return STDOUT Version:") or \
 366             msg.message.count(u"VPP Version:") or \
 367             msg.message.count(u"VPP version:"):
 368             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 369                                 group(2))
 370             self._data[u"metadata"][u"version"] = self._version
 371             self._msg_type = None
 372
 373     def _get_dpdk_version(self, msg):
 374         """Called when extraction of DPDK version is required.
 375
 376         :param msg: Message to process.
 377         :type msg: Message
 378         :returns: Nothing.
 379         """
 380
 381         if msg.message.count(u"DPDK Version:"):
 382             try:
 383                 self._version = str(re.search(
 384                     self.REGEX_VERSION_DPDK, msg.message).group(2))
 385                 self._data[u"metadata"][u"version"] = self._version
 386             except IndexError:
 387                 pass
 388             finally:
 389                 self._msg_type = None
 390
 391     def _get_timestamp(self, msg):
 392         """Called when extraction of timestamp is required.
 393
 394         :param msg: Message to process.
 395         :type msg: Message
 396         :returns: Nothing.
 397         """
 398
 399         self._timestamp = msg.timestamp[:14]
 400         self._data[u"metadata"][u"generated"] = self._timestamp
 401         self._msg_type = None
 402
 403     def _get_vat_history(self, msg):
 404         """Called when extraction of VAT command history is required.
 405
 406         TODO: Remove when not needed.
 407
 408         :param msg: Message to process.
 409         :type msg: Message
 410         :returns: Nothing.
 411         """
 412         if msg.message.count(u"VAT command history:"):
 413             self._conf_history_lookup_nr += 1
 414             if self._conf_history_lookup_nr == 1:
 415                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 416             else:
 417                 self._msg_type = None
 418             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 419                           r"VAT command history:", u"",
 420                           msg.message, count=1).replace(u'\n', u' |br| ').\
 421                 replace(u'"', u"'")
 422
 423             self._data[u"tests"][self._test_id][u"conf-history"] += (
 424                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 425             )
 426
 427     def _get_papi_history(self, msg):
 428         """Called when extraction of PAPI command history is required.
 429
 430         :param msg: Message to process.
 431         :type msg: Message
 432         :returns: Nothing.
 433         """
 434         if msg.message.count(u"PAPI command history:"):
 435             self._conf_history_lookup_nr += 1
 436             if self._conf_history_lookup_nr == 1:
 437                 self._data[u"tests"][self._test_id][u"conf-history"] = str()
 438             else:
 439                 self._msg_type = None
 440             text = re.sub(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 441                           r"PAPI command history:", u"",
 442                           msg.message, count=1).replace(u'\n', u' |br| ').\
 443                 replace(u'"', u"'")
 444             self._data[u"tests"][self._test_id][u"conf-history"] += (
 445                 f" |br| **DUT{str(self._conf_history_lookup_nr)}:** {text}"
 446             )
 447
 448     def _get_show_run(self, msg):
 449         """Called when extraction of VPP operational data (output of CLI command
 450         Show Runtime) is required.
 451
 452         :param msg: Message to process.
 453         :type msg: Message
 454         :returns: Nothing.
 455         """
 456
 457         if u"show-run" not in self._data[u"tests"][self._test_id].keys():
 458             self._data[u"tests"][self._test_id][u"show-run"] = str()
 459
 460         if msg.message.count(u"stats runtime") or \
 461                 msg.message.count(u"Runtime"):
 462             try:
 463                 host = str(re.search(self.REGEX_TC_PAPI_CLI, msg.message).
 464                            group(1))
 465             except (AttributeError, IndexError):
 466                 host = self._data[u"tests"][self._test_id][u"show-run"].\
 467                            count(u"DUT:") + 1
 468             try:
 469                 socket = str(re.search(self.REGEX_TC_PAPI_CLI, msg.message).
 470                              group(2))
 471                 socket = f"/{socket}"
 472             except (AttributeError, IndexError):
 473                 socket = u""
 474             runtime = loads(
 475                 str(msg.message).
 476                 replace(u' ', u'').
 477                 replace(u'\n', u'').
 478                 replace(u"'", u'"').
 479                 replace(u'b"', u'"').
 480                 replace(u'u"', u'"').
 481                 split(u":", 1)[1]
 482             )
 483             try:
 484                 threads_nr = len(runtime[0][u"clocks"])
 485             except (IndexError, KeyError):
 486                 return
 487             tbl_hdr = [
 488                 u"Name",
 489                 u"Calls",
 490                 u"Vectors",
 491                 u"Suspends",
 492                 u"Clocks",
 493                 u"Vectors/Calls"
 494             ]
 495             table = [[tbl_hdr, ] for _ in range(threads_nr)]
 496             for item in runtime:
 497                 for idx in range(threads_nr):
 498                     name = format(item[u"name"])
 499                     calls = format(item[u"calls"][idx])
 500                     vectors = format(item[u"vectors"][idx])
 501                     suspends = format(item[u"suspends"][idx])
 502                     if item[u"vectors"][idx] > 0:
 503                         clocks = format(
 504                             item[u"clocks"][idx]/item[u"vectors"][idx], u".2e")
 505                     elif item[u"calls"][idx] > 0:
 506                         clocks = format(
 507                             item[u"clocks"][idx]/item[u"calls"][idx], u".2e")
 508                     elif item[u"suspends"][idx] > 0:
 509                         clocks = format(
 510                             item[u"clocks"][idx]/item[u"suspends"][idx], u".2e")
 511                     else:
 512                         clocks = 0
 513                     if item[u"calls"][idx] > 0:
 514                         vectors_call = format(
 515                             item[u"vectors"][idx]/item[u"calls"][idx], u".2f")
 516                     else:
 517                         vectors_call = format(0, u".2f")
 518                     if int(calls) + int(vectors) + int(suspends):
 519                         table[idx].append([
 520                             name, calls, vectors, suspends, clocks, vectors_call
 521                         ])
 522             text = ""
 523             for idx in range(threads_nr):
 524                 text += f"Thread {idx} "
 525                 text += u"vpp_main\n" if idx == 0 else f"vpp_wk_{idx-1}\n"
 526                 txt_table = None
 527                 for row in table[idx]:
 528                     if txt_table is None:
 529                         txt_table = prettytable.PrettyTable(row)
 530                     else:
 531                         if any(row[1:]):
 532                             txt_table.add_row(row)
 533                 txt_table.set_style(prettytable.MSWORD_FRIENDLY)
 534                 txt_table.align[u"Name"] = u"l"
 535                 txt_table.align[u"Calls"] = u"r"
 536                 txt_table.align[u"Vectors"] = u"r"
 537                 txt_table.align[u"Suspends"] = u"r"
 538                 txt_table.align[u"Clocks"] = u"r"
 539                 txt_table.align[u"Vectors/Calls"] = u"r"
 540
 541                 text += txt_table.get_string(sortby=u"Name") + u'\n'
 542             text = f"\n**DUT: {host}{socket}**\n{text}".\
 543                 replace(u'\n', u' |br| ').\
 544                 replace(u'\r', u'').\
 545                 replace(u'"', u"'")
 546             self._data[u"tests"][self._test_id][u"show-run"] += text
 547
 548     def _get_ndrpdr_throughput(self, msg):
 549         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 550         message.
 551
 552         :param msg: The test message to be parsed.
 553         :type msg: str
 554         :returns: Parsed data as a dict and the status (PASS/FAIL).
 555         :rtype: tuple(dict, str)
 556         """
 557
 558         throughput = {
 559             u"NDR": {u"LOWER": -1.0, u"UPPER": -1.0},
 560             u"PDR": {u"LOWER": -1.0, u"UPPER": -1.0}
 561         }
 562         status = u"FAIL"
 563         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 564
 565         if groups is not None:
 566             try:
 567                 throughput[u"NDR"][u"LOWER"] = float(groups.group(1))
 568                 throughput[u"NDR"][u"UPPER"] = float(groups.group(2))
 569                 throughput[u"PDR"][u"LOWER"] = float(groups.group(3))
 570                 throughput[u"PDR"][u"UPPER"] = float(groups.group(4))
 571                 status = u"PASS"
 572             except (IndexError, ValueError):
 573                 pass
 574
 575         return throughput, status
 576
 577     def _get_plr_throughput(self, msg):
 578         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 579         message.
 580
 581         :param msg: The test message to be parsed.
 582         :type msg: str
 583         :returns: Parsed data as a dict and the status (PASS/FAIL).
 584         :rtype: tuple(dict, str)
 585         """
 586
 587         throughput = {
 588             u"LOWER": -1.0,
 589             u"UPPER": -1.0
 590         }
 591         status = u"FAIL"
 592         groups = re.search(self.REGEX_PLR_RATE, msg)
 593
 594         if groups is not None:
 595             try:
 596                 throughput[u"LOWER"] = float(groups.group(1))
 597                 throughput[u"UPPER"] = float(groups.group(2))
 598                 status = u"PASS"
 599             except (IndexError, ValueError):
 600                 pass
 601
 602         return throughput, status
 603
 604     def _get_ndrpdr_latency(self, msg):
 605         """Get LATENCY from the test message.
 606
 607         :param msg: The test message to be parsed.
 608         :type msg: str
 609         :returns: Parsed data as a dict and the status (PASS/FAIL).
 610         :rtype: tuple(dict, str)
 611         """
 612         latency_default = {
 613             u"min": -1.0,
 614             u"avg": -1.0,
 615             u"max": -1.0,
 616             u"hdrh": u""
 617         }
 618         latency = {
 619             u"NDR": {
 620                 u"direction1": copy.copy(latency_default),
 621                 u"direction2": copy.copy(latency_default)
 622             },
 623             u"PDR": {
 624                 u"direction1": copy.copy(latency_default),
 625                 u"direction2": copy.copy(latency_default)
 626             },
 627             u"LAT0": {
 628                 u"direction1": copy.copy(latency_default),
 629                 u"direction2": copy.copy(latency_default)
 630             },
 631             u"NDR10": {
 632                 u"direction1": copy.copy(latency_default),
 633                 u"direction2": copy.copy(latency_default)
 634             },
 635             u"NDR50": {
 636                 u"direction1": copy.copy(latency_default),
 637                 u"direction2": copy.copy(latency_default)
 638             },
 639             u"NDR90": {
 640                 u"direction1": copy.copy(latency_default),
 641                 u"direction2": copy.copy(latency_default)
 642             },
 643             u"PDR10": {
 644                 u"direction1": copy.copy(latency_default),
 645                 u"direction2": copy.copy(latency_default)
 646             },
 647             u"PDR50": {
 648                 u"direction1": copy.copy(latency_default),
 649                 u"direction2": copy.copy(latency_default)
 650             },
 651             u"PDR90": {
 652                 u"direction1": copy.copy(latency_default),
 653                 u"direction2": copy.copy(latency_default)
 654             },
 655         }
 656
 657         groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 658         if groups is None:
 659             groups = re.search(self.REGEX_NDRPDR_LAT_LONG, msg)
 660
 661         if groups is None:
 662             return latency, u"FAIL"
 663
 664         def process_latency(in_str):
 665             """Return object with parsed latency values.
 666
 667             TODO: Define class for the return type.
 668
 669             :param in_str: Input string, min/avg/max/hdrh format.
 670             :type in_str: str
 671             :returns: Dict with corresponding keys, except hdrh float values.
 672             :rtype dict:
 673             :throws IndexError: If in_str does not have enough substrings.
 674             :throws ValueError: If a substring does not convert to float.
 675             """
 676             in_list = in_str.split('/', 3)
 677
 678             rval = {
 679                 u"min": float(in_list[0]),
 680                 u"avg": float(in_list[1]),
 681                 u"max": float(in_list[2]),
 682                 u"hdrh": u""
 683             }
 684
 685             if len(in_list) == 4:
 686                 rval[u"hdrh"] = str(in_list[3])
 687
 688             return rval
 689
 690         try:
 691             latency[u"NDR"][u"direction1"] = process_latency(groups.group(1))
 692             latency[u"NDR"][u"direction2"] = process_latency(groups.group(2))
 693             latency[u"PDR"][u"direction1"] = process_latency(groups.group(3))
 694             latency[u"PDR"][u"direction2"] = process_latency(groups.group(4))
 695             if groups.lastindex == 4:
 696                 return latency, u"PASS"
 697         except (IndexError, ValueError):
 698             pass
 699
 700         try:
 701             latency[u"LAT0"][u"direction1"] = process_latency(groups.group(5))
 702             latency[u"LAT0"][u"direction2"] = process_latency(groups.group(6))
 703             latency[u"NDR10"][u"direction1"] = process_latency(groups.group(7))
 704             latency[u"NDR10"][u"direction2"] = process_latency(groups.group(8))
 705             latency[u"NDR50"][u"direction1"] = process_latency(groups.group(9))
 706             latency[u"NDR50"][u"direction2"] = process_latency(groups.group(10))
 707             latency[u"NDR90"][u"direction1"] = process_latency(groups.group(11))
 708             latency[u"NDR90"][u"direction2"] = process_latency(groups.group(12))
 709             latency[u"PDR10"][u"direction1"] = process_latency(groups.group(13))
 710             latency[u"PDR10"][u"direction2"] = process_latency(groups.group(14))
 711             latency[u"PDR50"][u"direction1"] = process_latency(groups.group(15))
 712             latency[u"PDR50"][u"direction2"] = process_latency(groups.group(16))
 713             latency[u"PDR90"][u"direction1"] = process_latency(groups.group(17))
 714             latency[u"PDR90"][u"direction2"] = process_latency(groups.group(18))
 715             return latency, u"PASS"
 716         except (IndexError, ValueError):
 717             pass
 718
 719         return latency, u"FAIL"
 720
 721     def visit_suite(self, suite):
 722         """Implements traversing through the suite and its direct children.
 723
 724         :param suite: Suite to process.
 725         :type suite: Suite
 726         :returns: Nothing.
 727         """
 728         if self.start_suite(suite) is not False:
 729             suite.suites.visit(self)
 730             suite.tests.visit(self)
 731             self.end_suite(suite)
 732
 733     def start_suite(self, suite):
 734         """Called when suite starts.
 735
 736         :param suite: Suite to process.
 737         :type suite: Suite
 738         :returns: Nothing.
 739         """
 740
 741         try:
 742             parent_name = suite.parent.name
 743         except AttributeError:
 744             return
 745
 746         doc_str = suite.doc.\
 747             replace(u'"', u"'").\
 748             replace(u'\n', u' ').\
 749             replace(u'\r', u'').\
 750             replace(u'*[', u' |br| *[').\
 751             replace(u"*", u"**").\
 752             replace(u' |br| *[', u'*[', 1)
 753
 754         self._data[u"suites"][suite.longname.lower().
 755                               replace(u'"', u"'").
 756                               replace(u" ", u"_")] = {
 757                                   u"name": suite.name.lower(),
 758                                   u"doc": doc_str,
 759                                   u"parent": parent_name,
 760                                   u"level": len(suite.longname.split(u"."))
 761                               }
 762
 763         suite.keywords.visit(self)
 764
 765     def end_suite(self, suite):
 766         """Called when suite ends.
 767
 768         :param suite: Suite to process.
 769         :type suite: Suite
 770         :returns: Nothing.
 771         """
 772
 773     def visit_test(self, test):
 774         """Implements traversing through the test.
 775
 776         :param test: Test to process.
 777         :type test: Test
 778         :returns: Nothing.
 779         """
 780         if self.start_test(test) is not False:
 781             test.keywords.visit(self)
 782             self.end_test(test)
 783
 784     def start_test(self, test):
 785         """Called when test starts.
 786
 787         :param test: Test to process.
 788         :type test: Test
 789         :returns: Nothing.
 790         """
 791
 792         longname_orig = test.longname.lower()
 793
 794         # Check the ignore list
 795         if longname_orig in self._ignore:
 796             return
 797
 798         tags = [str(tag) for tag in test.tags]
 799         test_result = dict()
 800
 801         # Change the TC long name and name if defined in the mapping table
 802         longname = self._mapping.get(longname_orig, None)
 803         if longname is not None:
 804             name = longname.split(u'.')[-1]
 805             logging.debug(
 806                 f"{self._data[u'metadata']}\n{longname_orig}\n{longname}\n"
 807                 f"{name}"
 808             )
 809         else:
 810             longname = longname_orig
 811             name = test.name.lower()
 812
 813         # Remove TC number from the TC long name (backward compatibility):
 814         self._test_id = re.sub(self.REGEX_TC_NUMBER, u"", longname)
 815         # Remove TC number from the TC name (not needed):
 816         test_result[u"name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
 817
 818         test_result[u"parent"] = test.parent.name.lower()
 819         test_result[u"tags"] = tags
 820         test_result["doc"] = test.doc.\
 821             replace(u'"', u"'").\
 822             replace(u'\n', u' ').\
 823             replace(u'\r', u'').\
 824             replace(u'[', u' |br| [').\
 825             replace(u' |br| [', u'[', 1)
 826         test_result[u"msg"] = test.message.\
 827             replace(u'\n', u' |br| ').\
 828             replace(u'\r', u'').\
 829             replace(u'"', u"'")
 830         test_result[u"type"] = u"FUNC"
 831         test_result[u"status"] = test.status
 832
 833         if u"PERFTEST" in tags:
 834             # Replace info about cores (e.g. -1c-) with the info about threads
 835             # and cores (e.g. -1t1c-) in the long test case names and in the
 836             # test case names if necessary.
 837             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_id)
 838             if not groups:
 839                 tag_count = 0
 840                 tag_tc = str()
 841                 for tag in test_result[u"tags"]:
 842                     groups = re.search(self.REGEX_TC_TAG, tag)
 843                     if groups:
 844                         tag_count += 1
 845                         tag_tc = tag
 846
 847                 if tag_count == 1:
 848                     self._test_id = re.sub(self.REGEX_TC_NAME_NEW,
 849                                            f"-{tag_tc.lower()}-",
 850                                            self._test_id,
 851                                            count=1)
 852                     test_result[u"name"] = re.sub(self.REGEX_TC_NAME_NEW,
 853                                                   f"-{tag_tc.lower()}-",
 854                                                   test_result["name"],
 855                                                   count=1)
 856                 else:
 857                     test_result[u"status"] = u"FAIL"
 858                     self._data[u"tests"][self._test_id] = test_result
 859                     logging.debug(
 860                         f"The test {self._test_id} has no or more than one "
 861                         f"multi-threading tags.\n"
 862                         f"Tags: {test_result[u'tags']}"
 863                     )
 864                     return
 865
 866         if test.status == u"PASS":
 867             if u"NDRPDR" in tags:
 868                 test_result[u"type"] = u"NDRPDR"
 869                 test_result[u"throughput"], test_result[u"status"] = \
 870                     self._get_ndrpdr_throughput(test.message)
 871                 test_result[u"latency"], test_result[u"status"] = \
 872                     self._get_ndrpdr_latency(test.message)
 873             elif u"SOAK" in tags:
 874                 test_result[u"type"] = u"SOAK"
 875                 test_result[u"throughput"], test_result[u"status"] = \
 876                     self._get_plr_throughput(test.message)
 877             elif u"TCP" in tags:
 878                 test_result[u"type"] = u"TCP"
 879                 groups = re.search(self.REGEX_TCP, test.message)
 880                 test_result[u"result"] = int(groups.group(2))
 881             elif u"MRR" in tags or u"FRMOBL" in tags or u"BMRR" in tags:
 882                 if u"MRR" in tags:
 883                     test_result[u"type"] = u"MRR"
 884                 else:
 885                     test_result[u"type"] = u"BMRR"
 886
 887                 test_result[u"result"] = dict()
 888                 groups = re.search(self.REGEX_BMRR, test.message)
 889                 if groups is not None:
 890                     items_str = groups.group(1)
 891                     items_float = [float(item.strip()) for item
 892                                    in items_str.split(",")]
 893                     # Use whole list in CSIT-1180.
 894                     stats = jumpavg.AvgStdevStats.for_runs(items_float)
 895                     test_result[u"result"][u"receive-rate"] = stats.avg
 896                 else:
 897                     groups = re.search(self.REGEX_MRR, test.message)
 898                     test_result[u"result"][u"receive-rate"] = \
 899                         float(groups.group(3)) / float(groups.group(1))
 900             elif u"RECONF" in tags:
 901                 test_result[u"type"] = u"RECONF"
 902                 test_result[u"result"] = None
 903                 try:
 904                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
 905                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
 906                     test_result[u"result"] = {
 907                         u"loss": int(grps_loss.group(1)),
 908                         u"time": float(grps_time.group(1))
 909                     }
 910                 except (AttributeError, IndexError, ValueError, TypeError):
 911                     test_result[u"status"] = u"FAIL"
 912             else:
 913                 test_result[u"status"] = u"FAIL"
 914                 self._data[u"tests"][self._test_id] = test_result
 915                 return
 916
 917         self._data[u"tests"][self._test_id] = test_result
 918
 919     def end_test(self, test):
 920         """Called when test ends.
 921
 922         :param test: Test to process.
 923         :type test: Test
 924         :returns: Nothing.
 925         """
 926
 927     def visit_keyword(self, keyword):
 928         """Implements traversing through the keyword and its child keywords.
 929
 930         :param keyword: Keyword to process.
 931         :type keyword: Keyword
 932         :returns: Nothing.
 933         """
 934         if self.start_keyword(keyword) is not False:
 935             self.end_keyword(keyword)
 936
 937     def start_keyword(self, keyword):
 938         """Called when keyword starts. Default implementation does nothing.
 939
 940         :param keyword: Keyword to process.
 941         :type keyword: Keyword
 942         :returns: Nothing.
 943         """
 944         try:
 945             if keyword.type == u"setup":
 946                 self.visit_setup_kw(keyword)
 947             elif keyword.type == u"teardown":
 948                 self._lookup_kw_nr = 0
 949                 self.visit_teardown_kw(keyword)
 950             else:
 951                 self._lookup_kw_nr = 0
 952                 self.visit_test_kw(keyword)
 953         except AttributeError:
 954             pass
 955
 956     def end_keyword(self, keyword):
 957         """Called when keyword ends. Default implementation does nothing.
 958
 959         :param keyword: Keyword to process.
 960         :type keyword: Keyword
 961         :returns: Nothing.
 962         """
 963
 964     def visit_test_kw(self, test_kw):
 965         """Implements traversing through the test keyword and its child
 966         keywords.
 967
 968         :param test_kw: Keyword to process.
 969         :type test_kw: Keyword
 970         :returns: Nothing.
 971         """
 972         for keyword in test_kw.keywords:
 973             if self.start_test_kw(keyword) is not False:
 974                 self.visit_test_kw(keyword)
 975                 self.end_test_kw(keyword)
 976
 977     def start_test_kw(self, test_kw):
 978         """Called when test keyword starts. Default implementation does
 979         nothing.
 980
 981         :param test_kw: Keyword to process.
 982         :type test_kw: Keyword
 983         :returns: Nothing.
 984         """
 985         if test_kw.name.count(u"Show Runtime On All Duts") or \
 986                 test_kw.name.count(u"Show Runtime Counters On All Duts"):
 987             self._lookup_kw_nr += 1
 988             self._show_run_lookup_nr = 0
 989             self._msg_type = u"test-show-runtime"
 990         elif test_kw.name.count(u"Install Dpdk Test") and not self._version:
 991             self._msg_type = u"dpdk-version"
 992         else:
 993             return
 994         test_kw.messages.visit(self)
 995
 996     def end_test_kw(self, test_kw):
 997         """Called when keyword ends. Default implementation does nothing.
 998
 999         :param test_kw: Keyword to process.
1000         :type test_kw: Keyword
1001         :returns: Nothing.
1002         """
1003
1004     def visit_setup_kw(self, setup_kw):
1005         """Implements traversing through the teardown keyword and its child
1006         keywords.
1007
1008         :param setup_kw: Keyword to process.
1009         :type setup_kw: Keyword
1010         :returns: Nothing.
1011         """
1012         for keyword in setup_kw.keywords:
1013             if self.start_setup_kw(keyword) is not False:
1014                 self.visit_setup_kw(keyword)
1015                 self.end_setup_kw(keyword)
1016
1017     def start_setup_kw(self, setup_kw):
1018         """Called when teardown keyword starts. Default implementation does
1019         nothing.
1020
1021         :param setup_kw: Keyword to process.
1022         :type setup_kw: Keyword
1023         :returns: Nothing.
1024         """
1025         if setup_kw.name.count(u"Show Vpp Version On All Duts") \
1026                 and not self._version:
1027             self._msg_type = u"vpp-version"
1028         elif setup_kw.name.count(u"Set Global Variable") \
1029                 and not self._timestamp:
1030             self._msg_type = u"timestamp"
1031         elif setup_kw.name.count(u"Setup Framework") and not self._testbed:
1032             self._msg_type = u"testbed"
1033         else:
1034             return
1035         setup_kw.messages.visit(self)
1036
1037     def end_setup_kw(self, setup_kw):
1038         """Called when keyword ends. Default implementation does nothing.
1039
1040         :param setup_kw: Keyword to process.
1041         :type setup_kw: Keyword
1042         :returns: Nothing.
1043         """
1044
1045     def visit_teardown_kw(self, teardown_kw):
1046         """Implements traversing through the teardown keyword and its child
1047         keywords.
1048
1049         :param teardown_kw: Keyword to process.
1050         :type teardown_kw: Keyword
1051         :returns: Nothing.
1052         """
1053         for keyword in teardown_kw.keywords:
1054             if self.start_teardown_kw(keyword) is not False:
1055                 self.visit_teardown_kw(keyword)
1056                 self.end_teardown_kw(keyword)
1057
1058     def start_teardown_kw(self, teardown_kw):
1059         """Called when teardown keyword starts
1060
1061         :param teardown_kw: Keyword to process.
1062         :type teardown_kw: Keyword
1063         :returns: Nothing.
1064         """
1065
1066         if teardown_kw.name.count(u"Show Vat History On All Duts"):
1067             # TODO: Remove when not needed:
1068             self._conf_history_lookup_nr = 0
1069             self._msg_type = u"teardown-vat-history"
1070             teardown_kw.messages.visit(self)
1071         elif teardown_kw.name.count(u"Show Papi History On All Duts"):
1072             self._conf_history_lookup_nr = 0
1073             self._msg_type = u"teardown-papi-history"
1074             teardown_kw.messages.visit(self)
1075
1076     def end_teardown_kw(self, teardown_kw):
1077         """Called when keyword ends. Default implementation does nothing.
1078
1079         :param teardown_kw: Keyword to process.
1080         :type teardown_kw: Keyword
1081         :returns: Nothing.
1082         """
1083
1084     def visit_message(self, msg):
1085         """Implements visiting the message.
1086
1087         :param msg: Message to process.
1088         :type msg: Message
1089         :returns: Nothing.
1090         """
1091         if self.start_message(msg) is not False:
1092             self.end_message(msg)
1093
1094     def start_message(self, msg):
1095         """Called when message starts. Get required information from messages:
1096         - VPP version.
1097
1098         :param msg: Message to process.
1099         :type msg: Message
1100         :returns: Nothing.
1101         """
1102
1103         if self._msg_type:
1104             self.parse_msg[self._msg_type](msg)
1105
1106     def end_message(self, msg):
1107         """Called when message ends. Default implementation does nothing.
1108
1109         :param msg: Message to process.
1110         :type msg: Message
1111         :returns: Nothing.
1112         """
1113
1114
1115 class InputData:
1116     """Input data
1117
1118     The data is extracted from output.xml files generated by Jenkins jobs and
1119     stored in pandas' DataFrames.
1120
1121     The data structure:
1122     - job name
1123       - build number
1124         - metadata
1125           (as described in ExecutionChecker documentation)
1126         - suites
1127           (as described in ExecutionChecker documentation)
1128         - tests
1129           (as described in ExecutionChecker documentation)
1130     """
1131
1132     def __init__(self, spec):
1133         """Initialization.
1134
1135         :param spec: Specification.
1136         :type spec: Specification
1137         """
1138
1139         # Specification:
1140         self._cfg = spec
1141
1142         # Data store:
1143         self._input_data = pd.Series()
1144
1145     @property
1146     def data(self):
1147         """Getter - Input data.
1148
1149         :returns: Input data
1150         :rtype: pandas.Series
1151         """
1152         return self._input_data
1153
1154     def metadata(self, job, build):
1155         """Getter - metadata
1156
1157         :param job: Job which metadata we want.
1158         :param build: Build which metadata we want.
1159         :type job: str
1160         :type build: str
1161         :returns: Metadata
1162         :rtype: pandas.Series
1163         """
1164
1165         return self.data[job][build][u"metadata"]
1166
1167     def suites(self, job, build):
1168         """Getter - suites
1169
1170         :param job: Job which suites we want.
1171         :param build: Build which suites we want.
1172         :type job: str
1173         :type build: str
1174         :returns: Suites.
1175         :rtype: pandas.Series
1176         """
1177
1178         return self.data[job][str(build)][u"suites"]
1179
1180     def tests(self, job, build):
1181         """Getter - tests
1182
1183         :param job: Job which tests we want.
1184         :param build: Build which tests we want.
1185         :type job: str
1186         :type build: str
1187         :returns: Tests.
1188         :rtype: pandas.Series
1189         """
1190
1191         return self.data[job][build][u"tests"]
1192
1193     def _parse_tests(self, job, build, log):
1194         """Process data from robot output.xml file and return JSON structured
1195         data.
1196
1197         :param job: The name of job which build output data will be processed.
1198         :param build: The build which output data will be processed.
1199         :param log: List of log messages.
1200         :type job: str
1201         :type build: dict
1202         :type log: list of tuples (severity, msg)
1203         :returns: JSON data structure.
1204         :rtype: dict
1205         """
1206
1207         metadata = {
1208             u"job": job,
1209             u"build": build
1210         }
1211
1212         with open(build[u"file-name"], u'r') as data_file:
1213             try:
1214                 result = ExecutionResult(data_file)
1215             except errors.DataError as err:
1216                 log.append(
1217                     (u"ERROR", f"Error occurred while parsing output.xml: "
1218                                f"{repr(err)}")
1219                 )
1220                 return None
1221         checker = ExecutionChecker(metadata, self._cfg.mapping,
1222                                    self._cfg.ignore)
1223         result.visit(checker)
1224
1225         return checker.data
1226
1227     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1228         """Download and parse the input data file.
1229
1230         :param pid: PID of the process executing this method.
1231         :param job: Name of the Jenkins job which generated the processed input
1232             file.
1233         :param build: Information about the Jenkins build which generated the
1234             processed input file.
1235         :param repeat: Repeat the download specified number of times if not
1236             successful.
1237         :type pid: int
1238         :type job: str
1239         :type build: dict
1240         :type repeat: int
1241         """
1242
1243         logs = list()
1244
1245         logs.append(
1246             (u"INFO", f"  Processing the job/build: {job}: {build[u'build']}")
1247         )
1248
1249         state = u"failed"
1250         success = False
1251         data = None
1252         do_repeat = repeat
1253         while do_repeat:
1254             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1255                                                    logs)
1256             if success:
1257                 break
1258             do_repeat -= 1
1259         if not success:
1260             logs.append(
1261                 (u"ERROR",
1262                  f"It is not possible to download the input data file from the "
1263                  f"job {job}, build {build[u'build']}, or it is damaged. "
1264                  f"Skipped.")
1265             )
1266         if success:
1267             logs.append(
1268                 (u"INFO",
1269                  f"    Processing data from the build {build[u'build']} ...")
1270             )
1271             data = self._parse_tests(job, build, logs)
1272             if data is None:
1273                 logs.append(
1274                     (u"ERROR",
1275                      f"Input data file from the job {job}, build "
1276                      f"{build[u'build']} is damaged. Skipped.")
1277                 )
1278             else:
1279                 state = u"processed"
1280
1281             try:
1282                 remove(build[u"file-name"])
1283             except OSError as err:
1284                 logs.append(
1285                     ("ERROR", f"Cannot remove the file {build[u'file-name']}: "
1286                               f"{repr(err)}")
1287                 )
1288
1289         # If the time-period is defined in the specification file, remove all
1290         # files which are outside the time period.
1291         timeperiod = self._cfg.input.get(u"time-period", None)
1292         if timeperiod and data:
1293             now = dt.utcnow()
1294             timeperiod = timedelta(int(timeperiod))
1295             metadata = data.get(u"metadata", None)
1296             if metadata:
1297                 generated = metadata.get(u"generated", None)
1298                 if generated:
1299                     generated = dt.strptime(generated, u"%Y%m%d %H:%M")
1300                     if (now - generated) > timeperiod:
1301                         # Remove the data and the file:
1302                         state = u"removed"
1303                         data = None
1304                         logs.append(
1305                             (u"INFO",
1306                              f"    The build {job}/{build[u'build']} is "
1307                              f"outdated, will be removed.")
1308                         )
1309         logs.append((u"INFO", u"  Done."))
1310
1311         for level, line in logs:
1312             if level == u"INFO":
1313                 logging.info(line)
1314             elif level == u"ERROR":
1315                 logging.error(line)
1316             elif level == u"DEBUG":
1317                 logging.debug(line)
1318             elif level == u"CRITICAL":
1319                 logging.critical(line)
1320             elif level == u"WARNING":
1321                 logging.warning(line)
1322
1323         return {u"data": data, u"state": state, u"job": job, u"build": build}
1324
1325     def download_and_parse_data(self, repeat=1):
1326         """Download the input data files, parse input data from input files and
1327         store in pandas' Series.
1328
1329         :param repeat: Repeat the download specified number of times if not
1330             successful.
1331         :type repeat: int
1332         """
1333
1334         logging.info(u"Downloading and parsing input files ...")
1335
1336         for job, builds in self._cfg.builds.items():
1337             for build in builds:
1338
1339                 result = self._download_and_parse_build(job, build, repeat)
1340                 build_nr = result[u"build"][u"build"]
1341
1342                 if result[u"data"]:
1343                     data = result[u"data"]
1344                     build_data = pd.Series({
1345                         u"metadata": pd.Series(
1346                             list(data[u"metadata"].values()),
1347                             index=list(data[u"metadata"].keys())
1348                         ),
1349                         u"suites": pd.Series(
1350                             list(data[u"suites"].values()),
1351                             index=list(data[u"suites"].keys())
1352                         ),
1353                         u"tests": pd.Series(
1354                             list(data[u"tests"].values()),
1355                             index=list(data[u"tests"].keys())
1356                         )
1357                     })
1358
1359                     if self._input_data.get(job, None) is None:
1360                         self._input_data[job] = pd.Series()
1361                     self._input_data[job][str(build_nr)] = build_data
1362
1363                     self._cfg.set_input_file_name(
1364                         job, build_nr, result[u"build"][u"file-name"])
1365
1366                 self._cfg.set_input_state(job, build_nr, result[u"state"])
1367
1368                 mem_alloc = \
1369                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
1370                 logging.info(f"Memory allocation: {mem_alloc:.0f}MB")
1371
1372         logging.info(u"Done.")
1373
1374     @staticmethod
1375     def _end_of_tag(tag_filter, start=0, closer=u"'"):
1376         """Return the index of character in the string which is the end of tag.
1377
1378         :param tag_filter: The string where the end of tag is being searched.
1379         :param start: The index where the searching is stated.
1380         :param closer: The character which is the tag closer.
1381         :type tag_filter: str
1382         :type start: int
1383         :type closer: str
1384         :returns: The index of the tag closer.
1385         :rtype: int
1386         """
1387
1388         try:
1389             idx_opener = tag_filter.index(closer, start)
1390             return tag_filter.index(closer, idx_opener + 1)
1391         except ValueError:
1392             return None
1393
1394     @staticmethod
1395     def _condition(tag_filter):
1396         """Create a conditional statement from the given tag filter.
1397
1398         :param tag_filter: Filter based on tags from the element specification.
1399         :type tag_filter: str
1400         :returns: Conditional statement which can be evaluated.
1401         :rtype: str
1402         """
1403
1404         index = 0
1405         while True:
1406             index = InputData._end_of_tag(tag_filter, index)
1407             if index is None:
1408                 return tag_filter
1409             index += 1
1410             tag_filter = tag_filter[:index] + u" in tags" + tag_filter[index:]
1411
1412     def filter_data(self, element, params=None, data=None, data_set=u"tests",
1413                     continue_on_error=False):
1414         """Filter required data from the given jobs and builds.
1415
1416         The output data structure is:
1417
1418         - job 1
1419           - build 1
1420             - test (or suite) 1 ID:
1421               - param 1
1422               - param 2
1423               ...
1424               - param n
1425             ...
1426             - test (or suite) n ID:
1427             ...
1428           ...
1429           - build n
1430         ...
1431         - job n
1432
1433         :param element: Element which will use the filtered data.
1434         :param params: Parameters which will be included in the output. If None,
1435             all parameters are included.
1436         :param data: If not None, this data is used instead of data specified
1437             in the element.
1438         :param data_set: The set of data to be filtered: tests, suites,
1439             metadata.
1440         :param continue_on_error: Continue if there is error while reading the
1441             data. The Item will be empty then
1442         :type element: pandas.Series
1443         :type params: list
1444         :type data: dict
1445         :type data_set: str
1446         :type continue_on_error: bool
1447         :returns: Filtered data.
1448         :rtype pandas.Series
1449         """
1450
1451         try:
1452             if element[u"filter"] in (u"all", u"template"):
1453                 cond = u"True"
1454             else:
1455                 cond = InputData._condition(element[u"filter"])
1456             logging.debug(f"   Filter: {cond}")
1457         except KeyError:
1458             logging.error(u"  No filter defined.")
1459             return None
1460
1461         if params is None:
1462             params = element.get(u"parameters", None)
1463             if params:
1464                 params.append(u"type")
1465
1466         data_to_filter = data if data else element[u"data"]
1467         data = pd.Series()
1468         try:
1469             for job, builds in data_to_filter.items():
1470                 data[job] = pd.Series()
1471                 for build in builds:
1472                     data[job][str(build)] = pd.Series()
1473                     try:
1474                         data_dict = dict(
1475                             self.data[job][str(build)][data_set].items())
1476                     except KeyError:
1477                         if continue_on_error:
1478                             continue
1479                         return None
1480
1481                     for test_id, test_data in data_dict.items():
1482                         if eval(cond, {u"tags": test_data.get(u"tags", u"")}):
1483                             data[job][str(build)][test_id] = pd.Series()
1484                             if params is None:
1485                                 for param, val in test_data.items():
1486                                     data[job][str(build)][test_id][param] = val
1487                             else:
1488                                 for param in params:
1489                                     try:
1490                                         data[job][str(build)][test_id][param] =\
1491                                             test_data[param]
1492                                     except KeyError:
1493                                         data[job][str(build)][test_id][param] =\
1494                                             u"No Data"
1495             return data
1496
1497         except (KeyError, IndexError, ValueError) as err:
1498             logging.error(
1499                 f"Missing mandatory parameter in the element specification: "
1500                 f"{repr(err)}"
1501             )
1502             return None
1503         except AttributeError as err:
1504             logging.error(repr(err))
1505             return None
1506         except SyntaxError as err:
1507             logging.error(
1508                 f"The filter {cond} is not correct. Check if all tags are "
1509                 f"enclosed by apostrophes.\n{repr(err)}"
1510             )
1511             return None
1512
1513     def filter_tests_by_name(self, element, params=None, data_set=u"tests",
1514                              continue_on_error=False):
1515         """Filter required data from the given jobs and builds.
1516
1517         The output data structure is:
1518
1519         - job 1
1520           - build 1
1521             - test (or suite) 1 ID:
1522               - param 1
1523               - param 2
1524               ...
1525               - param n
1526             ...
1527             - test (or suite) n ID:
1528             ...
1529           ...
1530           - build n
1531         ...
1532         - job n
1533
1534         :param element: Element which will use the filtered data.
1535         :param params: Parameters which will be included in the output. If None,
1536         all parameters are included.
1537         :param data_set: The set of data to be filtered: tests, suites,
1538         metadata.
1539         :param continue_on_error: Continue if there is error while reading the
1540         data. The Item will be empty then
1541         :type element: pandas.Series
1542         :type params: list
1543         :type data_set: str
1544         :type continue_on_error: bool
1545         :returns: Filtered data.
1546         :rtype pandas.Series
1547         """
1548
1549         include = element.get(u"include", None)
1550         if not include:
1551             logging.warning(u"No tests to include, skipping the element.")
1552             return None
1553
1554         if params is None:
1555             params = element.get(u"parameters", None)
1556             if params:
1557                 params.append(u"type")
1558
1559         data = pd.Series()
1560         try:
1561             for job, builds in element[u"data"].items():
1562                 data[job] = pd.Series()
1563                 for build in builds:
1564                     data[job][str(build)] = pd.Series()
1565                     for test in include:
1566                         try:
1567                             reg_ex = re.compile(str(test).lower())
1568                             for test_id in self.data[job][
1569                                     str(build)][data_set].keys():
1570                                 if re.match(reg_ex, str(test_id).lower()):
1571                                     test_data = self.data[job][
1572                                         str(build)][data_set][test_id]
1573                                     data[job][str(build)][test_id] = pd.Series()
1574                                     if params is None:
1575                                         for param, val in test_data.items():
1576                                             data[job][str(build)][test_id]\
1577                                                 [param] = val
1578                                     else:
1579                                         for param in params:
1580                                             try:
1581                                                 data[job][str(build)][
1582                                                     test_id][param] = \
1583                                                     test_data[param]
1584                                             except KeyError:
1585                                                 data[job][str(build)][
1586                                                     test_id][param] = u"No Data"
1587                         except KeyError as err:
1588                             logging.error(repr(err))
1589                             if continue_on_error:
1590                                 continue
1591                             return None
1592             return data
1593
1594         except (KeyError, IndexError, ValueError) as err:
1595             logging.error(
1596                 f"Missing mandatory parameter in the element "
1597                 f"specification: {repr(err)}"
1598             )
1599             return None
1600         except AttributeError as err:
1601             logging.error(repr(err))
1602             return None
1603
1604     @staticmethod
1605     def merge_data(data):
1606         """Merge data from more jobs and builds to a simple data structure.
1607
1608         The output data structure is:
1609
1610         - test (suite) 1 ID:
1611           - param 1
1612           - param 2
1613           ...
1614           - param n
1615         ...
1616         - test (suite) n ID:
1617         ...
1618
1619         :param data: Data to merge.
1620         :type data: pandas.Series
1621         :returns: Merged data.
1622         :rtype: pandas.Series
1623         """
1624
1625         logging.info(u"    Merging data ...")
1626
1627         merged_data = pd.Series()
1628         for builds in data.values:
1629             for item in builds.values:
1630                 for item_id, item_data in item.items():
1631                     merged_data[item_id] = item_data
1632
1633         return merged_data