resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import multiprocessing
  23 import os
  24 import re
  25 import resource
  26 import objgraph
  27 import pandas as pd
  28 import logging
  29
  30 from robot.api import ExecutionResult, ResultVisitor
  31 from robot import errors
  32 from collections import OrderedDict
  33 from string import replace
  34 from os import remove
  35 from os.path import join
  36 from datetime import datetime as dt
  37 from datetime import timedelta
  38 from json import loads
  39 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
  40
  41 from input_data_files import download_and_unzip_data_file
  42 from utils import Worker
  43
  44
  45 # Separator used in file names
  46 SEPARATOR = "__"
  47
  48
  49 class ExecutionChecker(ResultVisitor):
  50     """Class to traverse through the test suite structure.
  51
  52     The functionality implemented in this class generates a json structure:
  53
  54     Performance tests:
  55
  56     {
  57         "metadata": {
  58             "generated": "Timestamp",
  59             "version": "SUT version",
  60             "job": "Jenkins job name",
  61             "build": "Information about the build"
  62         },
  63         "suites": {
  64             "Suite long name 1": {
  65                 "name": Suite name,
  66                 "doc": "Suite 1 documentation",
  67                 "parent": "Suite 1 parent",
  68                 "level": "Level of the suite in the suite hierarchy"
  69             }
  70             "Suite long name N": {
  71                 "name": Suite name,
  72                 "doc": "Suite N documentation",
  73                 "parent": "Suite 2 parent",
  74                 "level": "Level of the suite in the suite hierarchy"
  75             }
  76         }
  77         "tests": {
  78             # NDRPDR tests:
  79             "ID": {
  80                 "name": "Test name",
  81                 "parent": "Name of the parent of the test",
  82                 "doc": "Test documentation",
  83                 "msg": "Test message",
  84                 "conf-history": "DUT1 and DUT2 VAT History",
  85                 "show-run": "Show Run",
  86                 "tags": ["tag 1", "tag 2", "tag n"],
  87                 "type": "NDRPDR",
  88                 "status": "PASS" | "FAIL",
  89                 "throughput": {
  90                     "NDR": {
  91                         "LOWER": float,
  92                         "UPPER": float
  93                     },
  94                     "PDR": {
  95                         "LOWER": float,
  96                         "UPPER": float
  97                     }
  98                 },
  99                 "latency": {
 100                     "NDR": {
 101                         "direction1": {
 102                             "min": float,
 103                             "avg": float,
 104                             "max": float
 105                         },
 106                         "direction2": {
 107                             "min": float,
 108                             "avg": float,
 109                             "max": float
 110                         }
 111                     },
 112                     "PDR": {
 113                         "direction1": {
 114                             "min": float,
 115                             "avg": float,
 116                             "max": float
 117                         },
 118                         "direction2": {
 119                             "min": float,
 120                             "avg": float,
 121                             "max": float
 122                         }
 123                     }
 124                 }
 125             }
 126
 127             # TCP tests:
 128             "ID": {
 129                 "name": "Test name",
 130                 "parent": "Name of the parent of the test",
 131                 "doc": "Test documentation",
 132                 "msg": "Test message",
 133                 "tags": ["tag 1", "tag 2", "tag n"],
 134                 "type": "TCP",
 135                 "status": "PASS" | "FAIL",
 136                 "result": int
 137             }
 138
 139             # MRR, BMRR tests:
 140             "ID": {
 141                 "name": "Test name",
 142                 "parent": "Name of the parent of the test",
 143                 "doc": "Test documentation",
 144                 "msg": "Test message",
 145                 "tags": ["tag 1", "tag 2", "tag n"],
 146                 "type": "MRR" | "BMRR",
 147                 "status": "PASS" | "FAIL",
 148                 "result": {
 149                     "receive-rate": AvgStdevMetadata,
 150                 }
 151             }
 152
 153             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 154             # NDRPDRDISC tests:
 155             "ID": {
 156                 "name": "Test name",
 157                 "parent": "Name of the parent of the test",
 158                 "doc": "Test documentation",
 159                 "msg": "Test message",
 160                 "tags": ["tag 1", "tag 2", "tag n"],
 161                 "type": "PDR" | "NDR",
 162                 "status": "PASS" | "FAIL",
 163                 "throughput": {  # Only type: "PDR" | "NDR"
 164                     "value": int,
 165                     "unit": "pps" | "bps" | "percentage"
 166                 },
 167                 "latency": {  # Only type: "PDR" | "NDR"
 168                     "direction1": {
 169                         "100": {
 170                             "min": int,
 171                             "avg": int,
 172                             "max": int
 173                         },
 174                         "50": {  # Only for NDR
 175                             "min": int,
 176                             "avg": int,
 177                             "max": int
 178                         },
 179                         "10": {  # Only for NDR
 180                             "min": int,
 181                             "avg": int,
 182                             "max": int
 183                         }
 184                     },
 185                     "direction2": {
 186                         "100": {
 187                             "min": int,
 188                             "avg": int,
 189                             "max": int
 190                         },
 191                         "50": {  # Only for NDR
 192                             "min": int,
 193                             "avg": int,
 194                             "max": int
 195                         },
 196                         "10": {  # Only for NDR
 197                             "min": int,
 198                             "avg": int,
 199                             "max": int
 200                         }
 201                     }
 202                 },
 203                 "lossTolerance": "lossTolerance",  # Only type: "PDR"
 204                 "conf-history": "DUT1 and DUT2 VAT History"
 205                 "show-run": "Show Run"
 206             },
 207             "ID" {
 208                 # next test
 209             }
 210         }
 211     }
 212
 213
 214     Functional tests:
 215
 216     {
 217         "metadata": {  # Optional
 218             "version": "VPP version",
 219             "job": "Jenkins job name",
 220             "build": "Information about the build"
 221         },
 222         "suites": {
 223             "Suite name 1": {
 224                 "doc": "Suite 1 documentation",
 225                 "parent": "Suite 1 parent",
 226                 "level": "Level of the suite in the suite hierarchy"
 227             }
 228             "Suite name N": {
 229                 "doc": "Suite N documentation",
 230                 "parent": "Suite 2 parent",
 231                 "level": "Level of the suite in the suite hierarchy"
 232             }
 233         }
 234         "tests": {
 235             "ID": {
 236                 "name": "Test name",
 237                 "parent": "Name of the parent of the test",
 238                 "doc": "Test documentation"
 239                 "msg": "Test message"
 240                 "tags": ["tag 1", "tag 2", "tag n"],
 241                 "conf-history": "DUT1 and DUT2 VAT History"
 242                 "show-run": "Show Run"
 243                 "status": "PASS" | "FAIL"
 244             },
 245             "ID" {
 246                 # next test
 247             }
 248         }
 249     }
 250
 251     .. note:: ID is the lowercase full path to the test.
 252     """
 253
 254     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 255     REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
 256
 257     REGEX_PLR_RATE = re.compile(r'PLRsearch lower bound::\s(\d+.\d+).*\n'
 258                                 r'PLRsearch upper bound::\s(\d+.\d+)')
 259
 260     REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 261                                    r'NDR_UPPER:\s(\d+.\d+).*\n'
 262                                    r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 263                                    r'PDR_UPPER:\s(\d+.\d+)')
 264
 265     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 266     REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
 267                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 268                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
 269                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 270                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
 271                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 272                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
 273
 274     REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
 275                                r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 276                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
 277
 278     REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 279                                   r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
 280
 281     REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
 282                                  r'[\D\d]*')
 283
 284     REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|"
 285                                    r"VPP Version:\s*|VPP version:\s*)(.*)")
 286
 287     REGEX_VERSION_DPDK = re.compile(r"(return STDOUT testpmd)([\d\D\n]*)"
 288                                     r"(RTE Version: 'DPDK )(.*)(')")
 289
 290     REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
 291
 292     REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 293                            r'tx\s(\d*),\srx\s(\d*)')
 294
 295     REGEX_BMRR = re.compile(r'Maximum Receive Rate trial results'
 296                             r' in packets per second: \[(.*)\]')
 297
 298     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 299
 300     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 301
 302     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 303
 304     REGEX_TC_NUMBER = re.compile(r'tc[0-9]{2}-')
 305
 306     def __init__(self, metadata, mapping, ignore):
 307         """Initialisation.
 308
 309         :param metadata: Key-value pairs to be included in "metadata" part of
 310             JSON structure.
 311         :param mapping: Mapping of the old names of test cases to the new
 312             (actual) one.
 313         :param ignore: List of TCs to be ignored.
 314         :type metadata: dict
 315         :type mapping: dict
 316         :type ignore: list
 317         """
 318
 319         # Type of message to parse out from the test messages
 320         self._msg_type = None
 321
 322         # VPP version
 323         self._version = None
 324
 325         # Timestamp
 326         self._timestamp = None
 327
 328         # Testbed. The testbed is identified by TG node IP address.
 329         self._testbed = None
 330
 331         # Mapping of TCs long names
 332         self._mapping = mapping
 333
 334         # Ignore list
 335         self._ignore = ignore
 336
 337         # Number of VAT History messages found:
 338         # 0 - no message
 339         # 1 - VAT History of DUT1
 340         # 2 - VAT History of DUT2
 341         self._lookup_kw_nr = 0
 342         self._conf_history_lookup_nr = 0
 343
 344         # Number of Show Running messages found
 345         # 0 - no message
 346         # 1 - Show run message found
 347         self._show_run_lookup_nr = 0
 348
 349         # Test ID of currently processed test- the lowercase full path to the
 350         # test
 351         self._test_ID = None
 352
 353         # The main data structure
 354         self._data = {
 355             "metadata": OrderedDict(),
 356             "suites": OrderedDict(),
 357             "tests": OrderedDict()
 358         }
 359
 360         # Save the provided metadata
 361         for key, val in metadata.items():
 362             self._data["metadata"][key] = val
 363
 364         # Dictionary defining the methods used to parse different types of
 365         # messages
 366         self.parse_msg = {
 367             "timestamp": self._get_timestamp,
 368             "vpp-version": self._get_vpp_version,
 369             "dpdk-version": self._get_dpdk_version,
 370             "teardown-vat-history": self._get_vat_history,
 371             "teardown-papi-history": self._get_papi_history,
 372             "test-show-runtime": self._get_show_run,
 373             "testbed": self._get_testbed
 374         }
 375
 376     @property
 377     def data(self):
 378         """Getter - Data parsed from the XML file.
 379
 380         :returns: Data parsed from the XML file.
 381         :rtype: dict
 382         """
 383         return self._data
 384
 385     def _get_testbed(self, msg):
 386         """Called when extraction of testbed IP is required.
 387         The testbed is identified by TG node IP address.
 388
 389         :param msg: Message to process.
 390         :type msg: Message
 391         :returns: Nothing.
 392         """
 393
 394         if msg.message.count("Arguments:"):
 395             message = str(msg.message).replace(' ', '').replace('\n', '').\
 396                 replace("'", '"').replace('b"', '"').\
 397                 replace("honeycom", "honeycomb")
 398             message = loads(message[11:-1])
 399             try:
 400                 self._testbed = message["TG"]["host"]
 401             except (KeyError, ValueError):
 402                 pass
 403             finally:
 404                 self._data["metadata"]["testbed"] = self._testbed
 405                 self._msg_type = None
 406
 407     def _get_vpp_version(self, msg):
 408         """Called when extraction of VPP version is required.
 409
 410         :param msg: Message to process.
 411         :type msg: Message
 412         :returns: Nothing.
 413         """
 414
 415         if msg.message.count("return STDOUT Version:") or \
 416             msg.message.count("VPP Version:") or \
 417             msg.message.count("VPP version:"):
 418             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 419                                 group(2))
 420             self._data["metadata"]["version"] = self._version
 421             self._msg_type = None
 422
 423     def _get_dpdk_version(self, msg):
 424         """Called when extraction of DPDK version is required.
 425
 426         :param msg: Message to process.
 427         :type msg: Message
 428         :returns: Nothing.
 429         """
 430
 431         if msg.message.count("return STDOUT testpmd"):
 432             try:
 433                 self._version = str(re.search(
 434                     self.REGEX_VERSION_DPDK, msg.message). group(4))
 435                 self._data["metadata"]["version"] = self._version
 436             except IndexError:
 437                 pass
 438             finally:
 439                 self._msg_type = None
 440
 441     def _get_timestamp(self, msg):
 442         """Called when extraction of timestamp is required.
 443
 444         :param msg: Message to process.
 445         :type msg: Message
 446         :returns: Nothing.
 447         """
 448
 449         self._timestamp = msg.timestamp[:14]
 450         self._data["metadata"]["generated"] = self._timestamp
 451         self._msg_type = None
 452
 453     def _get_vat_history(self, msg):
 454         """Called when extraction of VAT command history is required.
 455
 456         :param msg: Message to process.
 457         :type msg: Message
 458         :returns: Nothing.
 459         """
 460         if msg.message.count("VAT command history:"):
 461             self._conf_history_lookup_nr += 1
 462             if self._conf_history_lookup_nr == 1:
 463                 self._data["tests"][self._test_ID]["conf-history"] = str()
 464             else:
 465                 self._msg_type = None
 466             text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
 467                           "VAT command history:", "", msg.message, count=1). \
 468                 replace("\n\n", "\n").replace('\n', ' |br| ').\
 469                 replace('\r', '').replace('"', "'")
 470
 471             self._data["tests"][self._test_ID]["conf-history"] += " |br| "
 472             self._data["tests"][self._test_ID]["conf-history"] += \
 473                 "**DUT" + str(self._conf_history_lookup_nr) + ":** " + text
 474
 475     def _get_papi_history(self, msg):
 476         """Called when extraction of PAPI command history is required.
 477
 478         :param msg: Message to process.
 479         :type msg: Message
 480         :returns: Nothing.
 481         """
 482         if msg.message.count("PAPI command history:"):
 483             self._conf_history_lookup_nr += 1
 484             if self._conf_history_lookup_nr == 1:
 485                 self._data["tests"][self._test_ID]["conf-history"] = str()
 486             else:
 487                 self._msg_type = None
 488             text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
 489                           "PAPI command history:", "", msg.message, count=1). \
 490                 replace("\n\n", "\n").replace('\n', ' |br| ').\
 491                 replace('\r', '').replace('"', "'")
 492
 493             self._data["tests"][self._test_ID]["conf-history"] += " |br| "
 494             self._data["tests"][self._test_ID]["conf-history"] += \
 495                 "**DUT" + str(self._conf_history_lookup_nr) + ":** " + text
 496
 497     def _get_show_run(self, msg):
 498         """Called when extraction of VPP operational data (output of CLI command
 499         Show Runtime) is required.
 500
 501         :param msg: Message to process.
 502         :type msg: Message
 503         :returns: Nothing.
 504         """
 505         if msg.message.count("Thread 0 vpp_main"):
 506             self._show_run_lookup_nr += 1
 507             if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
 508                 self._data["tests"][self._test_ID]["show-run"] = str()
 509             if self._lookup_kw_nr > 1:
 510                 self._msg_type = None
 511             if self._show_run_lookup_nr == 1:
 512                 text = msg.message.replace("vat# ", "").\
 513                     replace("return STDOUT ", "").replace("\n\n", "\n").\
 514                     replace('\n', ' |br| ').\
 515                     replace('\r', '').replace('"', "'")
 516                 try:
 517                     self._data["tests"][self._test_ID]["show-run"] += " |br| "
 518                     self._data["tests"][self._test_ID]["show-run"] += \
 519                         "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
 520                 except KeyError:
 521                     pass
 522
 523     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 524     def _get_latency(self, msg, test_type):
 525         """Get the latency data from the test message.
 526
 527         :param msg: Message to be parsed.
 528         :param test_type: Type of the test - NDR or PDR.
 529         :type msg: str
 530         :type test_type: str
 531         :returns: Latencies parsed from the message.
 532         :rtype: dict
 533         """
 534
 535         if test_type == "NDR":
 536             groups = re.search(self.REGEX_LAT_NDR, msg)
 537             groups_range = range(1, 7)
 538         elif test_type == "PDR":
 539             groups = re.search(self.REGEX_LAT_PDR, msg)
 540             groups_range = range(1, 3)
 541         else:
 542             return {}
 543
 544         latencies = list()
 545         for idx in groups_range:
 546             try:
 547                 lat = [int(item) for item in str(groups.group(idx)).split('/')]
 548             except (AttributeError, ValueError):
 549                 lat = [-1, -1, -1]
 550             latencies.append(lat)
 551
 552         keys = ("min", "avg", "max")
 553         latency = {
 554             "direction1": {
 555             },
 556             "direction2": {
 557             }
 558         }
 559
 560         latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
 561         latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
 562         if test_type == "NDR":
 563             latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
 564             latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
 565             latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
 566             latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
 567
 568         return latency
 569
 570     def _get_ndrpdr_throughput(self, msg):
 571         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 572         message.
 573
 574         :param msg: The test message to be parsed.
 575         :type msg: str
 576         :returns: Parsed data as a dict and the status (PASS/FAIL).
 577         :rtype: tuple(dict, str)
 578         """
 579
 580         throughput = {
 581             "NDR": {"LOWER": -1.0, "UPPER": -1.0},
 582             "PDR": {"LOWER": -1.0, "UPPER": -1.0}
 583         }
 584         status = "FAIL"
 585         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 586
 587         if groups is not None:
 588             try:
 589                 throughput["NDR"]["LOWER"] = float(groups.group(1))
 590                 throughput["NDR"]["UPPER"] = float(groups.group(2))
 591                 throughput["PDR"]["LOWER"] = float(groups.group(3))
 592                 throughput["PDR"]["UPPER"] = float(groups.group(4))
 593                 status = "PASS"
 594             except (IndexError, ValueError):
 595                 pass
 596
 597         return throughput, status
 598
 599     def _get_plr_throughput(self, msg):
 600         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 601         message.
 602
 603         :param msg: The test message to be parsed.
 604         :type msg: str
 605         :returns: Parsed data as a dict and the status (PASS/FAIL).
 606         :rtype: tuple(dict, str)
 607         """
 608
 609         throughput = {
 610             "LOWER": -1.0,
 611             "UPPER": -1.0
 612         }
 613         status = "FAIL"
 614         groups = re.search(self.REGEX_PLR_RATE, msg)
 615
 616         if groups is not None:
 617             try:
 618                 throughput["LOWER"] = float(groups.group(1))
 619                 throughput["UPPER"] = float(groups.group(2))
 620                 status = "PASS"
 621             except (IndexError, ValueError):
 622                 pass
 623
 624         return throughput, status
 625
 626     def _get_ndrpdr_latency(self, msg):
 627         """Get LATENCY from the test message.
 628
 629         :param msg: The test message to be parsed.
 630         :type msg: str
 631         :returns: Parsed data as a dict and the status (PASS/FAIL).
 632         :rtype: tuple(dict, str)
 633         """
 634
 635         latency = {
 636             "NDR": {
 637                 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
 638                 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
 639             },
 640             "PDR": {
 641                 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
 642                 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
 643             }
 644         }
 645         status = "FAIL"
 646         groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 647
 648         if groups is not None:
 649             keys = ("min", "avg", "max")
 650             try:
 651                 latency["NDR"]["direction1"] = dict(
 652                     zip(keys, [float(l) for l in groups.group(1).split('/')]))
 653                 latency["NDR"]["direction2"] = dict(
 654                     zip(keys, [float(l) for l in groups.group(2).split('/')]))
 655                 latency["PDR"]["direction1"] = dict(
 656                     zip(keys, [float(l) for l in groups.group(3).split('/')]))
 657                 latency["PDR"]["direction2"] = dict(
 658                     zip(keys, [float(l) for l in groups.group(4).split('/')]))
 659                 status = "PASS"
 660             except (IndexError, ValueError):
 661                 pass
 662
 663         return latency, status
 664
 665     def visit_suite(self, suite):
 666         """Implements traversing through the suite and its direct children.
 667
 668         :param suite: Suite to process.
 669         :type suite: Suite
 670         :returns: Nothing.
 671         """
 672         if self.start_suite(suite) is not False:
 673             suite.suites.visit(self)
 674             suite.tests.visit(self)
 675             self.end_suite(suite)
 676
 677     def start_suite(self, suite):
 678         """Called when suite starts.
 679
 680         :param suite: Suite to process.
 681         :type suite: Suite
 682         :returns: Nothing.
 683         """
 684
 685         try:
 686             parent_name = suite.parent.name
 687         except AttributeError:
 688             return
 689
 690         doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
 691             replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
 692         doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
 693
 694         self._data["suites"][suite.longname.lower().replace('"', "'").
 695             replace(" ", "_")] = {
 696                 "name": suite.name.lower(),
 697                 "doc": doc_str,
 698                 "parent": parent_name,
 699                 "level": len(suite.longname.split("."))
 700             }
 701
 702         suite.keywords.visit(self)
 703
 704     def end_suite(self, suite):
 705         """Called when suite ends.
 706
 707         :param suite: Suite to process.
 708         :type suite: Suite
 709         :returns: Nothing.
 710         """
 711         pass
 712
 713     def visit_test(self, test):
 714         """Implements traversing through the test.
 715
 716         :param test: Test to process.
 717         :type test: Test
 718         :returns: Nothing.
 719         """
 720         if self.start_test(test) is not False:
 721             test.keywords.visit(self)
 722             self.end_test(test)
 723
 724     def start_test(self, test):
 725         """Called when test starts.
 726
 727         :param test: Test to process.
 728         :type test: Test
 729         :returns: Nothing.
 730         """
 731
 732         longname_orig = test.longname.lower()
 733
 734         # Check the ignore list
 735         if longname_orig in self._ignore:
 736             return
 737
 738         tags = [str(tag) for tag in test.tags]
 739         test_result = dict()
 740
 741         # Change the TC long name and name if defined in the mapping table
 742         longname = self._mapping.get(longname_orig, None)
 743         if longname is not None:
 744             name = longname.split('.')[-1]
 745             logging.debug("{0}\n{1}\n{2}\n{3}".format(
 746                 self._data["metadata"], longname_orig, longname, name))
 747         else:
 748             longname = longname_orig
 749             name = test.name.lower()
 750
 751         # Remove TC number from the TC long name (backward compatibility):
 752         self._test_ID = re.sub(self.REGEX_TC_NUMBER, "", longname)
 753         # Remove TC number from the TC name (not needed):
 754         test_result["name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
 755
 756         test_result["parent"] = test.parent.name.lower()
 757         test_result["tags"] = tags
 758         doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
 759             replace('\r', '').replace('[', ' |br| [')
 760         test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
 761         test_result["msg"] = test.message.replace('\n', ' |br| '). \
 762             replace('\r', '').replace('"', "'")
 763         test_result["type"] = "FUNC"
 764         test_result["status"] = test.status
 765
 766         if "PERFTEST" in tags:
 767             # Replace info about cores (e.g. -1c-) with the info about threads
 768             # and cores (e.g. -1t1c-) in the long test case names and in the
 769             # test case names if necessary.
 770             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_ID)
 771             if not groups:
 772                 tag_count = 0
 773                 for tag in test_result["tags"]:
 774                     groups = re.search(self.REGEX_TC_TAG, tag)
 775                     if groups:
 776                         tag_count += 1
 777                         tag_tc = tag
 778
 779                 if tag_count == 1:
 780                     self._test_ID = re.sub(self.REGEX_TC_NAME_NEW,
 781                                            "-{0}-".format(tag_tc.lower()),
 782                                            self._test_ID,
 783                                            count=1)
 784                     test_result["name"] = re.sub(self.REGEX_TC_NAME_NEW,
 785                                                  "-{0}-".format(tag_tc.lower()),
 786                                                  test_result["name"],
 787                                                  count=1)
 788                 else:
 789                     test_result["status"] = "FAIL"
 790                     self._data["tests"][self._test_ID] = test_result
 791                     logging.debug("The test '{0}' has no or more than one "
 792                                   "multi-threading tags.".format(self._test_ID))
 793                     logging.debug("Tags: {0}".format(test_result["tags"]))
 794                     return
 795
 796         if test.status == "PASS" and ("NDRPDRDISC" in tags or
 797                                       "NDRPDR" in tags or
 798                                       "SOAK" in tags or
 799                                       "TCP" in tags or
 800                                       "MRR" in tags or
 801                                       "BMRR" in tags):
 802             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 803             if "NDRDISC" in tags:
 804                 test_result["type"] = "NDR"
 805             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 806             elif "PDRDISC" in tags:
 807                 test_result["type"] = "PDR"
 808             elif "NDRPDR" in tags:
 809                 test_result["type"] = "NDRPDR"
 810             elif "SOAK" in tags:
 811                 test_result["type"] = "SOAK"
 812             elif "TCP" in tags:
 813                 test_result["type"] = "TCP"
 814             elif "MRR" in tags:
 815                 test_result["type"] = "MRR"
 816             elif "FRMOBL" in tags or "BMRR" in tags:
 817                 test_result["type"] = "BMRR"
 818             else:
 819                 test_result["status"] = "FAIL"
 820                 self._data["tests"][self._test_ID] = test_result
 821                 return
 822
 823             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 824             if test_result["type"] in ("NDR", "PDR"):
 825                 try:
 826                     rate_value = str(re.search(
 827                         self.REGEX_RATE, test.message).group(1))
 828                 except AttributeError:
 829                     rate_value = "-1"
 830                 try:
 831                     rate_unit = str(re.search(
 832                         self.REGEX_RATE, test.message).group(2))
 833                 except AttributeError:
 834                     rate_unit = "-1"
 835
 836                 test_result["throughput"] = dict()
 837                 test_result["throughput"]["value"] = \
 838                     int(rate_value.split('.')[0])
 839                 test_result["throughput"]["unit"] = rate_unit
 840                 test_result["latency"] = \
 841                     self._get_latency(test.message, test_result["type"])
 842                 if test_result["type"] == "PDR":
 843                     test_result["lossTolerance"] = str(re.search(
 844                         self.REGEX_TOLERANCE, test.message).group(1))
 845
 846             elif test_result["type"] in ("NDRPDR", ):
 847                 test_result["throughput"], test_result["status"] = \
 848                     self._get_ndrpdr_throughput(test.message)
 849                 test_result["latency"], test_result["status"] = \
 850                     self._get_ndrpdr_latency(test.message)
 851
 852             elif test_result["type"] in ("SOAK", ):
 853                 test_result["throughput"], test_result["status"] = \
 854                     self._get_plr_throughput(test.message)
 855
 856             elif test_result["type"] in ("TCP", ):
 857                 groups = re.search(self.REGEX_TCP, test.message)
 858                 test_result["result"] = int(groups.group(2))
 859
 860             elif test_result["type"] in ("MRR", "BMRR"):
 861                 test_result["result"] = dict()
 862                 groups = re.search(self.REGEX_BMRR, test.message)
 863                 if groups is not None:
 864                     items_str = groups.group(1)
 865                     items_float = [float(item.strip()) for item
 866                                    in items_str.split(",")]
 867                     metadata = AvgStdevMetadataFactory.from_data(items_float)
 868                     # Next two lines have been introduced in CSIT-1179,
 869                     # to be removed in CSIT-1180.
 870                     metadata.size = 1
 871                     metadata.stdev = 0.0
 872                     test_result["result"]["receive-rate"] = metadata
 873                 else:
 874                     groups = re.search(self.REGEX_MRR, test.message)
 875                     test_result["result"]["receive-rate"] = \
 876                         AvgStdevMetadataFactory.from_data([
 877                             float(groups.group(3)) / float(groups.group(1)), ])
 878
 879         self._data["tests"][self._test_ID] = test_result
 880
 881     def end_test(self, test):
 882         """Called when test ends.
 883
 884         :param test: Test to process.
 885         :type test: Test
 886         :returns: Nothing.
 887         """
 888         pass
 889
 890     def visit_keyword(self, keyword):
 891         """Implements traversing through the keyword and its child keywords.
 892
 893         :param keyword: Keyword to process.
 894         :type keyword: Keyword
 895         :returns: Nothing.
 896         """
 897         if self.start_keyword(keyword) is not False:
 898             self.end_keyword(keyword)
 899
 900     def start_keyword(self, keyword):
 901         """Called when keyword starts. Default implementation does nothing.
 902
 903         :param keyword: Keyword to process.
 904         :type keyword: Keyword
 905         :returns: Nothing.
 906         """
 907         try:
 908             if keyword.type == "setup":
 909                 self.visit_setup_kw(keyword)
 910             elif keyword.type == "teardown":
 911                 self._lookup_kw_nr = 0
 912                 self.visit_teardown_kw(keyword)
 913             else:
 914                 self._lookup_kw_nr = 0
 915                 self.visit_test_kw(keyword)
 916         except AttributeError:
 917             pass
 918
 919     def end_keyword(self, keyword):
 920         """Called when keyword ends. Default implementation does nothing.
 921
 922         :param keyword: Keyword to process.
 923         :type keyword: Keyword
 924         :returns: Nothing.
 925         """
 926         pass
 927
 928     def visit_test_kw(self, test_kw):
 929         """Implements traversing through the test keyword and its child
 930         keywords.
 931
 932         :param test_kw: Keyword to process.
 933         :type test_kw: Keyword
 934         :returns: Nothing.
 935         """
 936         for keyword in test_kw.keywords:
 937             if self.start_test_kw(keyword) is not False:
 938                 self.visit_test_kw(keyword)
 939                 self.end_test_kw(keyword)
 940
 941     def start_test_kw(self, test_kw):
 942         """Called when test keyword starts. Default implementation does
 943         nothing.
 944
 945         :param test_kw: Keyword to process.
 946         :type test_kw: Keyword
 947         :returns: Nothing.
 948         """
 949         if test_kw.name.count("Show Runtime Counters On All Duts"):
 950             self._lookup_kw_nr += 1
 951             self._show_run_lookup_nr = 0
 952             self._msg_type = "test-show-runtime"
 953         elif test_kw.name.count("Start The L2fwd Test") and not self._version:
 954             self._msg_type = "dpdk-version"
 955         else:
 956             return
 957         test_kw.messages.visit(self)
 958
 959     def end_test_kw(self, test_kw):
 960         """Called when keyword ends. Default implementation does nothing.
 961
 962         :param test_kw: Keyword to process.
 963         :type test_kw: Keyword
 964         :returns: Nothing.
 965         """
 966         pass
 967
 968     def visit_setup_kw(self, setup_kw):
 969         """Implements traversing through the teardown keyword and its child
 970         keywords.
 971
 972         :param setup_kw: Keyword to process.
 973         :type setup_kw: Keyword
 974         :returns: Nothing.
 975         """
 976         for keyword in setup_kw.keywords:
 977             if self.start_setup_kw(keyword) is not False:
 978                 self.visit_setup_kw(keyword)
 979                 self.end_setup_kw(keyword)
 980
 981     def start_setup_kw(self, setup_kw):
 982         """Called when teardown keyword starts. Default implementation does
 983         nothing.
 984
 985         :param setup_kw: Keyword to process.
 986         :type setup_kw: Keyword
 987         :returns: Nothing.
 988         """
 989         if setup_kw.name.count("Show Vpp Version On All Duts") \
 990                 and not self._version:
 991             self._msg_type = "vpp-version"
 992         elif setup_kw.name.count("Set Global Variable") \
 993                 and not self._timestamp:
 994             self._msg_type = "timestamp"
 995         elif setup_kw.name.count("Setup Framework") and not self._testbed:
 996             self._msg_type = "testbed"
 997         else:
 998             return
 999         setup_kw.messages.visit(self)
1000
1001     def end_setup_kw(self, setup_kw):
1002         """Called when keyword ends. Default implementation does nothing.
1003
1004         :param setup_kw: Keyword to process.
1005         :type setup_kw: Keyword
1006         :returns: Nothing.
1007         """
1008         pass
1009
1010     def visit_teardown_kw(self, teardown_kw):
1011         """Implements traversing through the teardown keyword and its child
1012         keywords.
1013
1014         :param teardown_kw: Keyword to process.
1015         :type teardown_kw: Keyword
1016         :returns: Nothing.
1017         """
1018         for keyword in teardown_kw.keywords:
1019             if self.start_teardown_kw(keyword) is not False:
1020                 self.visit_teardown_kw(keyword)
1021                 self.end_teardown_kw(keyword)
1022
1023     def start_teardown_kw(self, teardown_kw):
1024         """Called when teardown keyword starts. Default implementation does
1025         nothing.
1026
1027         :param teardown_kw: Keyword to process.
1028         :type teardown_kw: Keyword
1029         :returns: Nothing.
1030         """
1031
1032         if teardown_kw.name.count("Show Vat History On All Duts"):
1033             self._conf_history_lookup_nr = 0
1034             self._msg_type = "teardown-vat-history"
1035             teardown_kw.messages.visit(self)
1036         elif teardown_kw.name.count("Show Papi History On All Duts"):
1037             self._conf_history_lookup_nr = 0
1038             self._msg_type = "teardown-papi-history"
1039             teardown_kw.messages.visit(self)
1040
1041     def end_teardown_kw(self, teardown_kw):
1042         """Called when keyword ends. Default implementation does nothing.
1043
1044         :param teardown_kw: Keyword to process.
1045         :type teardown_kw: Keyword
1046         :returns: Nothing.
1047         """
1048         pass
1049
1050     def visit_message(self, msg):
1051         """Implements visiting the message.
1052
1053         :param msg: Message to process.
1054         :type msg: Message
1055         :returns: Nothing.
1056         """
1057         if self.start_message(msg) is not False:
1058             self.end_message(msg)
1059
1060     def start_message(self, msg):
1061         """Called when message starts. Get required information from messages:
1062         - VPP version.
1063
1064         :param msg: Message to process.
1065         :type msg: Message
1066         :returns: Nothing.
1067         """
1068
1069         if self._msg_type:
1070             self.parse_msg[self._msg_type](msg)
1071
1072     def end_message(self, msg):
1073         """Called when message ends. Default implementation does nothing.
1074
1075         :param msg: Message to process.
1076         :type msg: Message
1077         :returns: Nothing.
1078         """
1079         pass
1080
1081
1082 class InputData(object):
1083     """Input data
1084
1085     The data is extracted from output.xml files generated by Jenkins jobs and
1086     stored in pandas' DataFrames.
1087
1088     The data structure:
1089     - job name
1090       - build number
1091         - metadata
1092           (as described in ExecutionChecker documentation)
1093         - suites
1094           (as described in ExecutionChecker documentation)
1095         - tests
1096           (as described in ExecutionChecker documentation)
1097     """
1098
1099     def __init__(self, spec):
1100         """Initialization.
1101
1102         :param spec: Specification.
1103         :type spec: Specification
1104         """
1105
1106         # Specification:
1107         self._cfg = spec
1108
1109         # Data store:
1110         self._input_data = pd.Series()
1111
1112     @property
1113     def data(self):
1114         """Getter - Input data.
1115
1116         :returns: Input data
1117         :rtype: pandas.Series
1118         """
1119         return self._input_data
1120
1121     def metadata(self, job, build):
1122         """Getter - metadata
1123
1124         :param job: Job which metadata we want.
1125         :param build: Build which metadata we want.
1126         :type job: str
1127         :type build: str
1128         :returns: Metadata
1129         :rtype: pandas.Series
1130         """
1131
1132         return self.data[job][build]["metadata"]
1133
1134     def suites(self, job, build):
1135         """Getter - suites
1136
1137         :param job: Job which suites we want.
1138         :param build: Build which suites we want.
1139         :type job: str
1140         :type build: str
1141         :returns: Suites.
1142         :rtype: pandas.Series
1143         """
1144
1145         return self.data[job][str(build)]["suites"]
1146
1147     def tests(self, job, build):
1148         """Getter - tests
1149
1150         :param job: Job which tests we want.
1151         :param build: Build which tests we want.
1152         :type job: str
1153         :type build: str
1154         :returns: Tests.
1155         :rtype: pandas.Series
1156         """
1157
1158         return self.data[job][build]["tests"]
1159
1160     def _parse_tests(self, job, build, log):
1161         """Process data from robot output.xml file and return JSON structured
1162         data.
1163
1164         :param job: The name of job which build output data will be processed.
1165         :param build: The build which output data will be processed.
1166         :param log: List of log messages.
1167         :type job: str
1168         :type build: dict
1169         :type log: list of tuples (severity, msg)
1170         :returns: JSON data structure.
1171         :rtype: dict
1172         """
1173
1174         metadata = {
1175             "job": job,
1176             "build": build
1177         }
1178
1179         with open(build["file-name"], 'r') as data_file:
1180             try:
1181                 result = ExecutionResult(data_file)
1182             except errors.DataError as err:
1183                 log.append(("ERROR", "Error occurred while parsing output.xml: "
1184                                      "{0}".format(err)))
1185                 return None
1186         checker = ExecutionChecker(metadata, self._cfg.mapping,
1187                                    self._cfg.ignore)
1188         result.visit(checker)
1189
1190         return checker.data
1191
1192     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1193         """Download and parse the input data file.
1194
1195         :param pid: PID of the process executing this method.
1196         :param job: Name of the Jenkins job which generated the processed input
1197             file.
1198         :param build: Information about the Jenkins build which generated the
1199             processed input file.
1200         :param repeat: Repeat the download specified number of times if not
1201             successful.
1202         :type pid: int
1203         :type job: str
1204         :type build: dict
1205         :type repeat: int
1206         """
1207
1208         logs = list()
1209
1210         logs.append(("INFO", "  Processing the job/build: {0}: {1}".
1211                      format(job, build["build"])))
1212
1213         state = "failed"
1214         success = False
1215         data = None
1216         do_repeat = repeat
1217         while do_repeat:
1218             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1219                                                    logs)
1220             if success:
1221                 break
1222             do_repeat -= 1
1223         if not success:
1224             logs.append(("ERROR", "It is not possible to download the input "
1225                                   "data file from the job '{job}', build "
1226                                   "'{build}', or it is damaged. Skipped.".
1227                          format(job=job, build=build["build"])))
1228         if success:
1229             logs.append(("INFO", "    Processing data from the build '{0}' ...".
1230                          format(build["build"])))
1231             data = self._parse_tests(job, build, logs)
1232             if data is None:
1233                 logs.append(("ERROR", "Input data file from the job '{job}', "
1234                                       "build '{build}' is damaged. Skipped.".
1235                              format(job=job, build=build["build"])))
1236             else:
1237                 state = "processed"
1238
1239             try:
1240                 remove(build["file-name"])
1241             except OSError as err:
1242                 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
1243                              format(build["file-name"], repr(err))))
1244
1245         # If the time-period is defined in the specification file, remove all
1246         # files which are outside the time period.
1247         timeperiod = self._cfg.input.get("time-period", None)
1248         if timeperiod and data:
1249             now = dt.utcnow()
1250             timeperiod = timedelta(int(timeperiod))
1251             metadata = data.get("metadata", None)
1252             if metadata:
1253                 generated = metadata.get("generated", None)
1254                 if generated:
1255                     generated = dt.strptime(generated, "%Y%m%d %H:%M")
1256                     if (now - generated) > timeperiod:
1257                         # Remove the data and the file:
1258                         state = "removed"
1259                         data = None
1260                         logs.append(
1261                             ("INFO",
1262                              "    The build {job}/{build} is outdated, will be "
1263                              "removed".format(job=job, build=build["build"])))
1264                         file_name = self._cfg.input["file-name"]
1265                         full_name = join(
1266                             self._cfg.environment["paths"]["DIR[WORKING,DATA]"],
1267                             "{job}{sep}{build}{sep}{name}".format(
1268                                 job=job,
1269                                 sep=SEPARATOR,
1270                                 build=build["build"],
1271                                 name=file_name))
1272                         try:
1273                             remove(full_name)
1274                             logs.append(("INFO",
1275                                          "    The file {name} has been removed".
1276                                          format(name=full_name)))
1277                         except OSError as err:
1278                             logs.append(("ERROR",
1279                                          "Cannot remove the file '{0}': {1}".
1280                                          format(full_name, repr(err))))
1281         logs.append(("INFO", "  Done."))
1282
1283         for level, line in logs:
1284             if level == "INFO":
1285                 logging.info(line)
1286             elif level == "ERROR":
1287                 logging.error(line)
1288             elif level == "DEBUG":
1289                 logging.debug(line)
1290             elif level == "CRITICAL":
1291                 logging.critical(line)
1292             elif level == "WARNING":
1293                 logging.warning(line)
1294
1295         return {"data": data, "state": state, "job": job, "build": build}
1296
1297     def download_and_parse_data(self, repeat=1):
1298         """Download the input data files, parse input data from input files and
1299         store in pandas' Series.
1300
1301         :param repeat: Repeat the download specified number of times if not
1302             successful.
1303         :type repeat: int
1304         """
1305
1306         logging.info("Downloading and parsing input files ...")
1307
1308         for job, builds in self._cfg.builds.items():
1309             for build in builds:
1310
1311                 result = self._download_and_parse_build(job, build, repeat)
1312                 build_nr = result["build"]["build"]
1313
1314                 if result["data"]:
1315                     data = result["data"]
1316                     build_data = pd.Series({
1317                         "metadata": pd.Series(
1318                             data["metadata"].values(),
1319                             index=data["metadata"].keys()),
1320                         "suites": pd.Series(data["suites"].values(),
1321                                             index=data["suites"].keys()),
1322                         "tests": pd.Series(data["tests"].values(),
1323                                            index=data["tests"].keys())})
1324
1325                     if self._input_data.get(job, None) is None:
1326                         self._input_data[job] = pd.Series()
1327                     self._input_data[job][str(build_nr)] = build_data
1328
1329                     self._cfg.set_input_file_name(
1330                         job, build_nr, result["build"]["file-name"])
1331
1332                 self._cfg.set_input_state(job, build_nr, result["state"])
1333
1334                 logging.info("ru_maxrss = {0}".format(
1335                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))
1336
1337                 logging.info(objgraph.most_common_types())
1338
1339         logging.info("Done.")
1340
1341     @staticmethod
1342     def _end_of_tag(tag_filter, start=0, closer="'"):
1343         """Return the index of character in the string which is the end of tag.
1344
1345         :param tag_filter: The string where the end of tag is being searched.
1346         :param start: The index where the searching is stated.
1347         :param closer: The character which is the tag closer.
1348         :type tag_filter: str
1349         :type start: int
1350         :type closer: str
1351         :returns: The index of the tag closer.
1352         :rtype: int
1353         """
1354
1355         try:
1356             idx_opener = tag_filter.index(closer, start)
1357             return tag_filter.index(closer, idx_opener + 1)
1358         except ValueError:
1359             return None
1360
1361     @staticmethod
1362     def _condition(tag_filter):
1363         """Create a conditional statement from the given tag filter.
1364
1365         :param tag_filter: Filter based on tags from the element specification.
1366         :type tag_filter: str
1367         :returns: Conditional statement which can be evaluated.
1368         :rtype: str
1369         """
1370
1371         index = 0
1372         while True:
1373             index = InputData._end_of_tag(tag_filter, index)
1374             if index is None:
1375                 return tag_filter
1376             index += 1
1377             tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
1378
1379     def filter_data(self, element, params=None, data_set="tests",
1380                     continue_on_error=False):
1381         """Filter required data from the given jobs and builds.
1382
1383         The output data structure is:
1384
1385         - job 1
1386           - build 1
1387             - test (or suite) 1 ID:
1388               - param 1
1389               - param 2
1390               ...
1391               - param n
1392             ...
1393             - test (or suite) n ID:
1394             ...
1395           ...
1396           - build n
1397         ...
1398         - job n
1399
1400         :param element: Element which will use the filtered data.
1401         :param params: Parameters which will be included in the output. If None,
1402         all parameters are included.
1403         :param data_set: The set of data to be filtered: tests, suites,
1404         metadata.
1405         :param continue_on_error: Continue if there is error while reading the
1406         data. The Item will be empty then
1407         :type element: pandas.Series
1408         :type params: list
1409         :type data_set: str
1410         :type continue_on_error: bool
1411         :returns: Filtered data.
1412         :rtype pandas.Series
1413         """
1414
1415         try:
1416             if element["filter"] in ("all", "template"):
1417                 cond = "True"
1418             else:
1419                 cond = InputData._condition(element["filter"])
1420             logging.debug("   Filter: {0}".format(cond))
1421         except KeyError:
1422             logging.error("  No filter defined.")
1423             return None
1424
1425         if params is None:
1426             params = element.get("parameters", None)
1427             if params:
1428                 params.append("type")
1429
1430         data = pd.Series()
1431         try:
1432             for job, builds in element["data"].items():
1433                 data[job] = pd.Series()
1434                 for build in builds:
1435                     data[job][str(build)] = pd.Series()
1436                     try:
1437                         data_iter = self.data[job][str(build)][data_set].\
1438                             iteritems()
1439                     except KeyError:
1440                         if continue_on_error:
1441                             continue
1442                         else:
1443                             return None
1444                     for test_ID, test_data in data_iter:
1445                         if eval(cond, {"tags": test_data.get("tags", "")}):
1446                             data[job][str(build)][test_ID] = pd.Series()
1447                             if params is None:
1448                                 for param, val in test_data.items():
1449                                     data[job][str(build)][test_ID][param] = val
1450                             else:
1451                                 for param in params:
1452                                     try:
1453                                         data[job][str(build)][test_ID][param] =\
1454                                             test_data[param]
1455                                     except KeyError:
1456                                         data[job][str(build)][test_ID][param] =\
1457                                             "No Data"
1458             return data
1459
1460         except (KeyError, IndexError, ValueError) as err:
1461             logging.error("   Missing mandatory parameter in the element "
1462                           "specification: {0}".format(err))
1463             return None
1464         except AttributeError:
1465             return None
1466         except SyntaxError:
1467             logging.error("   The filter '{0}' is not correct. Check if all "
1468                           "tags are enclosed by apostrophes.".format(cond))
1469             return None
1470
1471     @staticmethod
1472     def merge_data(data):
1473         """Merge data from more jobs and builds to a simple data structure.
1474
1475         The output data structure is:
1476
1477         - test (suite) 1 ID:
1478           - param 1
1479           - param 2
1480           ...
1481           - param n
1482         ...
1483         - test (suite) n ID:
1484         ...
1485
1486         :param data: Data to merge.
1487         :type data: pandas.Series
1488         :returns: Merged data.
1489         :rtype: pandas.Series
1490         """
1491
1492         logging.info("    Merging data ...")
1493
1494         merged_data = pd.Series()
1495         for _, builds in data.iteritems():
1496             for _, item in builds.iteritems():
1497                 for ID, item_data in item.iteritems():
1498                     merged_data[ID] = item_data
1499
1500         return merged_data