resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import resource
  24 import pandas as pd
  25 import logging
  26
  27 from robot.api import ExecutionResult, ResultVisitor
  28 from robot import errors
  29 from collections import OrderedDict
  30 from string import replace
  31 from os import remove
  32 from os.path import join
  33 from datetime import datetime as dt
  34 from datetime import timedelta
  35 from json import loads
  36 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
  37
  38 from input_data_files import download_and_unzip_data_file
  39
  40
  41 # Separator used in file names
  42 SEPARATOR = "__"
  43
  44
  45 class ExecutionChecker(ResultVisitor):
  46     """Class to traverse through the test suite structure.
  47
  48     The functionality implemented in this class generates a json structure:
  49
  50     Performance tests:
  51
  52     {
  53         "metadata": {
  54             "generated": "Timestamp",
  55             "version": "SUT version",
  56             "job": "Jenkins job name",
  57             "build": "Information about the build"
  58         },
  59         "suites": {
  60             "Suite long name 1": {
  61                 "name": Suite name,
  62                 "doc": "Suite 1 documentation",
  63                 "parent": "Suite 1 parent",
  64                 "level": "Level of the suite in the suite hierarchy"
  65             }
  66             "Suite long name N": {
  67                 "name": Suite name,
  68                 "doc": "Suite N documentation",
  69                 "parent": "Suite 2 parent",
  70                 "level": "Level of the suite in the suite hierarchy"
  71             }
  72         }
  73         "tests": {
  74             # NDRPDR tests:
  75             "ID": {
  76                 "name": "Test name",
  77                 "parent": "Name of the parent of the test",
  78                 "doc": "Test documentation",
  79                 "msg": "Test message",
  80                 "conf-history": "DUT1 and DUT2 VAT History",
  81                 "show-run": "Show Run",
  82                 "tags": ["tag 1", "tag 2", "tag n"],
  83                 "type": "NDRPDR",
  84                 "status": "PASS" | "FAIL",
  85                 "throughput": {
  86                     "NDR": {
  87                         "LOWER": float,
  88                         "UPPER": float
  89                     },
  90                     "PDR": {
  91                         "LOWER": float,
  92                         "UPPER": float
  93                     }
  94                 },
  95                 "latency": {
  96                     "NDR": {
  97                         "direction1": {
  98                             "min": float,
  99                             "avg": float,
 100                             "max": float
 101                         },
 102                         "direction2": {
 103                             "min": float,
 104                             "avg": float,
 105                             "max": float
 106                         }
 107                     },
 108                     "PDR": {
 109                         "direction1": {
 110                             "min": float,
 111                             "avg": float,
 112                             "max": float
 113                         },
 114                         "direction2": {
 115                             "min": float,
 116                             "avg": float,
 117                             "max": float
 118                         }
 119                     }
 120                 }
 121             }
 122
 123             # TCP tests:
 124             "ID": {
 125                 "name": "Test name",
 126                 "parent": "Name of the parent of the test",
 127                 "doc": "Test documentation",
 128                 "msg": "Test message",
 129                 "tags": ["tag 1", "tag 2", "tag n"],
 130                 "type": "TCP",
 131                 "status": "PASS" | "FAIL",
 132                 "result": int
 133             }
 134
 135             # MRR, BMRR tests:
 136             "ID": {
 137                 "name": "Test name",
 138                 "parent": "Name of the parent of the test",
 139                 "doc": "Test documentation",
 140                 "msg": "Test message",
 141                 "tags": ["tag 1", "tag 2", "tag n"],
 142                 "type": "MRR" | "BMRR",
 143                 "status": "PASS" | "FAIL",
 144                 "result": {
 145                     "receive-rate": AvgStdevMetadata,
 146                 }
 147             }
 148
 149             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 150             # NDRPDRDISC tests:
 151             "ID": {
 152                 "name": "Test name",
 153                 "parent": "Name of the parent of the test",
 154                 "doc": "Test documentation",
 155                 "msg": "Test message",
 156                 "tags": ["tag 1", "tag 2", "tag n"],
 157                 "type": "PDR" | "NDR",
 158                 "status": "PASS" | "FAIL",
 159                 "throughput": {  # Only type: "PDR" | "NDR"
 160                     "value": int,
 161                     "unit": "pps" | "bps" | "percentage"
 162                 },
 163                 "latency": {  # Only type: "PDR" | "NDR"
 164                     "direction1": {
 165                         "100": {
 166                             "min": int,
 167                             "avg": int,
 168                             "max": int
 169                         },
 170                         "50": {  # Only for NDR
 171                             "min": int,
 172                             "avg": int,
 173                             "max": int
 174                         },
 175                         "10": {  # Only for NDR
 176                             "min": int,
 177                             "avg": int,
 178                             "max": int
 179                         }
 180                     },
 181                     "direction2": {
 182                         "100": {
 183                             "min": int,
 184                             "avg": int,
 185                             "max": int
 186                         },
 187                         "50": {  # Only for NDR
 188                             "min": int,
 189                             "avg": int,
 190                             "max": int
 191                         },
 192                         "10": {  # Only for NDR
 193                             "min": int,
 194                             "avg": int,
 195                             "max": int
 196                         }
 197                     }
 198                 },
 199                 "lossTolerance": "lossTolerance",  # Only type: "PDR"
 200                 "conf-history": "DUT1 and DUT2 VAT History"
 201                 "show-run": "Show Run"
 202             },
 203             "ID" {
 204                 # next test
 205             }
 206         }
 207     }
 208
 209
 210     Functional tests:
 211
 212     {
 213         "metadata": {  # Optional
 214             "version": "VPP version",
 215             "job": "Jenkins job name",
 216             "build": "Information about the build"
 217         },
 218         "suites": {
 219             "Suite name 1": {
 220                 "doc": "Suite 1 documentation",
 221                 "parent": "Suite 1 parent",
 222                 "level": "Level of the suite in the suite hierarchy"
 223             }
 224             "Suite name N": {
 225                 "doc": "Suite N documentation",
 226                 "parent": "Suite 2 parent",
 227                 "level": "Level of the suite in the suite hierarchy"
 228             }
 229         }
 230         "tests": {
 231             "ID": {
 232                 "name": "Test name",
 233                 "parent": "Name of the parent of the test",
 234                 "doc": "Test documentation"
 235                 "msg": "Test message"
 236                 "tags": ["tag 1", "tag 2", "tag n"],
 237                 "conf-history": "DUT1 and DUT2 VAT History"
 238                 "show-run": "Show Run"
 239                 "status": "PASS" | "FAIL"
 240             },
 241             "ID" {
 242                 # next test
 243             }
 244         }
 245     }
 246
 247     .. note:: ID is the lowercase full path to the test.
 248     """
 249
 250     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 251     REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
 252
 253     REGEX_PLR_RATE = re.compile(r'PLRsearch lower bound::\s(\d+.\d+).*\n'
 254                                 r'PLRsearch upper bound::\s(\d+.\d+)')
 255
 256     REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 257                                    r'NDR_UPPER:\s(\d+.\d+).*\n'
 258                                    r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 259                                    r'PDR_UPPER:\s(\d+.\d+)')
 260
 261     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 262     REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
 263                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 264                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
 265                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 266                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
 267                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 268                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
 269
 270     REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
 271                                r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 272                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
 273
 274     REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 275                                   r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
 276
 277     REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
 278                                  r'[\D\d]*')
 279
 280     REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|"
 281                                    r"VPP Version:\s*|VPP version:\s*)(.*)")
 282
 283     REGEX_VERSION_DPDK = re.compile(r"(return STDOUT testpmd)([\d\D\n]*)"
 284                                     r"(RTE Version: 'DPDK )(.*)(')")
 285
 286     REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
 287
 288     REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 289                            r'tx\s(\d*),\srx\s(\d*)')
 290
 291     REGEX_BMRR = re.compile(r'Maximum Receive Rate trial results'
 292                             r' in packets per second: \[(.*)\]')
 293
 294     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 295
 296     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 297
 298     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 299
 300     REGEX_TC_NUMBER = re.compile(r'tc[0-9]{2}-')
 301
 302     def __init__(self, metadata, mapping, ignore):
 303         """Initialisation.
 304
 305         :param metadata: Key-value pairs to be included in "metadata" part of
 306             JSON structure.
 307         :param mapping: Mapping of the old names of test cases to the new
 308             (actual) one.
 309         :param ignore: List of TCs to be ignored.
 310         :type metadata: dict
 311         :type mapping: dict
 312         :type ignore: list
 313         """
 314
 315         # Type of message to parse out from the test messages
 316         self._msg_type = None
 317
 318         # VPP version
 319         self._version = None
 320
 321         # Timestamp
 322         self._timestamp = None
 323
 324         # Testbed. The testbed is identified by TG node IP address.
 325         self._testbed = None
 326
 327         # Mapping of TCs long names
 328         self._mapping = mapping
 329
 330         # Ignore list
 331         self._ignore = ignore
 332
 333         # Number of VAT History messages found:
 334         # 0 - no message
 335         # 1 - VAT History of DUT1
 336         # 2 - VAT History of DUT2
 337         self._lookup_kw_nr = 0
 338         self._conf_history_lookup_nr = 0
 339
 340         # Number of Show Running messages found
 341         # 0 - no message
 342         # 1 - Show run message found
 343         self._show_run_lookup_nr = 0
 344
 345         # Test ID of currently processed test- the lowercase full path to the
 346         # test
 347         self._test_ID = None
 348
 349         # The main data structure
 350         self._data = {
 351             "metadata": OrderedDict(),
 352             "suites": OrderedDict(),
 353             "tests": OrderedDict()
 354         }
 355
 356         # Save the provided metadata
 357         for key, val in metadata.items():
 358             self._data["metadata"][key] = val
 359
 360         # Dictionary defining the methods used to parse different types of
 361         # messages
 362         self.parse_msg = {
 363             "timestamp": self._get_timestamp,
 364             "vpp-version": self._get_vpp_version,
 365             "dpdk-version": self._get_dpdk_version,
 366             "teardown-vat-history": self._get_vat_history,
 367             "teardown-papi-history": self._get_papi_history,
 368             "test-show-runtime": self._get_show_run,
 369             "testbed": self._get_testbed
 370         }
 371
 372     @property
 373     def data(self):
 374         """Getter - Data parsed from the XML file.
 375
 376         :returns: Data parsed from the XML file.
 377         :rtype: dict
 378         """
 379         return self._data
 380
 381     def _get_testbed(self, msg):
 382         """Called when extraction of testbed IP is required.
 383         The testbed is identified by TG node IP address.
 384
 385         :param msg: Message to process.
 386         :type msg: Message
 387         :returns: Nothing.
 388         """
 389
 390         if msg.message.count("Arguments:"):
 391             message = str(msg.message).replace(' ', '').replace('\n', '').\
 392                 replace("'", '"').replace('b"', '"').\
 393                 replace("honeycom", "honeycomb")
 394             message = loads(message[11:-1])
 395             try:
 396                 self._testbed = message["TG"]["host"]
 397             except (KeyError, ValueError):
 398                 pass
 399             finally:
 400                 self._data["metadata"]["testbed"] = self._testbed
 401                 self._msg_type = None
 402
 403     def _get_vpp_version(self, msg):
 404         """Called when extraction of VPP version is required.
 405
 406         :param msg: Message to process.
 407         :type msg: Message
 408         :returns: Nothing.
 409         """
 410
 411         if msg.message.count("return STDOUT Version:") or \
 412             msg.message.count("VPP Version:") or \
 413             msg.message.count("VPP version:"):
 414             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 415                                 group(2))
 416             self._data["metadata"]["version"] = self._version
 417             self._msg_type = None
 418
 419     def _get_dpdk_version(self, msg):
 420         """Called when extraction of DPDK version is required.
 421
 422         :param msg: Message to process.
 423         :type msg: Message
 424         :returns: Nothing.
 425         """
 426
 427         if msg.message.count("return STDOUT testpmd"):
 428             try:
 429                 self._version = str(re.search(
 430                     self.REGEX_VERSION_DPDK, msg.message). group(4))
 431                 self._data["metadata"]["version"] = self._version
 432             except IndexError:
 433                 pass
 434             finally:
 435                 self._msg_type = None
 436
 437     def _get_timestamp(self, msg):
 438         """Called when extraction of timestamp is required.
 439
 440         :param msg: Message to process.
 441         :type msg: Message
 442         :returns: Nothing.
 443         """
 444
 445         self._timestamp = msg.timestamp[:14]
 446         self._data["metadata"]["generated"] = self._timestamp
 447         self._msg_type = None
 448
 449     def _get_vat_history(self, msg):
 450         """Called when extraction of VAT command history is required.
 451
 452         :param msg: Message to process.
 453         :type msg: Message
 454         :returns: Nothing.
 455         """
 456         if msg.message.count("VAT command history:"):
 457             self._conf_history_lookup_nr += 1
 458             if self._conf_history_lookup_nr == 1:
 459                 self._data["tests"][self._test_ID]["conf-history"] = str()
 460             else:
 461                 self._msg_type = None
 462             text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
 463                           "VAT command history:", "", msg.message, count=1). \
 464                 replace("\n\n", "\n").replace('\n', ' |br| ').\
 465                 replace('\r', '').replace('"', "'")
 466
 467             self._data["tests"][self._test_ID]["conf-history"] += " |br| "
 468             self._data["tests"][self._test_ID]["conf-history"] += \
 469                 "**DUT" + str(self._conf_history_lookup_nr) + ":** " + text
 470
 471     def _get_papi_history(self, msg):
 472         """Called when extraction of PAPI command history is required.
 473
 474         :param msg: Message to process.
 475         :type msg: Message
 476         :returns: Nothing.
 477         """
 478         if msg.message.count("PAPI command history:"):
 479             self._conf_history_lookup_nr += 1
 480             if self._conf_history_lookup_nr == 1:
 481                 self._data["tests"][self._test_ID]["conf-history"] = str()
 482             else:
 483                 self._msg_type = None
 484             text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
 485                           "PAPI command history:", "", msg.message, count=1). \
 486                 replace("\n\n", "\n").replace('\n', ' |br| ').\
 487                 replace('\r', '').replace('"', "'")
 488
 489             self._data["tests"][self._test_ID]["conf-history"] += " |br| "
 490             self._data["tests"][self._test_ID]["conf-history"] += \
 491                 "**DUT" + str(self._conf_history_lookup_nr) + ":** " + text
 492
 493     def _get_show_run(self, msg):
 494         """Called when extraction of VPP operational data (output of CLI command
 495         Show Runtime) is required.
 496
 497         :param msg: Message to process.
 498         :type msg: Message
 499         :returns: Nothing.
 500         """
 501         if msg.message.count("Thread 0 vpp_main"):
 502             self._show_run_lookup_nr += 1
 503             if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
 504                 self._data["tests"][self._test_ID]["show-run"] = str()
 505             if self._lookup_kw_nr > 1:
 506                 self._msg_type = None
 507             if self._show_run_lookup_nr == 1:
 508                 text = msg.message.replace("vat# ", "").\
 509                     replace("return STDOUT ", "").replace("\n\n", "\n").\
 510                     replace('\n', ' |br| ').\
 511                     replace('\r', '').replace('"', "'")
 512                 try:
 513                     self._data["tests"][self._test_ID]["show-run"] += " |br| "
 514                     self._data["tests"][self._test_ID]["show-run"] += \
 515                         "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
 516                 except KeyError:
 517                     pass
 518
 519     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 520     def _get_latency(self, msg, test_type):
 521         """Get the latency data from the test message.
 522
 523         :param msg: Message to be parsed.
 524         :param test_type: Type of the test - NDR or PDR.
 525         :type msg: str
 526         :type test_type: str
 527         :returns: Latencies parsed from the message.
 528         :rtype: dict
 529         """
 530
 531         if test_type == "NDR":
 532             groups = re.search(self.REGEX_LAT_NDR, msg)
 533             groups_range = range(1, 7)
 534         elif test_type == "PDR":
 535             groups = re.search(self.REGEX_LAT_PDR, msg)
 536             groups_range = range(1, 3)
 537         else:
 538             return {}
 539
 540         latencies = list()
 541         for idx in groups_range:
 542             try:
 543                 lat = [int(item) for item in str(groups.group(idx)).split('/')]
 544             except (AttributeError, ValueError):
 545                 lat = [-1, -1, -1]
 546             latencies.append(lat)
 547
 548         keys = ("min", "avg", "max")
 549         latency = {
 550             "direction1": {
 551             },
 552             "direction2": {
 553             }
 554         }
 555
 556         latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
 557         latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
 558         if test_type == "NDR":
 559             latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
 560             latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
 561             latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
 562             latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
 563
 564         return latency
 565
 566     def _get_ndrpdr_throughput(self, msg):
 567         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 568         message.
 569
 570         :param msg: The test message to be parsed.
 571         :type msg: str
 572         :returns: Parsed data as a dict and the status (PASS/FAIL).
 573         :rtype: tuple(dict, str)
 574         """
 575
 576         throughput = {
 577             "NDR": {"LOWER": -1.0, "UPPER": -1.0},
 578             "PDR": {"LOWER": -1.0, "UPPER": -1.0}
 579         }
 580         status = "FAIL"
 581         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 582
 583         if groups is not None:
 584             try:
 585                 throughput["NDR"]["LOWER"] = float(groups.group(1))
 586                 throughput["NDR"]["UPPER"] = float(groups.group(2))
 587                 throughput["PDR"]["LOWER"] = float(groups.group(3))
 588                 throughput["PDR"]["UPPER"] = float(groups.group(4))
 589                 status = "PASS"
 590             except (IndexError, ValueError):
 591                 pass
 592
 593         return throughput, status
 594
 595     def _get_plr_throughput(self, msg):
 596         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 597         message.
 598
 599         :param msg: The test message to be parsed.
 600         :type msg: str
 601         :returns: Parsed data as a dict and the status (PASS/FAIL).
 602         :rtype: tuple(dict, str)
 603         """
 604
 605         throughput = {
 606             "LOWER": -1.0,
 607             "UPPER": -1.0
 608         }
 609         status = "FAIL"
 610         groups = re.search(self.REGEX_PLR_RATE, msg)
 611
 612         if groups is not None:
 613             try:
 614                 throughput["LOWER"] = float(groups.group(1))
 615                 throughput["UPPER"] = float(groups.group(2))
 616                 status = "PASS"
 617             except (IndexError, ValueError):
 618                 pass
 619
 620         return throughput, status
 621
 622     def _get_ndrpdr_latency(self, msg):
 623         """Get LATENCY from the test message.
 624
 625         :param msg: The test message to be parsed.
 626         :type msg: str
 627         :returns: Parsed data as a dict and the status (PASS/FAIL).
 628         :rtype: tuple(dict, str)
 629         """
 630
 631         latency = {
 632             "NDR": {
 633                 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
 634                 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
 635             },
 636             "PDR": {
 637                 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
 638                 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
 639             }
 640         }
 641         status = "FAIL"
 642         groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 643
 644         if groups is not None:
 645             keys = ("min", "avg", "max")
 646             try:
 647                 latency["NDR"]["direction1"] = dict(
 648                     zip(keys, [float(l) for l in groups.group(1).split('/')]))
 649                 latency["NDR"]["direction2"] = dict(
 650                     zip(keys, [float(l) for l in groups.group(2).split('/')]))
 651                 latency["PDR"]["direction1"] = dict(
 652                     zip(keys, [float(l) for l in groups.group(3).split('/')]))
 653                 latency["PDR"]["direction2"] = dict(
 654                     zip(keys, [float(l) for l in groups.group(4).split('/')]))
 655                 status = "PASS"
 656             except (IndexError, ValueError):
 657                 pass
 658
 659         return latency, status
 660
 661     def visit_suite(self, suite):
 662         """Implements traversing through the suite and its direct children.
 663
 664         :param suite: Suite to process.
 665         :type suite: Suite
 666         :returns: Nothing.
 667         """
 668         if self.start_suite(suite) is not False:
 669             suite.suites.visit(self)
 670             suite.tests.visit(self)
 671             self.end_suite(suite)
 672
 673     def start_suite(self, suite):
 674         """Called when suite starts.
 675
 676         :param suite: Suite to process.
 677         :type suite: Suite
 678         :returns: Nothing.
 679         """
 680
 681         try:
 682             parent_name = suite.parent.name
 683         except AttributeError:
 684             return
 685
 686         doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
 687             replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
 688         doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
 689
 690         self._data["suites"][suite.longname.lower().replace('"', "'").
 691             replace(" ", "_")] = {
 692                 "name": suite.name.lower(),
 693                 "doc": doc_str,
 694                 "parent": parent_name,
 695                 "level": len(suite.longname.split("."))
 696             }
 697
 698         suite.keywords.visit(self)
 699
 700     def end_suite(self, suite):
 701         """Called when suite ends.
 702
 703         :param suite: Suite to process.
 704         :type suite: Suite
 705         :returns: Nothing.
 706         """
 707         pass
 708
 709     def visit_test(self, test):
 710         """Implements traversing through the test.
 711
 712         :param test: Test to process.
 713         :type test: Test
 714         :returns: Nothing.
 715         """
 716         if self.start_test(test) is not False:
 717             test.keywords.visit(self)
 718             self.end_test(test)
 719
 720     def start_test(self, test):
 721         """Called when test starts.
 722
 723         :param test: Test to process.
 724         :type test: Test
 725         :returns: Nothing.
 726         """
 727
 728         longname_orig = test.longname.lower()
 729
 730         # Check the ignore list
 731         if longname_orig in self._ignore:
 732             return
 733
 734         tags = [str(tag) for tag in test.tags]
 735         test_result = dict()
 736
 737         # Change the TC long name and name if defined in the mapping table
 738         longname = self._mapping.get(longname_orig, None)
 739         if longname is not None:
 740             name = longname.split('.')[-1]
 741             logging.debug("{0}\n{1}\n{2}\n{3}".format(
 742                 self._data["metadata"], longname_orig, longname, name))
 743         else:
 744             longname = longname_orig
 745             name = test.name.lower()
 746
 747         # Remove TC number from the TC long name (backward compatibility):
 748         self._test_ID = re.sub(self.REGEX_TC_NUMBER, "", longname)
 749         # Remove TC number from the TC name (not needed):
 750         test_result["name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
 751
 752         test_result["parent"] = test.parent.name.lower()
 753         test_result["tags"] = tags
 754         doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
 755             replace('\r', '').replace('[', ' |br| [')
 756         test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
 757         test_result["msg"] = test.message.replace('\n', ' |br| '). \
 758             replace('\r', '').replace('"', "'")
 759         test_result["type"] = "FUNC"
 760         test_result["status"] = test.status
 761
 762         if "PERFTEST" in tags:
 763             # Replace info about cores (e.g. -1c-) with the info about threads
 764             # and cores (e.g. -1t1c-) in the long test case names and in the
 765             # test case names if necessary.
 766             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_ID)
 767             if not groups:
 768                 tag_count = 0
 769                 for tag in test_result["tags"]:
 770                     groups = re.search(self.REGEX_TC_TAG, tag)
 771                     if groups:
 772                         tag_count += 1
 773                         tag_tc = tag
 774
 775                 if tag_count == 1:
 776                     self._test_ID = re.sub(self.REGEX_TC_NAME_NEW,
 777                                            "-{0}-".format(tag_tc.lower()),
 778                                            self._test_ID,
 779                                            count=1)
 780                     test_result["name"] = re.sub(self.REGEX_TC_NAME_NEW,
 781                                                  "-{0}-".format(tag_tc.lower()),
 782                                                  test_result["name"],
 783                                                  count=1)
 784                 else:
 785                     test_result["status"] = "FAIL"
 786                     self._data["tests"][self._test_ID] = test_result
 787                     logging.debug("The test '{0}' has no or more than one "
 788                                   "multi-threading tags.".format(self._test_ID))
 789                     logging.debug("Tags: {0}".format(test_result["tags"]))
 790                     return
 791
 792         if test.status == "PASS" and ("NDRPDRDISC" in tags or
 793                                       "NDRPDR" in tags or
 794                                       "SOAK" in tags or
 795                                       "TCP" in tags or
 796                                       "MRR" in tags or
 797                                       "BMRR" in tags):
 798             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 799             if "NDRDISC" in tags:
 800                 test_result["type"] = "NDR"
 801             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 802             elif "PDRDISC" in tags:
 803                 test_result["type"] = "PDR"
 804             elif "NDRPDR" in tags:
 805                 test_result["type"] = "NDRPDR"
 806             elif "SOAK" in tags:
 807                 test_result["type"] = "SOAK"
 808             elif "TCP" in tags:
 809                 test_result["type"] = "TCP"
 810             elif "MRR" in tags:
 811                 test_result["type"] = "MRR"
 812             elif "FRMOBL" in tags or "BMRR" in tags:
 813                 test_result["type"] = "BMRR"
 814             else:
 815                 test_result["status"] = "FAIL"
 816                 self._data["tests"][self._test_ID] = test_result
 817                 return
 818
 819             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 820             if test_result["type"] in ("NDR", "PDR"):
 821                 try:
 822                     rate_value = str(re.search(
 823                         self.REGEX_RATE, test.message).group(1))
 824                 except AttributeError:
 825                     rate_value = "-1"
 826                 try:
 827                     rate_unit = str(re.search(
 828                         self.REGEX_RATE, test.message).group(2))
 829                 except AttributeError:
 830                     rate_unit = "-1"
 831
 832                 test_result["throughput"] = dict()
 833                 test_result["throughput"]["value"] = \
 834                     int(rate_value.split('.')[0])
 835                 test_result["throughput"]["unit"] = rate_unit
 836                 test_result["latency"] = \
 837                     self._get_latency(test.message, test_result["type"])
 838                 if test_result["type"] == "PDR":
 839                     test_result["lossTolerance"] = str(re.search(
 840                         self.REGEX_TOLERANCE, test.message).group(1))
 841
 842             elif test_result["type"] in ("NDRPDR", ):
 843                 test_result["throughput"], test_result["status"] = \
 844                     self._get_ndrpdr_throughput(test.message)
 845                 test_result["latency"], test_result["status"] = \
 846                     self._get_ndrpdr_latency(test.message)
 847
 848             elif test_result["type"] in ("SOAK", ):
 849                 test_result["throughput"], test_result["status"] = \
 850                     self._get_plr_throughput(test.message)
 851
 852             elif test_result["type"] in ("TCP", ):
 853                 groups = re.search(self.REGEX_TCP, test.message)
 854                 test_result["result"] = int(groups.group(2))
 855
 856             elif test_result["type"] in ("MRR", "BMRR"):
 857                 test_result["result"] = dict()
 858                 groups = re.search(self.REGEX_BMRR, test.message)
 859                 if groups is not None:
 860                     items_str = groups.group(1)
 861                     items_float = [float(item.strip()) for item
 862                                    in items_str.split(",")]
 863                     metadata = AvgStdevMetadataFactory.from_data(items_float)
 864                     # Next two lines have been introduced in CSIT-1179,
 865                     # to be removed in CSIT-1180.
 866                     metadata.size = 1
 867                     metadata.stdev = 0.0
 868                     test_result["result"]["receive-rate"] = metadata
 869                 else:
 870                     groups = re.search(self.REGEX_MRR, test.message)
 871                     test_result["result"]["receive-rate"] = \
 872                         AvgStdevMetadataFactory.from_data([
 873                             float(groups.group(3)) / float(groups.group(1)), ])
 874
 875         self._data["tests"][self._test_ID] = test_result
 876
 877     def end_test(self, test):
 878         """Called when test ends.
 879
 880         :param test: Test to process.
 881         :type test: Test
 882         :returns: Nothing.
 883         """
 884         pass
 885
 886     def visit_keyword(self, keyword):
 887         """Implements traversing through the keyword and its child keywords.
 888
 889         :param keyword: Keyword to process.
 890         :type keyword: Keyword
 891         :returns: Nothing.
 892         """
 893         if self.start_keyword(keyword) is not False:
 894             self.end_keyword(keyword)
 895
 896     def start_keyword(self, keyword):
 897         """Called when keyword starts. Default implementation does nothing.
 898
 899         :param keyword: Keyword to process.
 900         :type keyword: Keyword
 901         :returns: Nothing.
 902         """
 903         try:
 904             if keyword.type == "setup":
 905                 self.visit_setup_kw(keyword)
 906             elif keyword.type == "teardown":
 907                 self._lookup_kw_nr = 0
 908                 self.visit_teardown_kw(keyword)
 909             else:
 910                 self._lookup_kw_nr = 0
 911                 self.visit_test_kw(keyword)
 912         except AttributeError:
 913             pass
 914
 915     def end_keyword(self, keyword):
 916         """Called when keyword ends. Default implementation does nothing.
 917
 918         :param keyword: Keyword to process.
 919         :type keyword: Keyword
 920         :returns: Nothing.
 921         """
 922         pass
 923
 924     def visit_test_kw(self, test_kw):
 925         """Implements traversing through the test keyword and its child
 926         keywords.
 927
 928         :param test_kw: Keyword to process.
 929         :type test_kw: Keyword
 930         :returns: Nothing.
 931         """
 932         for keyword in test_kw.keywords:
 933             if self.start_test_kw(keyword) is not False:
 934                 self.visit_test_kw(keyword)
 935                 self.end_test_kw(keyword)
 936
 937     def start_test_kw(self, test_kw):
 938         """Called when test keyword starts. Default implementation does
 939         nothing.
 940
 941         :param test_kw: Keyword to process.
 942         :type test_kw: Keyword
 943         :returns: Nothing.
 944         """
 945         if test_kw.name.count("Show Runtime Counters On All Duts"):
 946             self._lookup_kw_nr += 1
 947             self._show_run_lookup_nr = 0
 948             self._msg_type = "test-show-runtime"
 949         elif test_kw.name.count("Start The L2fwd Test") and not self._version:
 950             self._msg_type = "dpdk-version"
 951         else:
 952             return
 953         test_kw.messages.visit(self)
 954
 955     def end_test_kw(self, test_kw):
 956         """Called when keyword ends. Default implementation does nothing.
 957
 958         :param test_kw: Keyword to process.
 959         :type test_kw: Keyword
 960         :returns: Nothing.
 961         """
 962         pass
 963
 964     def visit_setup_kw(self, setup_kw):
 965         """Implements traversing through the teardown keyword and its child
 966         keywords.
 967
 968         :param setup_kw: Keyword to process.
 969         :type setup_kw: Keyword
 970         :returns: Nothing.
 971         """
 972         for keyword in setup_kw.keywords:
 973             if self.start_setup_kw(keyword) is not False:
 974                 self.visit_setup_kw(keyword)
 975                 self.end_setup_kw(keyword)
 976
 977     def start_setup_kw(self, setup_kw):
 978         """Called when teardown keyword starts. Default implementation does
 979         nothing.
 980
 981         :param setup_kw: Keyword to process.
 982         :type setup_kw: Keyword
 983         :returns: Nothing.
 984         """
 985         if setup_kw.name.count("Show Vpp Version On All Duts") \
 986                 and not self._version:
 987             self._msg_type = "vpp-version"
 988         elif setup_kw.name.count("Set Global Variable") \
 989                 and not self._timestamp:
 990             self._msg_type = "timestamp"
 991         elif setup_kw.name.count("Setup Framework") and not self._testbed:
 992             self._msg_type = "testbed"
 993         else:
 994             return
 995         setup_kw.messages.visit(self)
 996
 997     def end_setup_kw(self, setup_kw):
 998         """Called when keyword ends. Default implementation does nothing.
 999
1000         :param setup_kw: Keyword to process.
1001         :type setup_kw: Keyword
1002         :returns: Nothing.
1003         """
1004         pass
1005
1006     def visit_teardown_kw(self, teardown_kw):
1007         """Implements traversing through the teardown keyword and its child
1008         keywords.
1009
1010         :param teardown_kw: Keyword to process.
1011         :type teardown_kw: Keyword
1012         :returns: Nothing.
1013         """
1014         for keyword in teardown_kw.keywords:
1015             if self.start_teardown_kw(keyword) is not False:
1016                 self.visit_teardown_kw(keyword)
1017                 self.end_teardown_kw(keyword)
1018
1019     def start_teardown_kw(self, teardown_kw):
1020         """Called when teardown keyword starts. Default implementation does
1021         nothing.
1022
1023         :param teardown_kw: Keyword to process.
1024         :type teardown_kw: Keyword
1025         :returns: Nothing.
1026         """
1027
1028         if teardown_kw.name.count("Show Vat History On All Duts"):
1029             self._conf_history_lookup_nr = 0
1030             self._msg_type = "teardown-vat-history"
1031             teardown_kw.messages.visit(self)
1032         elif teardown_kw.name.count("Show Papi History On All Duts"):
1033             self._conf_history_lookup_nr = 0
1034             self._msg_type = "teardown-papi-history"
1035             teardown_kw.messages.visit(self)
1036
1037     def end_teardown_kw(self, teardown_kw):
1038         """Called when keyword ends. Default implementation does nothing.
1039
1040         :param teardown_kw: Keyword to process.
1041         :type teardown_kw: Keyword
1042         :returns: Nothing.
1043         """
1044         pass
1045
1046     def visit_message(self, msg):
1047         """Implements visiting the message.
1048
1049         :param msg: Message to process.
1050         :type msg: Message
1051         :returns: Nothing.
1052         """
1053         if self.start_message(msg) is not False:
1054             self.end_message(msg)
1055
1056     def start_message(self, msg):
1057         """Called when message starts. Get required information from messages:
1058         - VPP version.
1059
1060         :param msg: Message to process.
1061         :type msg: Message
1062         :returns: Nothing.
1063         """
1064
1065         if self._msg_type:
1066             self.parse_msg[self._msg_type](msg)
1067
1068     def end_message(self, msg):
1069         """Called when message ends. Default implementation does nothing.
1070
1071         :param msg: Message to process.
1072         :type msg: Message
1073         :returns: Nothing.
1074         """
1075         pass
1076
1077
1078 class InputData(object):
1079     """Input data
1080
1081     The data is extracted from output.xml files generated by Jenkins jobs and
1082     stored in pandas' DataFrames.
1083
1084     The data structure:
1085     - job name
1086       - build number
1087         - metadata
1088           (as described in ExecutionChecker documentation)
1089         - suites
1090           (as described in ExecutionChecker documentation)
1091         - tests
1092           (as described in ExecutionChecker documentation)
1093     """
1094
1095     def __init__(self, spec):
1096         """Initialization.
1097
1098         :param spec: Specification.
1099         :type spec: Specification
1100         """
1101
1102         # Specification:
1103         self._cfg = spec
1104
1105         # Data store:
1106         self._input_data = pd.Series()
1107
1108     @property
1109     def data(self):
1110         """Getter - Input data.
1111
1112         :returns: Input data
1113         :rtype: pandas.Series
1114         """
1115         return self._input_data
1116
1117     def metadata(self, job, build):
1118         """Getter - metadata
1119
1120         :param job: Job which metadata we want.
1121         :param build: Build which metadata we want.
1122         :type job: str
1123         :type build: str
1124         :returns: Metadata
1125         :rtype: pandas.Series
1126         """
1127
1128         return self.data[job][build]["metadata"]
1129
1130     def suites(self, job, build):
1131         """Getter - suites
1132
1133         :param job: Job which suites we want.
1134         :param build: Build which suites we want.
1135         :type job: str
1136         :type build: str
1137         :returns: Suites.
1138         :rtype: pandas.Series
1139         """
1140
1141         return self.data[job][str(build)]["suites"]
1142
1143     def tests(self, job, build):
1144         """Getter - tests
1145
1146         :param job: Job which tests we want.
1147         :param build: Build which tests we want.
1148         :type job: str
1149         :type build: str
1150         :returns: Tests.
1151         :rtype: pandas.Series
1152         """
1153
1154         return self.data[job][build]["tests"]
1155
1156     def _parse_tests(self, job, build, log):
1157         """Process data from robot output.xml file and return JSON structured
1158         data.
1159
1160         :param job: The name of job which build output data will be processed.
1161         :param build: The build which output data will be processed.
1162         :param log: List of log messages.
1163         :type job: str
1164         :type build: dict
1165         :type log: list of tuples (severity, msg)
1166         :returns: JSON data structure.
1167         :rtype: dict
1168         """
1169
1170         metadata = {
1171             "job": job,
1172             "build": build
1173         }
1174
1175         with open(build["file-name"], 'r') as data_file:
1176             try:
1177                 result = ExecutionResult(data_file)
1178             except errors.DataError as err:
1179                 log.append(("ERROR", "Error occurred while parsing output.xml: "
1180                                      "{0}".format(err)))
1181                 return None
1182         checker = ExecutionChecker(metadata, self._cfg.mapping,
1183                                    self._cfg.ignore)
1184         result.visit(checker)
1185
1186         return checker.data
1187
1188     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1189         """Download and parse the input data file.
1190
1191         :param pid: PID of the process executing this method.
1192         :param job: Name of the Jenkins job which generated the processed input
1193             file.
1194         :param build: Information about the Jenkins build which generated the
1195             processed input file.
1196         :param repeat: Repeat the download specified number of times if not
1197             successful.
1198         :type pid: int
1199         :type job: str
1200         :type build: dict
1201         :type repeat: int
1202         """
1203
1204         logs = list()
1205
1206         logs.append(("INFO", "  Processing the job/build: {0}: {1}".
1207                      format(job, build["build"])))
1208
1209         state = "failed"
1210         success = False
1211         data = None
1212         do_repeat = repeat
1213         while do_repeat:
1214             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1215                                                    logs)
1216             if success:
1217                 break
1218             do_repeat -= 1
1219         if not success:
1220             logs.append(("ERROR", "It is not possible to download the input "
1221                                   "data file from the job '{job}', build "
1222                                   "'{build}', or it is damaged. Skipped.".
1223                          format(job=job, build=build["build"])))
1224         if success:
1225             logs.append(("INFO", "    Processing data from the build '{0}' ...".
1226                          format(build["build"])))
1227             data = self._parse_tests(job, build, logs)
1228             if data is None:
1229                 logs.append(("ERROR", "Input data file from the job '{job}', "
1230                                       "build '{build}' is damaged. Skipped.".
1231                              format(job=job, build=build["build"])))
1232             else:
1233                 state = "processed"
1234
1235             try:
1236                 remove(build["file-name"])
1237             except OSError as err:
1238                 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
1239                              format(build["file-name"], repr(err))))
1240
1241         # If the time-period is defined in the specification file, remove all
1242         # files which are outside the time period.
1243         timeperiod = self._cfg.input.get("time-period", None)
1244         if timeperiod and data:
1245             now = dt.utcnow()
1246             timeperiod = timedelta(int(timeperiod))
1247             metadata = data.get("metadata", None)
1248             if metadata:
1249                 generated = metadata.get("generated", None)
1250                 if generated:
1251                     generated = dt.strptime(generated, "%Y%m%d %H:%M")
1252                     if (now - generated) > timeperiod:
1253                         # Remove the data and the file:
1254                         state = "removed"
1255                         data = None
1256                         logs.append(
1257                             ("INFO",
1258                              "    The build {job}/{build} is outdated, will be "
1259                              "removed".format(job=job, build=build["build"])))
1260                         file_name = self._cfg.input["file-name"]
1261                         full_name = join(
1262                             self._cfg.environment["paths"]["DIR[WORKING,DATA]"],
1263                             "{job}{sep}{build}{sep}{name}".format(
1264                                 job=job,
1265                                 sep=SEPARATOR,
1266                                 build=build["build"],
1267                                 name=file_name))
1268                         try:
1269                             remove(full_name)
1270                             logs.append(("INFO",
1271                                          "    The file {name} has been removed".
1272                                          format(name=full_name)))
1273                         except OSError as err:
1274                             logs.append(("ERROR",
1275                                          "Cannot remove the file '{0}': {1}".
1276                                          format(full_name, repr(err))))
1277         logs.append(("INFO", "  Done."))
1278
1279         for level, line in logs:
1280             if level == "INFO":
1281                 logging.info(line)
1282             elif level == "ERROR":
1283                 logging.error(line)
1284             elif level == "DEBUG":
1285                 logging.debug(line)
1286             elif level == "CRITICAL":
1287                 logging.critical(line)
1288             elif level == "WARNING":
1289                 logging.warning(line)
1290
1291         return {"data": data, "state": state, "job": job, "build": build}
1292
1293     def download_and_parse_data(self, repeat=1):
1294         """Download the input data files, parse input data from input files and
1295         store in pandas' Series.
1296
1297         :param repeat: Repeat the download specified number of times if not
1298             successful.
1299         :type repeat: int
1300         """
1301
1302         logging.info("Downloading and parsing input files ...")
1303
1304         for job, builds in self._cfg.builds.items():
1305             for build in builds:
1306
1307                 result = self._download_and_parse_build(job, build, repeat)
1308                 build_nr = result["build"]["build"]
1309
1310                 if result["data"]:
1311                     data = result["data"]
1312                     build_data = pd.Series({
1313                         "metadata": pd.Series(
1314                             data["metadata"].values(),
1315                             index=data["metadata"].keys()),
1316                         "suites": pd.Series(data["suites"].values(),
1317                                             index=data["suites"].keys()),
1318                         "tests": pd.Series(data["tests"].values(),
1319                                            index=data["tests"].keys())})
1320
1321                     if self._input_data.get(job, None) is None:
1322                         self._input_data[job] = pd.Series()
1323                     self._input_data[job][str(build_nr)] = build_data
1324
1325                     self._cfg.set_input_file_name(
1326                         job, build_nr, result["build"]["file-name"])
1327
1328                 self._cfg.set_input_state(job, build_nr, result["state"])
1329
1330                 logging.info("Memory allocation: {0:,d}MB".format(
1331                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000))
1332
1333         logging.info("Done.")
1334
1335     @staticmethod
1336     def _end_of_tag(tag_filter, start=0, closer="'"):
1337         """Return the index of character in the string which is the end of tag.
1338
1339         :param tag_filter: The string where the end of tag is being searched.
1340         :param start: The index where the searching is stated.
1341         :param closer: The character which is the tag closer.
1342         :type tag_filter: str
1343         :type start: int
1344         :type closer: str
1345         :returns: The index of the tag closer.
1346         :rtype: int
1347         """
1348
1349         try:
1350             idx_opener = tag_filter.index(closer, start)
1351             return tag_filter.index(closer, idx_opener + 1)
1352         except ValueError:
1353             return None
1354
1355     @staticmethod
1356     def _condition(tag_filter):
1357         """Create a conditional statement from the given tag filter.
1358
1359         :param tag_filter: Filter based on tags from the element specification.
1360         :type tag_filter: str
1361         :returns: Conditional statement which can be evaluated.
1362         :rtype: str
1363         """
1364
1365         index = 0
1366         while True:
1367             index = InputData._end_of_tag(tag_filter, index)
1368             if index is None:
1369                 return tag_filter
1370             index += 1
1371             tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
1372
1373     def filter_data(self, element, params=None, data_set="tests",
1374                     continue_on_error=False):
1375         """Filter required data from the given jobs and builds.
1376
1377         The output data structure is:
1378
1379         - job 1
1380           - build 1
1381             - test (or suite) 1 ID:
1382               - param 1
1383               - param 2
1384               ...
1385               - param n
1386             ...
1387             - test (or suite) n ID:
1388             ...
1389           ...
1390           - build n
1391         ...
1392         - job n
1393
1394         :param element: Element which will use the filtered data.
1395         :param params: Parameters which will be included in the output. If None,
1396         all parameters are included.
1397         :param data_set: The set of data to be filtered: tests, suites,
1398         metadata.
1399         :param continue_on_error: Continue if there is error while reading the
1400         data. The Item will be empty then
1401         :type element: pandas.Series
1402         :type params: list
1403         :type data_set: str
1404         :type continue_on_error: bool
1405         :returns: Filtered data.
1406         :rtype pandas.Series
1407         """
1408
1409         try:
1410             if element["filter"] in ("all", "template"):
1411                 cond = "True"
1412             else:
1413                 cond = InputData._condition(element["filter"])
1414             logging.debug("   Filter: {0}".format(cond))
1415         except KeyError:
1416             logging.error("  No filter defined.")
1417             return None
1418
1419         if params is None:
1420             params = element.get("parameters", None)
1421             if params:
1422                 params.append("type")
1423
1424         data = pd.Series()
1425         try:
1426             for job, builds in element["data"].items():
1427                 data[job] = pd.Series()
1428                 for build in builds:
1429                     data[job][str(build)] = pd.Series()
1430                     try:
1431                         data_iter = self.data[job][str(build)][data_set].\
1432                             iteritems()
1433                     except KeyError:
1434                         if continue_on_error:
1435                             continue
1436                         else:
1437                             return None
1438                     for test_ID, test_data in data_iter:
1439                         if eval(cond, {"tags": test_data.get("tags", "")}):
1440                             data[job][str(build)][test_ID] = pd.Series()
1441                             if params is None:
1442                                 for param, val in test_data.items():
1443                                     data[job][str(build)][test_ID][param] = val
1444                             else:
1445                                 for param in params:
1446                                     try:
1447                                         data[job][str(build)][test_ID][param] =\
1448                                             test_data[param]
1449                                     except KeyError:
1450                                         data[job][str(build)][test_ID][param] =\
1451                                             "No Data"
1452             return data
1453
1454         except (KeyError, IndexError, ValueError) as err:
1455             logging.error("   Missing mandatory parameter in the element "
1456                           "specification: {0}".format(err))
1457             return None
1458         except AttributeError:
1459             return None
1460         except SyntaxError:
1461             logging.error("   The filter '{0}' is not correct. Check if all "
1462                           "tags are enclosed by apostrophes.".format(cond))
1463             return None
1464
1465     @staticmethod
1466     def merge_data(data):
1467         """Merge data from more jobs and builds to a simple data structure.
1468
1469         The output data structure is:
1470
1471         - test (suite) 1 ID:
1472           - param 1
1473           - param 2
1474           ...
1475           - param n
1476         ...
1477         - test (suite) n ID:
1478         ...
1479
1480         :param data: Data to merge.
1481         :type data: pandas.Series
1482         :returns: Merged data.
1483         :rtype: pandas.Series
1484         """
1485
1486         logging.info("    Merging data ...")
1487
1488         merged_data = pd.Series()
1489         for _, builds in data.iteritems():
1490             for _, item in builds.iteritems():
1491                 for ID, item_data in item.iteritems():
1492                     merged_data[ID] = item_data
1493
1494         return merged_data