resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import multiprocessing
  23 import os
  24 import re
  25 import pandas as pd
  26 import logging
  27
  28 from robot.api import ExecutionResult, ResultVisitor
  29 from robot import errors
  30 from collections import OrderedDict
  31 from string import replace
  32 from os import remove
  33 from os.path import join
  34 from datetime import datetime as dt
  35 from datetime import timedelta
  36 from json import loads
  37 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
  38
  39 from input_data_files import download_and_unzip_data_file
  40 from utils import Worker
  41
  42
  43 # Separator used in file names
  44 SEPARATOR = "__"
  45
  46
  47 class ExecutionChecker(ResultVisitor):
  48     """Class to traverse through the test suite structure.
  49
  50     The functionality implemented in this class generates a json structure:
  51
  52     Performance tests:
  53
  54     {
  55         "metadata": {
  56             "generated": "Timestamp",
  57             "version": "SUT version",
  58             "job": "Jenkins job name",
  59             "build": "Information about the build"
  60         },
  61         "suites": {
  62             "Suite long name 1": {
  63                 "name": Suite name,
  64                 "doc": "Suite 1 documentation",
  65                 "parent": "Suite 1 parent",
  66                 "level": "Level of the suite in the suite hierarchy"
  67             }
  68             "Suite long name N": {
  69                 "name": Suite name,
  70                 "doc": "Suite N documentation",
  71                 "parent": "Suite 2 parent",
  72                 "level": "Level of the suite in the suite hierarchy"
  73             }
  74         }
  75         "tests": {
  76             # NDRPDR tests:
  77             "ID": {
  78                 "name": "Test name",
  79                 "parent": "Name of the parent of the test",
  80                 "doc": "Test documentation",
  81                 "msg": "Test message",
  82                 "vat-history": "DUT1 and DUT2 VAT History",
  83                 "show-run": "Show Run",
  84                 "tags": ["tag 1", "tag 2", "tag n"],
  85                 "type": "NDRPDR",
  86                 "status": "PASS" | "FAIL",
  87                 "throughput": {
  88                     "NDR": {
  89                         "LOWER": float,
  90                         "UPPER": float
  91                     },
  92                     "PDR": {
  93                         "LOWER": float,
  94                         "UPPER": float
  95                     }
  96                 },
  97                 "latency": {
  98                     "NDR": {
  99                         "direction1": {
 100                             "min": float,
 101                             "avg": float,
 102                             "max": float
 103                         },
 104                         "direction2": {
 105                             "min": float,
 106                             "avg": float,
 107                             "max": float
 108                         }
 109                     },
 110                     "PDR": {
 111                         "direction1": {
 112                             "min": float,
 113                             "avg": float,
 114                             "max": float
 115                         },
 116                         "direction2": {
 117                             "min": float,
 118                             "avg": float,
 119                             "max": float
 120                         }
 121                     }
 122                 }
 123             }
 124
 125             # TCP tests:
 126             "ID": {
 127                 "name": "Test name",
 128                 "parent": "Name of the parent of the test",
 129                 "doc": "Test documentation",
 130                 "msg": "Test message",
 131                 "tags": ["tag 1", "tag 2", "tag n"],
 132                 "type": "TCP",
 133                 "status": "PASS" | "FAIL",
 134                 "result": int
 135             }
 136
 137             # MRR, BMRR tests:
 138             "ID": {
 139                 "name": "Test name",
 140                 "parent": "Name of the parent of the test",
 141                 "doc": "Test documentation",
 142                 "msg": "Test message",
 143                 "tags": ["tag 1", "tag 2", "tag n"],
 144                 "type": "MRR" | "BMRR",
 145                 "status": "PASS" | "FAIL",
 146                 "result": {
 147                     "receive-rate": AvgStdevMetadata,
 148                 }
 149             }
 150
 151             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 152             # NDRPDRDISC tests:
 153             "ID": {
 154                 "name": "Test name",
 155                 "parent": "Name of the parent of the test",
 156                 "doc": "Test documentation",
 157                 "msg": "Test message",
 158                 "tags": ["tag 1", "tag 2", "tag n"],
 159                 "type": "PDR" | "NDR",
 160                 "status": "PASS" | "FAIL",
 161                 "throughput": {  # Only type: "PDR" | "NDR"
 162                     "value": int,
 163                     "unit": "pps" | "bps" | "percentage"
 164                 },
 165                 "latency": {  # Only type: "PDR" | "NDR"
 166                     "direction1": {
 167                         "100": {
 168                             "min": int,
 169                             "avg": int,
 170                             "max": int
 171                         },
 172                         "50": {  # Only for NDR
 173                             "min": int,
 174                             "avg": int,
 175                             "max": int
 176                         },
 177                         "10": {  # Only for NDR
 178                             "min": int,
 179                             "avg": int,
 180                             "max": int
 181                         }
 182                     },
 183                     "direction2": {
 184                         "100": {
 185                             "min": int,
 186                             "avg": int,
 187                             "max": int
 188                         },
 189                         "50": {  # Only for NDR
 190                             "min": int,
 191                             "avg": int,
 192                             "max": int
 193                         },
 194                         "10": {  # Only for NDR
 195                             "min": int,
 196                             "avg": int,
 197                             "max": int
 198                         }
 199                     }
 200                 },
 201                 "lossTolerance": "lossTolerance",  # Only type: "PDR"
 202                 "vat-history": "DUT1 and DUT2 VAT History"
 203                 "show-run": "Show Run"
 204             },
 205             "ID" {
 206                 # next test
 207             }
 208         }
 209     }
 210
 211
 212     Functional tests:
 213
 214     {
 215         "metadata": {  # Optional
 216             "version": "VPP version",
 217             "job": "Jenkins job name",
 218             "build": "Information about the build"
 219         },
 220         "suites": {
 221             "Suite name 1": {
 222                 "doc": "Suite 1 documentation",
 223                 "parent": "Suite 1 parent",
 224                 "level": "Level of the suite in the suite hierarchy"
 225             }
 226             "Suite name N": {
 227                 "doc": "Suite N documentation",
 228                 "parent": "Suite 2 parent",
 229                 "level": "Level of the suite in the suite hierarchy"
 230             }
 231         }
 232         "tests": {
 233             "ID": {
 234                 "name": "Test name",
 235                 "parent": "Name of the parent of the test",
 236                 "doc": "Test documentation"
 237                 "msg": "Test message"
 238                 "tags": ["tag 1", "tag 2", "tag n"],
 239                 "vat-history": "DUT1 and DUT2 VAT History"
 240                 "show-run": "Show Run"
 241                 "status": "PASS" | "FAIL"
 242             },
 243             "ID" {
 244                 # next test
 245             }
 246         }
 247     }
 248
 249     .. note:: ID is the lowercase full path to the test.
 250     """
 251
 252     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 253     REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
 254
 255     REGEX_PLR_RATE = re.compile(r'PLRsearch lower bound::\s(\d+.\d+).*\n'
 256                                 r'PLRsearch upper bound::\s(\d+.\d+)')
 257
 258     REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 259                                    r'NDR_UPPER:\s(\d+.\d+).*\n'
 260                                    r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 261                                    r'PDR_UPPER:\s(\d+.\d+)')
 262
 263     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 264     REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
 265                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 266                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
 267                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 268                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
 269                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 270                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
 271
 272     REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
 273                                r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 274                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
 275
 276     REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 277                                   r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
 278
 279     REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
 280                                  r'[\D\d]*')
 281
 282     REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|"
 283                                    r"VPP Version:\s*)(.*)")
 284
 285     REGEX_VERSION_DPDK = re.compile(r"(return STDOUT testpmd)([\d\D\n]*)"
 286                                     r"(RTE Version: 'DPDK )(.*)(')")
 287
 288     REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
 289
 290     REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 291                            r'tx\s(\d*),\srx\s(\d*)')
 292
 293     REGEX_BMRR = re.compile(r'Maximum Receive Rate trial results'
 294                             r' in packets per second: \[(.*)\]')
 295
 296     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 297
 298     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 299
 300     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 301
 302     REGEX_TC_NUMBER = re.compile(r'tc[0-9]{2}-')
 303
 304     def __init__(self, metadata, mapping, ignore):
 305         """Initialisation.
 306
 307         :param metadata: Key-value pairs to be included in "metadata" part of
 308             JSON structure.
 309         :param mapping: Mapping of the old names of test cases to the new
 310             (actual) one.
 311         :param ignore: List of TCs to be ignored.
 312         :type metadata: dict
 313         :type mapping: dict
 314         :type ignore: list
 315         """
 316
 317         # Type of message to parse out from the test messages
 318         self._msg_type = None
 319
 320         # VPP version
 321         self._version = None
 322
 323         # Timestamp
 324         self._timestamp = None
 325
 326         # Testbed. The testbed is identified by TG node IP address.
 327         self._testbed = None
 328
 329         # Mapping of TCs long names
 330         self._mapping = mapping
 331
 332         # Ignore list
 333         self._ignore = ignore
 334
 335         # Number of VAT History messages found:
 336         # 0 - no message
 337         # 1 - VAT History of DUT1
 338         # 2 - VAT History of DUT2
 339         self._lookup_kw_nr = 0
 340         self._vat_history_lookup_nr = 0
 341
 342         # Number of Show Running messages found
 343         # 0 - no message
 344         # 1 - Show run message found
 345         self._show_run_lookup_nr = 0
 346
 347         # Test ID of currently processed test- the lowercase full path to the
 348         # test
 349         self._test_ID = None
 350
 351         # The main data structure
 352         self._data = {
 353             "metadata": OrderedDict(),
 354             "suites": OrderedDict(),
 355             "tests": OrderedDict()
 356         }
 357
 358         # Save the provided metadata
 359         for key, val in metadata.items():
 360             self._data["metadata"][key] = val
 361
 362         # Dictionary defining the methods used to parse different types of
 363         # messages
 364         self.parse_msg = {
 365             "timestamp": self._get_timestamp,
 366             "vpp-version": self._get_vpp_version,
 367             "dpdk-version": self._get_dpdk_version,
 368             "teardown-vat-history": self._get_vat_history,
 369             "test-show-runtime": self._get_show_run,
 370             "testbed": self._get_testbed
 371         }
 372
 373     @property
 374     def data(self):
 375         """Getter - Data parsed from the XML file.
 376
 377         :returns: Data parsed from the XML file.
 378         :rtype: dict
 379         """
 380         return self._data
 381
 382     def _get_testbed(self, msg):
 383         """Called when extraction of testbed IP is required.
 384         The testbed is identified by TG node IP address.
 385
 386         :param msg: Message to process.
 387         :type msg: Message
 388         :returns: Nothing.
 389         """
 390
 391         if msg.message.count("Arguments:"):
 392             message = str(msg.message).replace(' ', '').replace('\n', '').\
 393                 replace("'", '"').replace('b"', '"').\
 394                 replace("honeycom", "honeycomb")
 395             message = loads(message[11:-1])
 396             try:
 397                 self._testbed = message["TG"]["host"]
 398             except (KeyError, ValueError):
 399                 pass
 400             finally:
 401                 self._data["metadata"]["testbed"] = self._testbed
 402                 self._msg_type = None
 403
 404     def _get_vpp_version(self, msg):
 405         """Called when extraction of VPP version is required.
 406
 407         :param msg: Message to process.
 408         :type msg: Message
 409         :returns: Nothing.
 410         """
 411
 412         if msg.message.count("return STDOUT Version:") or \
 413             msg.message.count("VPP Version:"):
 414             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 415                                 group(2))
 416             self._data["metadata"]["version"] = self._version
 417             self._msg_type = None
 418
 419     def _get_dpdk_version(self, msg):
 420         """Called when extraction of DPDK version is required.
 421
 422         :param msg: Message to process.
 423         :type msg: Message
 424         :returns: Nothing.
 425         """
 426
 427         if msg.message.count("return STDOUT testpmd"):
 428             try:
 429                 self._version = str(re.search(
 430                     self.REGEX_VERSION_DPDK, msg.message). group(4))
 431                 self._data["metadata"]["version"] = self._version
 432             except IndexError:
 433                 pass
 434             finally:
 435                 self._msg_type = None
 436
 437     def _get_timestamp(self, msg):
 438         """Called when extraction of timestamp is required.
 439
 440         :param msg: Message to process.
 441         :type msg: Message
 442         :returns: Nothing.
 443         """
 444
 445         self._timestamp = msg.timestamp[:14]
 446         self._data["metadata"]["generated"] = self._timestamp
 447         self._msg_type = None
 448
 449     def _get_vat_history(self, msg):
 450         """Called when extraction of VAT command history is required.
 451
 452         :param msg: Message to process.
 453         :type msg: Message
 454         :returns: Nothing.
 455         """
 456         if msg.message.count("VAT command history:"):
 457             self._vat_history_lookup_nr += 1
 458             if self._vat_history_lookup_nr == 1:
 459                 self._data["tests"][self._test_ID]["vat-history"] = str()
 460             else:
 461                 self._msg_type = None
 462             text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
 463                           "VAT command history:", "", msg.message, count=1). \
 464                 replace("\n\n", "\n").replace('\n', ' |br| ').\
 465                 replace('\r', '').replace('"', "'")
 466
 467             self._data["tests"][self._test_ID]["vat-history"] += " |br| "
 468             self._data["tests"][self._test_ID]["vat-history"] += \
 469                 "**DUT" + str(self._vat_history_lookup_nr) + ":** " + text
 470
 471     def _get_show_run(self, msg):
 472         """Called when extraction of VPP operational data (output of CLI command
 473         Show Runtime) is required.
 474
 475         :param msg: Message to process.
 476         :type msg: Message
 477         :returns: Nothing.
 478         """
 479         if msg.message.count("return STDOUT Thread "):
 480             self._show_run_lookup_nr += 1
 481             if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
 482                 self._data["tests"][self._test_ID]["show-run"] = str()
 483             if self._lookup_kw_nr > 1:
 484                 self._msg_type = None
 485             if self._show_run_lookup_nr == 1:
 486                 text = msg.message.replace("vat# ", "").\
 487                     replace("return STDOUT ", "").replace("\n\n", "\n").\
 488                     replace('\n', ' |br| ').\
 489                     replace('\r', '').replace('"', "'")
 490                 try:
 491                     self._data["tests"][self._test_ID]["show-run"] += " |br| "
 492                     self._data["tests"][self._test_ID]["show-run"] += \
 493                         "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
 494                 except KeyError:
 495                     pass
 496
 497     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 498     def _get_latency(self, msg, test_type):
 499         """Get the latency data from the test message.
 500
 501         :param msg: Message to be parsed.
 502         :param test_type: Type of the test - NDR or PDR.
 503         :type msg: str
 504         :type test_type: str
 505         :returns: Latencies parsed from the message.
 506         :rtype: dict
 507         """
 508
 509         if test_type == "NDR":
 510             groups = re.search(self.REGEX_LAT_NDR, msg)
 511             groups_range = range(1, 7)
 512         elif test_type == "PDR":
 513             groups = re.search(self.REGEX_LAT_PDR, msg)
 514             groups_range = range(1, 3)
 515         else:
 516             return {}
 517
 518         latencies = list()
 519         for idx in groups_range:
 520             try:
 521                 lat = [int(item) for item in str(groups.group(idx)).split('/')]
 522             except (AttributeError, ValueError):
 523                 lat = [-1, -1, -1]
 524             latencies.append(lat)
 525
 526         keys = ("min", "avg", "max")
 527         latency = {
 528             "direction1": {
 529             },
 530             "direction2": {
 531             }
 532         }
 533
 534         latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
 535         latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
 536         if test_type == "NDR":
 537             latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
 538             latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
 539             latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
 540             latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
 541
 542         return latency
 543
 544     def _get_ndrpdr_throughput(self, msg):
 545         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 546         message.
 547
 548         :param msg: The test message to be parsed.
 549         :type msg: str
 550         :returns: Parsed data as a dict and the status (PASS/FAIL).
 551         :rtype: tuple(dict, str)
 552         """
 553
 554         throughput = {
 555             "NDR": {"LOWER": -1.0, "UPPER": -1.0},
 556             "PDR": {"LOWER": -1.0, "UPPER": -1.0}
 557         }
 558         status = "FAIL"
 559         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 560
 561         if groups is not None:
 562             try:
 563                 throughput["NDR"]["LOWER"] = float(groups.group(1))
 564                 throughput["NDR"]["UPPER"] = float(groups.group(2))
 565                 throughput["PDR"]["LOWER"] = float(groups.group(3))
 566                 throughput["PDR"]["UPPER"] = float(groups.group(4))
 567                 status = "PASS"
 568             except (IndexError, ValueError):
 569                 pass
 570
 571         return throughput, status
 572
 573     def _get_plr_throughput(self, msg):
 574         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 575         message.
 576
 577         :param msg: The test message to be parsed.
 578         :type msg: str
 579         :returns: Parsed data as a dict and the status (PASS/FAIL).
 580         :rtype: tuple(dict, str)
 581         """
 582
 583         throughput = {
 584             "LOWER": -1.0,
 585             "UPPER": -1.0
 586         }
 587         status = "FAIL"
 588         groups = re.search(self.REGEX_PLR_RATE, msg)
 589
 590         if groups is not None:
 591             try:
 592                 throughput["LOWER"] = float(groups.group(1))
 593                 throughput["UPPER"] = float(groups.group(2))
 594                 status = "PASS"
 595             except (IndexError, ValueError):
 596                 pass
 597
 598         return throughput, status
 599
 600     def _get_ndrpdr_latency(self, msg):
 601         """Get LATENCY from the test message.
 602
 603         :param msg: The test message to be parsed.
 604         :type msg: str
 605         :returns: Parsed data as a dict and the status (PASS/FAIL).
 606         :rtype: tuple(dict, str)
 607         """
 608
 609         latency = {
 610             "NDR": {
 611                 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
 612                 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
 613             },
 614             "PDR": {
 615                 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
 616                 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
 617             }
 618         }
 619         status = "FAIL"
 620         groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 621
 622         if groups is not None:
 623             keys = ("min", "avg", "max")
 624             try:
 625                 latency["NDR"]["direction1"] = dict(
 626                     zip(keys, [float(l) for l in groups.group(1).split('/')]))
 627                 latency["NDR"]["direction2"] = dict(
 628                     zip(keys, [float(l) for l in groups.group(2).split('/')]))
 629                 latency["PDR"]["direction1"] = dict(
 630                     zip(keys, [float(l) for l in groups.group(3).split('/')]))
 631                 latency["PDR"]["direction2"] = dict(
 632                     zip(keys, [float(l) for l in groups.group(4).split('/')]))
 633                 status = "PASS"
 634             except (IndexError, ValueError):
 635                 pass
 636
 637         return latency, status
 638
 639     def visit_suite(self, suite):
 640         """Implements traversing through the suite and its direct children.
 641
 642         :param suite: Suite to process.
 643         :type suite: Suite
 644         :returns: Nothing.
 645         """
 646         if self.start_suite(suite) is not False:
 647             suite.suites.visit(self)
 648             suite.tests.visit(self)
 649             self.end_suite(suite)
 650
 651     def start_suite(self, suite):
 652         """Called when suite starts.
 653
 654         :param suite: Suite to process.
 655         :type suite: Suite
 656         :returns: Nothing.
 657         """
 658
 659         try:
 660             parent_name = suite.parent.name
 661         except AttributeError:
 662             return
 663
 664         doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
 665             replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
 666         doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
 667
 668         self._data["suites"][suite.longname.lower().replace('"', "'").
 669             replace(" ", "_")] = {
 670                 "name": suite.name.lower(),
 671                 "doc": doc_str,
 672                 "parent": parent_name,
 673                 "level": len(suite.longname.split("."))
 674             }
 675
 676         suite.keywords.visit(self)
 677
 678     def end_suite(self, suite):
 679         """Called when suite ends.
 680
 681         :param suite: Suite to process.
 682         :type suite: Suite
 683         :returns: Nothing.
 684         """
 685         pass
 686
 687     def visit_test(self, test):
 688         """Implements traversing through the test.
 689
 690         :param test: Test to process.
 691         :type test: Test
 692         :returns: Nothing.
 693         """
 694         if self.start_test(test) is not False:
 695             test.keywords.visit(self)
 696             self.end_test(test)
 697
 698     def start_test(self, test):
 699         """Called when test starts.
 700
 701         :param test: Test to process.
 702         :type test: Test
 703         :returns: Nothing.
 704         """
 705
 706         longname_orig = test.longname.lower()
 707
 708         # Check the ignore list
 709         if longname_orig in self._ignore:
 710             return
 711
 712         tags = [str(tag) for tag in test.tags]
 713         test_result = dict()
 714
 715         # Change the TC long name and name if defined in the mapping table
 716         longname = self._mapping.get(longname_orig, None)
 717         if longname is not None:
 718             name = longname.split('.')[-1]
 719             logging.debug("{0}\n{1}\n{2}\n{3}".format(
 720                 self._data["metadata"], longname_orig, longname, name))
 721         else:
 722             longname = longname_orig
 723             name = test.name.lower()
 724
 725         # Remove TC number from the TC long name (backward compatibility):
 726         self._test_ID = re.sub(self.REGEX_TC_NUMBER, "", longname)
 727         # Remove TC number from the TC name (not needed):
 728         test_result["name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
 729
 730         test_result["parent"] = test.parent.name.lower()
 731         test_result["tags"] = tags
 732         doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
 733             replace('\r', '').replace('[', ' |br| [')
 734         test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
 735         test_result["msg"] = test.message.replace('\n', ' |br| '). \
 736             replace('\r', '').replace('"', "'")
 737         test_result["type"] = "FUNC"
 738         test_result["status"] = test.status
 739
 740         if "PERFTEST" in tags:
 741             # Replace info about cores (e.g. -1c-) with the info about threads
 742             # and cores (e.g. -1t1c-) in the long test case names and in the
 743             # test case names if necessary.
 744             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_ID)
 745             if not groups:
 746                 tag_count = 0
 747                 for tag in test_result["tags"]:
 748                     groups = re.search(self.REGEX_TC_TAG, tag)
 749                     if groups:
 750                         tag_count += 1
 751                         tag_tc = tag
 752
 753                 if tag_count == 1:
 754                     self._test_ID = re.sub(self.REGEX_TC_NAME_NEW,
 755                                            "-{0}-".format(tag_tc.lower()),
 756                                            self._test_ID,
 757                                            count=1)
 758                     test_result["name"] = re.sub(self.REGEX_TC_NAME_NEW,
 759                                                  "-{0}-".format(tag_tc.lower()),
 760                                                  test_result["name"],
 761                                                  count=1)
 762                 else:
 763                     test_result["status"] = "FAIL"
 764                     self._data["tests"][self._test_ID] = test_result
 765                     logging.debug("The test '{0}' has no or more than one "
 766                                   "multi-threading tags.".format(self._test_ID))
 767                     logging.debug("Tags: {0}".format(test_result["tags"]))
 768                     return
 769
 770         if test.status == "PASS" and ("NDRPDRDISC" in tags or
 771                                       "NDRPDR" in tags or
 772                                       "SOAK" in tags or
 773                                       "TCP" in tags or
 774                                       "MRR" in tags or
 775                                       "BMRR" in tags):
 776             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 777             if "NDRDISC" in tags:
 778                 test_result["type"] = "NDR"
 779             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 780             elif "PDRDISC" in tags:
 781                 test_result["type"] = "PDR"
 782             elif "NDRPDR" in tags:
 783                 test_result["type"] = "NDRPDR"
 784             elif "SOAK" in tags:
 785                 test_result["type"] = "SOAK"
 786             elif "TCP" in tags:
 787                 test_result["type"] = "TCP"
 788             elif "MRR" in tags:
 789                 test_result["type"] = "MRR"
 790             elif "FRMOBL" in tags or "BMRR" in tags:
 791                 test_result["type"] = "BMRR"
 792             else:
 793                 test_result["status"] = "FAIL"
 794                 self._data["tests"][self._test_ID] = test_result
 795                 return
 796
 797             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 798             if test_result["type"] in ("NDR", "PDR"):
 799                 try:
 800                     rate_value = str(re.search(
 801                         self.REGEX_RATE, test.message).group(1))
 802                 except AttributeError:
 803                     rate_value = "-1"
 804                 try:
 805                     rate_unit = str(re.search(
 806                         self.REGEX_RATE, test.message).group(2))
 807                 except AttributeError:
 808                     rate_unit = "-1"
 809
 810                 test_result["throughput"] = dict()
 811                 test_result["throughput"]["value"] = \
 812                     int(rate_value.split('.')[0])
 813                 test_result["throughput"]["unit"] = rate_unit
 814                 test_result["latency"] = \
 815                     self._get_latency(test.message, test_result["type"])
 816                 if test_result["type"] == "PDR":
 817                     test_result["lossTolerance"] = str(re.search(
 818                         self.REGEX_TOLERANCE, test.message).group(1))
 819
 820             elif test_result["type"] in ("NDRPDR", ):
 821                 test_result["throughput"], test_result["status"] = \
 822                     self._get_ndrpdr_throughput(test.message)
 823                 test_result["latency"], test_result["status"] = \
 824                     self._get_ndrpdr_latency(test.message)
 825
 826             elif test_result["type"] in ("SOAK", ):
 827                 test_result["throughput"], test_result["status"] = \
 828                     self._get_plr_throughput(test.message)
 829
 830             elif test_result["type"] in ("TCP", ):
 831                 groups = re.search(self.REGEX_TCP, test.message)
 832                 test_result["result"] = int(groups.group(2))
 833
 834             elif test_result["type"] in ("MRR", "BMRR"):
 835                 test_result["result"] = dict()
 836                 groups = re.search(self.REGEX_BMRR, test.message)
 837                 if groups is not None:
 838                     items_str = groups.group(1)
 839                     items_float = [float(item.strip()) for item
 840                                    in items_str.split(",")]
 841                     metadata = AvgStdevMetadataFactory.from_data(items_float)
 842                     # Next two lines have been introduced in CSIT-1179,
 843                     # to be removed in CSIT-1180.
 844                     metadata.size = 1
 845                     metadata.stdev = 0.0
 846                     test_result["result"]["receive-rate"] = metadata
 847                 else:
 848                     groups = re.search(self.REGEX_MRR, test.message)
 849                     test_result["result"]["receive-rate"] = \
 850                         AvgStdevMetadataFactory.from_data([
 851                             float(groups.group(3)) / float(groups.group(1)), ])
 852
 853         self._data["tests"][self._test_ID] = test_result
 854
 855     def end_test(self, test):
 856         """Called when test ends.
 857
 858         :param test: Test to process.
 859         :type test: Test
 860         :returns: Nothing.
 861         """
 862         pass
 863
 864     def visit_keyword(self, keyword):
 865         """Implements traversing through the keyword and its child keywords.
 866
 867         :param keyword: Keyword to process.
 868         :type keyword: Keyword
 869         :returns: Nothing.
 870         """
 871         if self.start_keyword(keyword) is not False:
 872             self.end_keyword(keyword)
 873
 874     def start_keyword(self, keyword):
 875         """Called when keyword starts. Default implementation does nothing.
 876
 877         :param keyword: Keyword to process.
 878         :type keyword: Keyword
 879         :returns: Nothing.
 880         """
 881         try:
 882             if keyword.type == "setup":
 883                 self.visit_setup_kw(keyword)
 884             elif keyword.type == "teardown":
 885                 self._lookup_kw_nr = 0
 886                 self.visit_teardown_kw(keyword)
 887             else:
 888                 self._lookup_kw_nr = 0
 889                 self.visit_test_kw(keyword)
 890         except AttributeError:
 891             pass
 892
 893     def end_keyword(self, keyword):
 894         """Called when keyword ends. Default implementation does nothing.
 895
 896         :param keyword: Keyword to process.
 897         :type keyword: Keyword
 898         :returns: Nothing.
 899         """
 900         pass
 901
 902     def visit_test_kw(self, test_kw):
 903         """Implements traversing through the test keyword and its child
 904         keywords.
 905
 906         :param test_kw: Keyword to process.
 907         :type test_kw: Keyword
 908         :returns: Nothing.
 909         """
 910         for keyword in test_kw.keywords:
 911             if self.start_test_kw(keyword) is not False:
 912                 self.visit_test_kw(keyword)
 913                 self.end_test_kw(keyword)
 914
 915     def start_test_kw(self, test_kw):
 916         """Called when test keyword starts. Default implementation does
 917         nothing.
 918
 919         :param test_kw: Keyword to process.
 920         :type test_kw: Keyword
 921         :returns: Nothing.
 922         """
 923         if test_kw.name.count("Show Runtime Counters On All Duts"):
 924             self._lookup_kw_nr += 1
 925             self._show_run_lookup_nr = 0
 926             self._msg_type = "test-show-runtime"
 927         elif test_kw.name.count("Start The L2fwd Test") and not self._version:
 928             self._msg_type = "dpdk-version"
 929         else:
 930             return
 931         test_kw.messages.visit(self)
 932
 933     def end_test_kw(self, test_kw):
 934         """Called when keyword ends. Default implementation does nothing.
 935
 936         :param test_kw: Keyword to process.
 937         :type test_kw: Keyword
 938         :returns: Nothing.
 939         """
 940         pass
 941
 942     def visit_setup_kw(self, setup_kw):
 943         """Implements traversing through the teardown keyword and its child
 944         keywords.
 945
 946         :param setup_kw: Keyword to process.
 947         :type setup_kw: Keyword
 948         :returns: Nothing.
 949         """
 950         for keyword in setup_kw.keywords:
 951             if self.start_setup_kw(keyword) is not False:
 952                 self.visit_setup_kw(keyword)
 953                 self.end_setup_kw(keyword)
 954
 955     def start_setup_kw(self, setup_kw):
 956         """Called when teardown keyword starts. Default implementation does
 957         nothing.
 958
 959         :param setup_kw: Keyword to process.
 960         :type setup_kw: Keyword
 961         :returns: Nothing.
 962         """
 963         if setup_kw.name.count("Show Vpp Version On All Duts") \
 964                 and not self._version:
 965             self._msg_type = "vpp-version"
 966
 967         elif setup_kw.name.count("Setup performance global Variables") \
 968                 and not self._timestamp:
 969             self._msg_type = "timestamp"
 970         elif setup_kw.name.count("Setup Framework") and not self._testbed:
 971             self._msg_type = "testbed"
 972         else:
 973             return
 974         setup_kw.messages.visit(self)
 975
 976     def end_setup_kw(self, setup_kw):
 977         """Called when keyword ends. Default implementation does nothing.
 978
 979         :param setup_kw: Keyword to process.
 980         :type setup_kw: Keyword
 981         :returns: Nothing.
 982         """
 983         pass
 984
 985     def visit_teardown_kw(self, teardown_kw):
 986         """Implements traversing through the teardown keyword and its child
 987         keywords.
 988
 989         :param teardown_kw: Keyword to process.
 990         :type teardown_kw: Keyword
 991         :returns: Nothing.
 992         """
 993         for keyword in teardown_kw.keywords:
 994             if self.start_teardown_kw(keyword) is not False:
 995                 self.visit_teardown_kw(keyword)
 996                 self.end_teardown_kw(keyword)
 997
 998     def start_teardown_kw(self, teardown_kw):
 999         """Called when teardown keyword starts. Default implementation does
1000         nothing.
1001
1002         :param teardown_kw: Keyword to process.
1003         :type teardown_kw: Keyword
1004         :returns: Nothing.
1005         """
1006
1007         if teardown_kw.name.count("Show Vat History On All Duts"):
1008             self._vat_history_lookup_nr = 0
1009             self._msg_type = "teardown-vat-history"
1010             teardown_kw.messages.visit(self)
1011
1012     def end_teardown_kw(self, teardown_kw):
1013         """Called when keyword ends. Default implementation does nothing.
1014
1015         :param teardown_kw: Keyword to process.
1016         :type teardown_kw: Keyword
1017         :returns: Nothing.
1018         """
1019         pass
1020
1021     def visit_message(self, msg):
1022         """Implements visiting the message.
1023
1024         :param msg: Message to process.
1025         :type msg: Message
1026         :returns: Nothing.
1027         """
1028         if self.start_message(msg) is not False:
1029             self.end_message(msg)
1030
1031     def start_message(self, msg):
1032         """Called when message starts. Get required information from messages:
1033         - VPP version.
1034
1035         :param msg: Message to process.
1036         :type msg: Message
1037         :returns: Nothing.
1038         """
1039
1040         if self._msg_type:
1041             self.parse_msg[self._msg_type](msg)
1042
1043     def end_message(self, msg):
1044         """Called when message ends. Default implementation does nothing.
1045
1046         :param msg: Message to process.
1047         :type msg: Message
1048         :returns: Nothing.
1049         """
1050         pass
1051
1052
1053 class InputData(object):
1054     """Input data
1055
1056     The data is extracted from output.xml files generated by Jenkins jobs and
1057     stored in pandas' DataFrames.
1058
1059     The data structure:
1060     - job name
1061       - build number
1062         - metadata
1063           (as described in ExecutionChecker documentation)
1064         - suites
1065           (as described in ExecutionChecker documentation)
1066         - tests
1067           (as described in ExecutionChecker documentation)
1068     """
1069
1070     def __init__(self, spec):
1071         """Initialization.
1072
1073         :param spec: Specification.
1074         :type spec: Specification
1075         """
1076
1077         # Specification:
1078         self._cfg = spec
1079
1080         # Data store:
1081         self._input_data = pd.Series()
1082
1083     @property
1084     def data(self):
1085         """Getter - Input data.
1086
1087         :returns: Input data
1088         :rtype: pandas.Series
1089         """
1090         return self._input_data
1091
1092     def metadata(self, job, build):
1093         """Getter - metadata
1094
1095         :param job: Job which metadata we want.
1096         :param build: Build which metadata we want.
1097         :type job: str
1098         :type build: str
1099         :returns: Metadata
1100         :rtype: pandas.Series
1101         """
1102
1103         return self.data[job][build]["metadata"]
1104
1105     def suites(self, job, build):
1106         """Getter - suites
1107
1108         :param job: Job which suites we want.
1109         :param build: Build which suites we want.
1110         :type job: str
1111         :type build: str
1112         :returns: Suites.
1113         :rtype: pandas.Series
1114         """
1115
1116         return self.data[job][str(build)]["suites"]
1117
1118     def tests(self, job, build):
1119         """Getter - tests
1120
1121         :param job: Job which tests we want.
1122         :param build: Build which tests we want.
1123         :type job: str
1124         :type build: str
1125         :returns: Tests.
1126         :rtype: pandas.Series
1127         """
1128
1129         return self.data[job][build]["tests"]
1130
1131     def _parse_tests(self, job, build, log):
1132         """Process data from robot output.xml file and return JSON structured
1133         data.
1134
1135         :param job: The name of job which build output data will be processed.
1136         :param build: The build which output data will be processed.
1137         :param log: List of log messages.
1138         :type job: str
1139         :type build: dict
1140         :type log: list of tuples (severity, msg)
1141         :returns: JSON data structure.
1142         :rtype: dict
1143         """
1144
1145         metadata = {
1146             "job": job,
1147             "build": build
1148         }
1149
1150         with open(build["file-name"], 'r') as data_file:
1151             try:
1152                 result = ExecutionResult(data_file)
1153             except errors.DataError as err:
1154                 log.append(("ERROR", "Error occurred while parsing output.xml: "
1155                                      "{0}".format(err)))
1156                 return None
1157         checker = ExecutionChecker(metadata, self._cfg.mapping,
1158                                    self._cfg.ignore)
1159         result.visit(checker)
1160
1161         return checker.data
1162
1163     def _download_and_parse_build(self, pid, data_queue, job, build, repeat):
1164         """Download and parse the input data file.
1165
1166         :param pid: PID of the process executing this method.
1167         :param data_queue: Shared memory between processes. Queue which keeps
1168             the result data. This data is then read by the main process and used
1169             in further processing.
1170         :param job: Name of the Jenkins job which generated the processed input
1171             file.
1172         :param build: Information about the Jenkins build which generated the
1173             processed input file.
1174         :param repeat: Repeat the download specified number of times if not
1175             successful.
1176         :type pid: int
1177         :type data_queue: multiprocessing.Manager().Queue()
1178         :type job: str
1179         :type build: dict
1180         :type repeat: int
1181         """
1182
1183         logs = list()
1184
1185         logging.info("  Processing the job/build: {0}: {1}".
1186                      format(job, build["build"]))
1187
1188         logs.append(("INFO", "  Processing the job/build: {0}: {1}".
1189                      format(job, build["build"])))
1190
1191         state = "failed"
1192         success = False
1193         data = None
1194         do_repeat = repeat
1195         while do_repeat:
1196             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1197                                                    logs)
1198             if success:
1199                 break
1200             do_repeat -= 1
1201         if not success:
1202             logs.append(("ERROR", "It is not possible to download the input "
1203                                   "data file from the job '{job}', build "
1204                                   "'{build}', or it is damaged. Skipped.".
1205                          format(job=job, build=build["build"])))
1206         if success:
1207             logs.append(("INFO", "  Processing data from the build '{0}' ...".
1208                          format(build["build"])))
1209             data = self._parse_tests(job, build, logs)
1210             if data is None:
1211                 logs.append(("ERROR", "Input data file from the job '{job}', "
1212                                       "build '{build}' is damaged. Skipped.".
1213                              format(job=job, build=build["build"])))
1214             else:
1215                 state = "processed"
1216
1217             try:
1218                 remove(build["file-name"])
1219             except OSError as err:
1220                 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
1221                              format(build["file-name"], repr(err))))
1222
1223         # If the time-period is defined in the specification file, remove all
1224         # files which are outside the time period.
1225         timeperiod = self._cfg.input.get("time-period", None)
1226         if timeperiod and data:
1227             now = dt.utcnow()
1228             timeperiod = timedelta(int(timeperiod))
1229             metadata = data.get("metadata", None)
1230             if metadata:
1231                 generated = metadata.get("generated", None)
1232                 if generated:
1233                     generated = dt.strptime(generated, "%Y%m%d %H:%M")
1234                     if (now - generated) > timeperiod:
1235                         # Remove the data and the file:
1236                         state = "removed"
1237                         data = None
1238                         logs.append(
1239                             ("INFO",
1240                              "    The build {job}/{build} is outdated, will be "
1241                              "removed".format(job=job, build=build["build"])))
1242                         file_name = self._cfg.input["file-name"]
1243                         full_name = join(
1244                             self._cfg.environment["paths"]["DIR[WORKING,DATA]"],
1245                             "{job}{sep}{build}{sep}{name}".
1246                                 format(job=job,
1247                                        sep=SEPARATOR,
1248                                        build=build["build"],
1249                                        name=file_name))
1250                         try:
1251                             remove(full_name)
1252                             logs.append(("INFO",
1253                                          "    The file {name} has been removed".
1254                                          format(name=full_name)))
1255                         except OSError as err:
1256                             logs.append(("ERROR",
1257                                         "Cannot remove the file '{0}': {1}".
1258                                         format(full_name, repr(err))))
1259
1260         logs.append(("INFO", "  Done."))
1261
1262         result = {
1263             "data": data,
1264             "state": state,
1265             "job": job,
1266             "build": build,
1267             "logs": logs
1268         }
1269         data_queue.put(result)
1270
1271     def download_and_parse_data(self, repeat=1):
1272         """Download the input data files, parse input data from input files and
1273         store in pandas' Series.
1274
1275         :param repeat: Repeat the download specified number of times if not
1276             successful.
1277         :type repeat: int
1278         """
1279
1280         logging.info("Downloading and parsing input files ...")
1281
1282         work_queue = multiprocessing.JoinableQueue()
1283         manager = multiprocessing.Manager()
1284         data_queue = manager.Queue()
1285         cpus = multiprocessing.cpu_count()
1286
1287         workers = list()
1288         for cpu in range(cpus):
1289             worker = Worker(work_queue,
1290                             data_queue,
1291                             self._download_and_parse_build)
1292             worker.daemon = True
1293             worker.start()
1294             workers.append(worker)
1295             os.system("taskset -p -c {0} {1} > /dev/null 2>&1".
1296                       format(cpu, worker.pid))
1297
1298         for job, builds in self._cfg.builds.items():
1299             for build in builds:
1300                 work_queue.put((job, build, repeat))
1301
1302         work_queue.join()
1303
1304         logging.info("Done.")
1305
1306         while not data_queue.empty():
1307             result = data_queue.get()
1308
1309             job = result["job"]
1310             build_nr = result["build"]["build"]
1311
1312             if result["data"]:
1313                 data = result["data"]
1314                 build_data = pd.Series({
1315                     "metadata": pd.Series(data["metadata"].values(),
1316                                           index=data["metadata"].keys()),
1317                     "suites": pd.Series(data["suites"].values(),
1318                                         index=data["suites"].keys()),
1319                     "tests": pd.Series(data["tests"].values(),
1320                                        index=data["tests"].keys())})
1321
1322                 if self._input_data.get(job, None) is None:
1323                     self._input_data[job] = pd.Series()
1324                 self._input_data[job][str(build_nr)] = build_data
1325
1326                 self._cfg.set_input_file_name(job, build_nr,
1327                                               result["build"]["file-name"])
1328
1329             self._cfg.set_input_state(job, build_nr, result["state"])
1330
1331             for item in result["logs"]:
1332                 if item[0] == "INFO":
1333                     logging.info(item[1])
1334                 elif item[0] == "ERROR":
1335                     logging.error(item[1])
1336                 elif item[0] == "DEBUG":
1337                     logging.debug(item[1])
1338                 elif item[0] == "CRITICAL":
1339                     logging.critical(item[1])
1340                 elif item[0] == "WARNING":
1341                     logging.warning(item[1])
1342
1343         del data_queue
1344
1345         # Terminate all workers
1346         for worker in workers:
1347             worker.terminate()
1348             worker.join()
1349
1350         logging.info("Done.")
1351
1352     @staticmethod
1353     def _end_of_tag(tag_filter, start=0, closer="'"):
1354         """Return the index of character in the string which is the end of tag.
1355
1356         :param tag_filter: The string where the end of tag is being searched.
1357         :param start: The index where the searching is stated.
1358         :param closer: The character which is the tag closer.
1359         :type tag_filter: str
1360         :type start: int
1361         :type closer: str
1362         :returns: The index of the tag closer.
1363         :rtype: int
1364         """
1365
1366         try:
1367             idx_opener = tag_filter.index(closer, start)
1368             return tag_filter.index(closer, idx_opener + 1)
1369         except ValueError:
1370             return None
1371
1372     @staticmethod
1373     def _condition(tag_filter):
1374         """Create a conditional statement from the given tag filter.
1375
1376         :param tag_filter: Filter based on tags from the element specification.
1377         :type tag_filter: str
1378         :returns: Conditional statement which can be evaluated.
1379         :rtype: str
1380         """
1381
1382         index = 0
1383         while True:
1384             index = InputData._end_of_tag(tag_filter, index)
1385             if index is None:
1386                 return tag_filter
1387             index += 1
1388             tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
1389
1390     def filter_data(self, element, params=None, data_set="tests",
1391                     continue_on_error=False):
1392         """Filter required data from the given jobs and builds.
1393
1394         The output data structure is:
1395
1396         - job 1
1397           - build 1
1398             - test (or suite) 1 ID:
1399               - param 1
1400               - param 2
1401               ...
1402               - param n
1403             ...
1404             - test (or suite) n ID:
1405             ...
1406           ...
1407           - build n
1408         ...
1409         - job n
1410
1411         :param element: Element which will use the filtered data.
1412         :param params: Parameters which will be included in the output. If None,
1413         all parameters are included.
1414         :param data_set: The set of data to be filtered: tests, suites,
1415         metadata.
1416         :param continue_on_error: Continue if there is error while reading the
1417         data. The Item will be empty then
1418         :type element: pandas.Series
1419         :type params: list
1420         :type data_set: str
1421         :type continue_on_error: bool
1422         :returns: Filtered data.
1423         :rtype pandas.Series
1424         """
1425
1426         try:
1427             if element["filter"] in ("all", "template"):
1428                 cond = "True"
1429             else:
1430                 cond = InputData._condition(element["filter"])
1431             logging.debug("   Filter: {0}".format(cond))
1432         except KeyError:
1433             logging.error("  No filter defined.")
1434             return None
1435
1436         if params is None:
1437             params = element.get("parameters", None)
1438             if params:
1439                 params.append("type")
1440
1441         data = pd.Series()
1442         try:
1443             for job, builds in element["data"].items():
1444                 data[job] = pd.Series()
1445                 for build in builds:
1446                     data[job][str(build)] = pd.Series()
1447                     try:
1448                         data_iter = self.data[job][str(build)][data_set].\
1449                             iteritems()
1450                     except KeyError:
1451                         if continue_on_error:
1452                             continue
1453                         else:
1454                             return None
1455                     for test_ID, test_data in data_iter:
1456                         if eval(cond, {"tags": test_data.get("tags", "")}):
1457                             data[job][str(build)][test_ID] = pd.Series()
1458                             if params is None:
1459                                 for param, val in test_data.items():
1460                                     data[job][str(build)][test_ID][param] = val
1461                             else:
1462                                 for param in params:
1463                                     try:
1464                                         data[job][str(build)][test_ID][param] =\
1465                                             test_data[param]
1466                                     except KeyError:
1467                                         data[job][str(build)][test_ID][param] =\
1468                                             "No Data"
1469             return data
1470
1471         except (KeyError, IndexError, ValueError) as err:
1472             logging.error("   Missing mandatory parameter in the element "
1473                           "specification: {0}".format(err))
1474             return None
1475         except AttributeError:
1476             return None
1477         except SyntaxError:
1478             logging.error("   The filter '{0}' is not correct. Check if all "
1479                           "tags are enclosed by apostrophes.".format(cond))
1480             return None
1481
1482     @staticmethod
1483     def merge_data(data):
1484         """Merge data from more jobs and builds to a simple data structure.
1485
1486         The output data structure is:
1487
1488         - test (suite) 1 ID:
1489           - param 1
1490           - param 2
1491           ...
1492           - param n
1493         ...
1494         - test (suite) n ID:
1495         ...
1496
1497         :param data: Data to merge.
1498         :type data: pandas.Series
1499         :returns: Merged data.
1500         :rtype: pandas.Series
1501         """
1502
1503         logging.info("    Merging data ...")
1504
1505         merged_data = pd.Series()
1506         for _, builds in data.iteritems():
1507             for _, item in builds.iteritems():
1508                 for ID, item_data in item.iteritems():
1509                     merged_data[ID] = item_data
1510
1511         return merged_data