resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import copy
  23 import re
  24 import resource
  25 import pandas as pd
  26 import logging
  27 import prettytable
  28
  29 from robot.api import ExecutionResult, ResultVisitor
  30 from robot import errors
  31 from collections import OrderedDict
  32 from string import replace
  33 from os import remove
  34 from datetime import datetime as dt
  35 from datetime import timedelta
  36 from json import loads
  37 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
  38
  39 from input_data_files import download_and_unzip_data_file
  40
  41
  42 # Separator used in file names
  43 SEPARATOR = "__"
  44
  45
  46 class ExecutionChecker(ResultVisitor):
  47     """Class to traverse through the test suite structure.
  48
  49     The functionality implemented in this class generates a json structure:
  50
  51     Performance tests:
  52
  53     {
  54         "metadata": {
  55             "generated": "Timestamp",
  56             "version": "SUT version",
  57             "job": "Jenkins job name",
  58             "build": "Information about the build"
  59         },
  60         "suites": {
  61             "Suite long name 1": {
  62                 "name": Suite name,
  63                 "doc": "Suite 1 documentation",
  64                 "parent": "Suite 1 parent",
  65                 "level": "Level of the suite in the suite hierarchy"
  66             }
  67             "Suite long name N": {
  68                 "name": Suite name,
  69                 "doc": "Suite N documentation",
  70                 "parent": "Suite 2 parent",
  71                 "level": "Level of the suite in the suite hierarchy"
  72             }
  73         }
  74         "tests": {
  75             # NDRPDR tests:
  76             "ID": {
  77                 "name": "Test name",
  78                 "parent": "Name of the parent of the test",
  79                 "doc": "Test documentation",
  80                 "msg": "Test message",
  81                 "conf-history": "DUT1 and DUT2 VAT History",
  82                 "show-run": "Show Run",
  83                 "tags": ["tag 1", "tag 2", "tag n"],
  84                 "type": "NDRPDR",
  85                 "status": "PASS" | "FAIL",
  86                 "throughput": {
  87                     "NDR": {
  88                         "LOWER": float,
  89                         "UPPER": float
  90                     },
  91                     "PDR": {
  92                         "LOWER": float,
  93                         "UPPER": float
  94                     }
  95                 },
  96                 "latency": {
  97                     "NDR": {
  98                         "direction1": {
  99                             "min": float,
 100                             "avg": float,
 101                             "max": float,
 102                             "hdrh": str
 103                         },
 104                         "direction2": {
 105                             "min": float,
 106                             "avg": float,
 107                             "max": float,
 108                             "hdrh": str
 109                         }
 110                     },
 111                     "PDR": {
 112                         "direction1": {
 113                             "min": float,
 114                             "avg": float,
 115                             "max": float,
 116                             "hdrh": str
 117                         },
 118                         "direction2": {
 119                             "min": float,
 120                             "avg": float,
 121                             "max": float,
 122                             "hdrh": str
 123                         }
 124                     }
 125                 }
 126             }
 127
 128             # TCP tests:
 129             "ID": {
 130                 "name": "Test name",
 131                 "parent": "Name of the parent of the test",
 132                 "doc": "Test documentation",
 133                 "msg": "Test message",
 134                 "tags": ["tag 1", "tag 2", "tag n"],
 135                 "type": "TCP",
 136                 "status": "PASS" | "FAIL",
 137                 "result": int
 138             }
 139
 140             # MRR, BMRR tests:
 141             "ID": {
 142                 "name": "Test name",
 143                 "parent": "Name of the parent of the test",
 144                 "doc": "Test documentation",
 145                 "msg": "Test message",
 146                 "tags": ["tag 1", "tag 2", "tag n"],
 147                 "type": "MRR" | "BMRR",
 148                 "status": "PASS" | "FAIL",
 149                 "result": {
 150                     "receive-rate": AvgStdevMetadata,
 151                 }
 152             }
 153
 154             "ID" {
 155                 # next test
 156             }
 157         }
 158     }
 159
 160
 161     Functional tests:
 162
 163     {
 164         "metadata": {  # Optional
 165             "version": "VPP version",
 166             "job": "Jenkins job name",
 167             "build": "Information about the build"
 168         },
 169         "suites": {
 170             "Suite name 1": {
 171                 "doc": "Suite 1 documentation",
 172                 "parent": "Suite 1 parent",
 173                 "level": "Level of the suite in the suite hierarchy"
 174             }
 175             "Suite name N": {
 176                 "doc": "Suite N documentation",
 177                 "parent": "Suite 2 parent",
 178                 "level": "Level of the suite in the suite hierarchy"
 179             }
 180         }
 181         "tests": {
 182             "ID": {
 183                 "name": "Test name",
 184                 "parent": "Name of the parent of the test",
 185                 "doc": "Test documentation"
 186                 "msg": "Test message"
 187                 "tags": ["tag 1", "tag 2", "tag n"],
 188                 "conf-history": "DUT1 and DUT2 VAT History"
 189                 "show-run": "Show Run"
 190                 "status": "PASS" | "FAIL"
 191             },
 192             "ID" {
 193                 # next test
 194             }
 195         }
 196     }
 197
 198     .. note:: ID is the lowercase full path to the test.
 199     """
 200
 201     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 202     REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
 203
 204     REGEX_PLR_RATE = re.compile(r'PLRsearch lower bound::?\s(\d+.\d+).*\n'
 205                                 r'PLRsearch upper bound::?\s(\d+.\d+)')
 206
 207     REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 208                                    r'NDR_UPPER:\s(\d+.\d+).*\n'
 209                                    r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 210                                    r'PDR_UPPER:\s(\d+.\d+)')
 211
 212     REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 213                                   r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
 214
 215     REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
 216                                  r'[\D\d]*')
 217
 218     REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|"
 219                                    r"VPP Version:\s*|VPP version:\s*)(.*)")
 220
 221     REGEX_VERSION_DPDK = re.compile(r"(DPDK version:\s*|DPDK Version:\s*)(.*)")
 222
 223     REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s(\d*).*$')
 224
 225     REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 226                            r'tx\s(\d*),\srx\s(\d*)')
 227
 228     REGEX_BMRR = re.compile(r'Maximum Receive Rate trial results'
 229                             r' in packets per second: \[(.*)\]')
 230
 231     REGEX_RECONF_LOSS = re.compile(r'Packets lost due to reconfig: (\d*)')
 232     REGEX_RECONF_TIME = re.compile(r'Implied time lost: (\d*.[\de-]*)')
 233
 234     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 235
 236     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 237
 238     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 239
 240     REGEX_TC_NUMBER = re.compile(r'tc\d{2}-')
 241
 242     REGEX_TC_PAPI_CLI = re.compile(r'.*\((\d+.\d+.\d+.\d+.) - (.*)\)')
 243
 244     def __init__(self, metadata, mapping, ignore):
 245         """Initialisation.
 246
 247         :param metadata: Key-value pairs to be included in "metadata" part of
 248             JSON structure.
 249         :param mapping: Mapping of the old names of test cases to the new
 250             (actual) one.
 251         :param ignore: List of TCs to be ignored.
 252         :type metadata: dict
 253         :type mapping: dict
 254         :type ignore: list
 255         """
 256
 257         # Type of message to parse out from the test messages
 258         self._msg_type = None
 259
 260         # VPP version
 261         self._version = None
 262
 263         # Timestamp
 264         self._timestamp = None
 265
 266         # Testbed. The testbed is identified by TG node IP address.
 267         self._testbed = None
 268
 269         # Mapping of TCs long names
 270         self._mapping = mapping
 271
 272         # Ignore list
 273         self._ignore = ignore
 274
 275         # Number of VAT History messages found:
 276         # 0 - no message
 277         # 1 - VAT History of DUT1
 278         # 2 - VAT History of DUT2
 279         self._lookup_kw_nr = 0
 280         self._conf_history_lookup_nr = 0
 281
 282         # Number of Show Running messages found
 283         # 0 - no message
 284         # 1 - Show run message found
 285         self._show_run_lookup_nr = 0
 286
 287         # Test ID of currently processed test- the lowercase full path to the
 288         # test
 289         self._test_ID = None
 290
 291         # The main data structure
 292         self._data = {
 293             "metadata": OrderedDict(),
 294             "suites": OrderedDict(),
 295             "tests": OrderedDict()
 296         }
 297
 298         # Save the provided metadata
 299         for key, val in metadata.items():
 300             self._data["metadata"][key] = val
 301
 302         # Dictionary defining the methods used to parse different types of
 303         # messages
 304         self.parse_msg = {
 305             "timestamp": self._get_timestamp,
 306             "vpp-version": self._get_vpp_version,
 307             "dpdk-version": self._get_dpdk_version,
 308             "teardown-vat-history": self._get_vat_history,
 309             "teardown-papi-history": self._get_papi_history,
 310             "test-show-runtime": self._get_show_run,
 311             "testbed": self._get_testbed
 312         }
 313
 314     @property
 315     def data(self):
 316         """Getter - Data parsed from the XML file.
 317
 318         :returns: Data parsed from the XML file.
 319         :rtype: dict
 320         """
 321         return self._data
 322
 323     def _get_testbed(self, msg):
 324         """Called when extraction of testbed IP is required.
 325         The testbed is identified by TG node IP address.
 326
 327         :param msg: Message to process.
 328         :type msg: Message
 329         :returns: Nothing.
 330         """
 331
 332         if msg.message.count("Setup of TG node"):
 333             reg_tg_ip = re.compile(
 334                 r'Setup of TG node (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}) done')
 335             try:
 336                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 337             except (KeyError, ValueError, IndexError, AttributeError):
 338                 pass
 339             finally:
 340                 self._data["metadata"]["testbed"] = self._testbed
 341                 self._msg_type = None
 342
 343     def _get_vpp_version(self, msg):
 344         """Called when extraction of VPP version is required.
 345
 346         :param msg: Message to process.
 347         :type msg: Message
 348         :returns: Nothing.
 349         """
 350
 351         if msg.message.count("return STDOUT Version:") or \
 352             msg.message.count("VPP Version:") or \
 353             msg.message.count("VPP version:"):
 354             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 355                                 group(2))
 356             self._data["metadata"]["version"] = self._version
 357             self._msg_type = None
 358
 359     def _get_dpdk_version(self, msg):
 360         """Called when extraction of DPDK version is required.
 361
 362         :param msg: Message to process.
 363         :type msg: Message
 364         :returns: Nothing.
 365         """
 366
 367         if msg.message.count("DPDK Version:"):
 368             try:
 369                 self._version = str(re.search(
 370                     self.REGEX_VERSION_DPDK, msg.message). group(2))
 371                 self._data["metadata"]["version"] = self._version
 372             except IndexError:
 373                 pass
 374             finally:
 375                 self._msg_type = None
 376
 377     def _get_timestamp(self, msg):
 378         """Called when extraction of timestamp is required.
 379
 380         :param msg: Message to process.
 381         :type msg: Message
 382         :returns: Nothing.
 383         """
 384
 385         self._timestamp = msg.timestamp[:14]
 386         self._data["metadata"]["generated"] = self._timestamp
 387         self._msg_type = None
 388
 389     def _get_vat_history(self, msg):
 390         """Called when extraction of VAT command history is required.
 391
 392         :param msg: Message to process.
 393         :type msg: Message
 394         :returns: Nothing.
 395         """
 396         if msg.message.count("VAT command history:"):
 397             self._conf_history_lookup_nr += 1
 398             if self._conf_history_lookup_nr == 1:
 399                 self._data["tests"][self._test_ID]["conf-history"] = str()
 400             else:
 401                 self._msg_type = None
 402             text = re.sub("\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 403                           "VAT command history:", "", msg.message, count=1). \
 404                 replace("\n\n", "\n").replace('\n', ' |br| ').\
 405                 replace('\r', '').replace('"', "'")
 406
 407             self._data["tests"][self._test_ID]["conf-history"] += " |br| "
 408             self._data["tests"][self._test_ID]["conf-history"] += \
 409                 "**DUT" + str(self._conf_history_lookup_nr) + ":** " + text
 410
 411     def _get_papi_history(self, msg):
 412         """Called when extraction of PAPI command history is required.
 413
 414         :param msg: Message to process.
 415         :type msg: Message
 416         :returns: Nothing.
 417         """
 418         if msg.message.count("PAPI command history:"):
 419             self._conf_history_lookup_nr += 1
 420             if self._conf_history_lookup_nr == 1:
 421                 self._data["tests"][self._test_ID]["conf-history"] = str()
 422             else:
 423                 self._msg_type = None
 424             text = re.sub("\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} "
 425                           "PAPI command history:", "", msg.message, count=1). \
 426                 replace("\n\n", "\n").replace('\n', ' |br| ').\
 427                 replace('\r', '').replace('"', "'")
 428
 429             self._data["tests"][self._test_ID]["conf-history"] += " |br| "
 430             self._data["tests"][self._test_ID]["conf-history"] += \
 431                 "**DUT" + str(self._conf_history_lookup_nr) + ":** " + text
 432
 433     def _get_show_run(self, msg):
 434         """Called when extraction of VPP operational data (output of CLI command
 435         Show Runtime) is required.
 436
 437         :param msg: Message to process.
 438         :type msg: Message
 439         :returns: Nothing.
 440         """
 441         if not "show-run" in self._data["tests"][self._test_ID].keys():
 442             self._data["tests"][self._test_ID]["show-run"] = str()
 443
 444         if msg.message.count("stats runtime"):
 445             host = str(re.search(self.REGEX_TC_PAPI_CLI, msg.message).\
 446                        group(1))
 447             socket = str(re.search(self.REGEX_TC_PAPI_CLI, msg.message).\
 448                          group(2))
 449             message = str(msg.message).replace(' ', '').replace('\n', '').\
 450                 replace("'", '"').replace('b"', '"').replace('u"', '"').\
 451                 split(":",1)[1]
 452             runtime = loads(message)
 453             try:
 454                 threads_nr = len(runtime[0]["clocks"])
 455             except (IndexError, KeyError):
 456                 return
 457             tbl_hdr = ["Name", "Calls", "Vectors", "Suspends", "Clocks",
 458                        "Vectors/Calls"]
 459             table = [[tbl_hdr, ] for _ in range(threads_nr)]
 460             for item in runtime:
 461                 for idx in range(threads_nr):
 462                     name = format(item["name"])
 463                     calls = format(item["calls"][idx])
 464                     vectors = format(item["vectors"][idx])
 465                     suspends = format(item["suspends"][idx])
 466                     if item["vectors"][idx] > 0:
 467                         clocks = format(
 468                             item["clocks"][idx]/item["vectors"][idx], ".2e")
 469                     elif item["calls"][idx] > 0:
 470                         clocks = format(
 471                             item["clocks"][idx]/item["calls"][idx], ".2e")
 472                     elif item["suspends"][idx] > 0:
 473                         clocks = format(
 474                             item["clocks"][idx]/item["suspends"][idx], ".2e")
 475                     else:
 476                         clocks = 0
 477                     if item["calls"][idx] > 0:
 478                         vectors_call = format(
 479                             item["vectors"][idx]/item["calls"][idx], ".2f")
 480                     else:
 481                         vectors_call = format(0, ".2f")
 482                     if int(calls) + int(vectors) + int(suspends):
 483                         table[idx].append([
 484                             name, calls, vectors, suspends, clocks, vectors_call
 485                         ])
 486             text = ""
 487             for idx in range(threads_nr):
 488                 text += "Thread {idx} ".format(idx=idx)
 489                 text += "vpp_main\n" if idx == 0 else \
 490                     "vpp_wk_{idx}\n".format(idx=idx-1)
 491                 txt_table = None
 492                 for row in table[idx]:
 493                     if txt_table is None:
 494                         txt_table = prettytable.PrettyTable(row)
 495                     else:
 496                         if any(row[1:]):
 497                             txt_table.add_row(row)
 498                 txt_table.set_style(prettytable.MSWORD_FRIENDLY)
 499                 txt_table.align["Name"] = "l"
 500                 txt_table.align["Calls"] = "r"
 501                 txt_table.align["Vectors"] = "r"
 502                 txt_table.align["Suspends"] = "r"
 503                 txt_table.align["Clocks"] = "r"
 504                 txt_table.align["Vectors/Calls"] = "r"
 505
 506                 text += txt_table.get_string(sortby="Name") + '\n'
 507             text = (" \n **DUT: {host}/{socket}** \n {text}".
 508                     format(host=host, socket=socket, text=text))
 509             text = text.replace('\n', ' |br| ').replace('\r', '').\
 510                 replace('"', "'")
 511             self._data["tests"][self._test_ID]["show-run"] += text
 512
 513     def _get_ndrpdr_throughput(self, msg):
 514         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 515         message.
 516
 517         :param msg: The test message to be parsed.
 518         :type msg: str
 519         :returns: Parsed data as a dict and the status (PASS/FAIL).
 520         :rtype: tuple(dict, str)
 521         """
 522
 523         throughput = {
 524             "NDR": {"LOWER": -1.0, "UPPER": -1.0},
 525             "PDR": {"LOWER": -1.0, "UPPER": -1.0}
 526         }
 527         status = "FAIL"
 528         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 529
 530         if groups is not None:
 531             try:
 532                 throughput["NDR"]["LOWER"] = float(groups.group(1))
 533                 throughput["NDR"]["UPPER"] = float(groups.group(2))
 534                 throughput["PDR"]["LOWER"] = float(groups.group(3))
 535                 throughput["PDR"]["UPPER"] = float(groups.group(4))
 536                 status = "PASS"
 537             except (IndexError, ValueError):
 538                 pass
 539
 540         return throughput, status
 541
 542     def _get_plr_throughput(self, msg):
 543         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 544         message.
 545
 546         :param msg: The test message to be parsed.
 547         :type msg: str
 548         :returns: Parsed data as a dict and the status (PASS/FAIL).
 549         :rtype: tuple(dict, str)
 550         """
 551
 552         throughput = {
 553             "LOWER": -1.0,
 554             "UPPER": -1.0
 555         }
 556         status = "FAIL"
 557         groups = re.search(self.REGEX_PLR_RATE, msg)
 558
 559         if groups is not None:
 560             try:
 561                 throughput["LOWER"] = float(groups.group(1))
 562                 throughput["UPPER"] = float(groups.group(2))
 563                 status = "PASS"
 564             except (IndexError, ValueError):
 565                 pass
 566
 567         return throughput, status
 568
 569     def _get_ndrpdr_latency(self, msg):
 570         """Get LATENCY from the test message.
 571
 572         :param msg: The test message to be parsed.
 573         :type msg: str
 574         :returns: Parsed data as a dict and the status (PASS/FAIL).
 575         :rtype: tuple(dict, str)
 576         """
 577         latency_default = {"min": -1.0, "avg": -1.0, "max": -1.0, "hdrh": ""}
 578         latency = {
 579             "NDR": {
 580                 "direction1": copy.copy(latency_default),
 581                 "direction2": copy.copy(latency_default)
 582             },
 583             "PDR": {
 584                 "direction1": copy.copy(latency_default),
 585                 "direction2": copy.copy(latency_default)
 586             }
 587         }
 588         status = "FAIL"
 589         groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 590
 591         def process_latency(in_str):
 592             """Return object with parsed latency values.
 593
 594             TODO: Define class for the return type.
 595
 596             :param in_str: Input string, min/avg/max/hdrh format.
 597             :type in_str: str
 598             :returns: Dict with corresponding keys, except hdrh float values.
 599             :rtype dict:
 600             :throws IndexError: If in_str does not have enough substrings.
 601             :throws ValueError: If a substring does not convert to float.
 602             """
 603             in_list = in_str.split('/')
 604
 605             rval = {
 606                 "min": float(in_list[0]),
 607                 "avg": float(in_list[1]),
 608                 "max": float(in_list[2]),
 609                 "hdrh": ""
 610             }
 611
 612             if len(in_list) == 4:
 613                 rval["hdrh"] = str(in_list[3])
 614
 615             return rval
 616
 617         if groups is not None:
 618             try:
 619                 latency["NDR"]["direction1"] = process_latency(groups.group(1))
 620                 latency["NDR"]["direction2"] = process_latency(groups.group(2))
 621                 latency["PDR"]["direction1"] = process_latency(groups.group(3))
 622                 latency["PDR"]["direction2"] = process_latency(groups.group(4))
 623                 status = "PASS"
 624             except (IndexError, ValueError):
 625                 pass
 626
 627         return latency, status
 628
 629     def visit_suite(self, suite):
 630         """Implements traversing through the suite and its direct children.
 631
 632         :param suite: Suite to process.
 633         :type suite: Suite
 634         :returns: Nothing.
 635         """
 636         if self.start_suite(suite) is not False:
 637             suite.suites.visit(self)
 638             suite.tests.visit(self)
 639             self.end_suite(suite)
 640
 641     def start_suite(self, suite):
 642         """Called when suite starts.
 643
 644         :param suite: Suite to process.
 645         :type suite: Suite
 646         :returns: Nothing.
 647         """
 648
 649         try:
 650             parent_name = suite.parent.name
 651         except AttributeError:
 652             return
 653
 654         doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
 655             replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
 656         doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
 657
 658         self._data["suites"][suite.longname.lower().replace('"', "'").
 659             replace(" ", "_")] = {
 660                 "name": suite.name.lower(),
 661                 "doc": doc_str,
 662                 "parent": parent_name,
 663                 "level": len(suite.longname.split("."))
 664             }
 665
 666         suite.keywords.visit(self)
 667
 668     def end_suite(self, suite):
 669         """Called when suite ends.
 670
 671         :param suite: Suite to process.
 672         :type suite: Suite
 673         :returns: Nothing.
 674         """
 675         pass
 676
 677     def visit_test(self, test):
 678         """Implements traversing through the test.
 679
 680         :param test: Test to process.
 681         :type test: Test
 682         :returns: Nothing.
 683         """
 684         if self.start_test(test) is not False:
 685             test.keywords.visit(self)
 686             self.end_test(test)
 687
 688     def start_test(self, test):
 689         """Called when test starts.
 690
 691         :param test: Test to process.
 692         :type test: Test
 693         :returns: Nothing.
 694         """
 695
 696         longname_orig = test.longname.lower()
 697
 698         # Check the ignore list
 699         if longname_orig in self._ignore:
 700             return
 701
 702         tags = [str(tag) for tag in test.tags]
 703         test_result = dict()
 704
 705         # Change the TC long name and name if defined in the mapping table
 706         longname = self._mapping.get(longname_orig, None)
 707         if longname is not None:
 708             name = longname.split('.')[-1]
 709             logging.debug("{0}\n{1}\n{2}\n{3}".format(
 710                 self._data["metadata"], longname_orig, longname, name))
 711         else:
 712             longname = longname_orig
 713             name = test.name.lower()
 714
 715         # Remove TC number from the TC long name (backward compatibility):
 716         self._test_ID = re.sub(self.REGEX_TC_NUMBER, "", longname)
 717         # Remove TC number from the TC name (not needed):
 718         test_result["name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
 719
 720         test_result["parent"] = test.parent.name.lower()
 721         test_result["tags"] = tags
 722         doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
 723             replace('\r', '').replace('[', ' |br| [')
 724         test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
 725         test_result["msg"] = test.message.replace('\n', ' |br| '). \
 726             replace('\r', '').replace('"', "'")
 727         test_result["type"] = "FUNC"
 728         test_result["status"] = test.status
 729
 730         if "PERFTEST" in tags:
 731             # Replace info about cores (e.g. -1c-) with the info about threads
 732             # and cores (e.g. -1t1c-) in the long test case names and in the
 733             # test case names if necessary.
 734             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_ID)
 735             if not groups:
 736                 tag_count = 0
 737                 tag_tc = str()
 738                 for tag in test_result["tags"]:
 739                     groups = re.search(self.REGEX_TC_TAG, tag)
 740                     if groups:
 741                         tag_count += 1
 742                         tag_tc = tag
 743
 744                 if tag_count == 1:
 745                     self._test_ID = re.sub(self.REGEX_TC_NAME_NEW,
 746                                            "-{0}-".format(tag_tc.lower()),
 747                                            self._test_ID,
 748                                            count=1)
 749                     test_result["name"] = re.sub(self.REGEX_TC_NAME_NEW,
 750                                                  "-{0}-".format(tag_tc.lower()),
 751                                                  test_result["name"],
 752                                                  count=1)
 753                 else:
 754                     test_result["status"] = "FAIL"
 755                     self._data["tests"][self._test_ID] = test_result
 756                     logging.debug("The test '{0}' has no or more than one "
 757                                   "multi-threading tags.".format(self._test_ID))
 758                     logging.debug("Tags: {0}".format(test_result["tags"]))
 759                     return
 760
 761         if test.status == "PASS" and ("NDRPDRDISC" in tags or
 762                                       "NDRPDR" in tags or
 763                                       "SOAK" in tags or
 764                                       "TCP" in tags or
 765                                       "MRR" in tags or
 766                                       "BMRR" in tags or
 767                                       "RECONF" in tags):
 768             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 769             if "NDRDISC" in tags:
 770                 test_result["type"] = "NDR"
 771             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 772             elif "PDRDISC" in tags:
 773                 test_result["type"] = "PDR"
 774             elif "NDRPDR" in tags:
 775                 test_result["type"] = "NDRPDR"
 776             elif "SOAK" in tags:
 777                 test_result["type"] = "SOAK"
 778             elif "TCP" in tags:
 779                 test_result["type"] = "TCP"
 780             elif "MRR" in tags:
 781                 test_result["type"] = "MRR"
 782             elif "FRMOBL" in tags or "BMRR" in tags:
 783                 test_result["type"] = "BMRR"
 784             elif "RECONF" in tags:
 785                 test_result["type"] = "RECONF"
 786             else:
 787                 test_result["status"] = "FAIL"
 788                 self._data["tests"][self._test_ID] = test_result
 789                 return
 790
 791             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 792             if test_result["type"] in ("NDR", "PDR"):
 793                 try:
 794                     rate_value = str(re.search(
 795                         self.REGEX_RATE, test.message).group(1))
 796                 except AttributeError:
 797                     rate_value = "-1"
 798                 try:
 799                     rate_unit = str(re.search(
 800                         self.REGEX_RATE, test.message).group(2))
 801                 except AttributeError:
 802                     rate_unit = "-1"
 803
 804                 test_result["throughput"] = dict()
 805                 test_result["throughput"]["value"] = \
 806                     int(rate_value.split('.')[0])
 807                 test_result["throughput"]["unit"] = rate_unit
 808                 test_result["latency"] = \
 809                     self._get_latency(test.message, test_result["type"])
 810                 if test_result["type"] == "PDR":
 811                     test_result["lossTolerance"] = str(re.search(
 812                         self.REGEX_TOLERANCE, test.message).group(1))
 813
 814             elif test_result["type"] in ("NDRPDR", ):
 815                 test_result["throughput"], test_result["status"] = \
 816                     self._get_ndrpdr_throughput(test.message)
 817                 test_result["latency"], test_result["status"] = \
 818                     self._get_ndrpdr_latency(test.message)
 819
 820             elif test_result["type"] in ("SOAK", ):
 821                 test_result["throughput"], test_result["status"] = \
 822                     self._get_plr_throughput(test.message)
 823
 824             elif test_result["type"] in ("TCP", ):
 825                 groups = re.search(self.REGEX_TCP, test.message)
 826                 test_result["result"] = int(groups.group(2))
 827
 828             elif test_result["type"] in ("MRR", "BMRR"):
 829                 test_result["result"] = dict()
 830                 groups = re.search(self.REGEX_BMRR, test.message)
 831                 if groups is not None:
 832                     items_str = groups.group(1)
 833                     items_float = [float(item.strip()) for item
 834                                    in items_str.split(",")]
 835                     metadata = AvgStdevMetadataFactory.from_data(items_float)
 836                     # Next two lines have been introduced in CSIT-1179,
 837                     # to be removed in CSIT-1180.
 838                     metadata.size = 1
 839                     metadata.stdev = 0.0
 840                     test_result["result"]["receive-rate"] = metadata
 841                 else:
 842                     groups = re.search(self.REGEX_MRR, test.message)
 843                     test_result["result"]["receive-rate"] = \
 844                         AvgStdevMetadataFactory.from_data([
 845                             float(groups.group(3)) / float(groups.group(1)), ])
 846
 847             elif test_result["type"] == "RECONF":
 848                 test_result["result"] = None
 849                 try:
 850                     grps_loss = re.search(self.REGEX_RECONF_LOSS, test.message)
 851                     grps_time = re.search(self.REGEX_RECONF_TIME, test.message)
 852                     test_result["result"] = {
 853                         "loss": int(grps_loss.group(1)),
 854                         "time": float(grps_time.group(1))
 855                     }
 856                 except (AttributeError, IndexError, ValueError, TypeError):
 857                     test_result["status"] = "FAIL"
 858
 859         self._data["tests"][self._test_ID] = test_result
 860
 861     def end_test(self, test):
 862         """Called when test ends.
 863
 864         :param test: Test to process.
 865         :type test: Test
 866         :returns: Nothing.
 867         """
 868         pass
 869
 870     def visit_keyword(self, keyword):
 871         """Implements traversing through the keyword and its child keywords.
 872
 873         :param keyword: Keyword to process.
 874         :type keyword: Keyword
 875         :returns: Nothing.
 876         """
 877         if self.start_keyword(keyword) is not False:
 878             self.end_keyword(keyword)
 879
 880     def start_keyword(self, keyword):
 881         """Called when keyword starts. Default implementation does nothing.
 882
 883         :param keyword: Keyword to process.
 884         :type keyword: Keyword
 885         :returns: Nothing.
 886         """
 887         try:
 888             if keyword.type == "setup":
 889                 self.visit_setup_kw(keyword)
 890             elif keyword.type == "teardown":
 891                 self._lookup_kw_nr = 0
 892                 self.visit_teardown_kw(keyword)
 893             else:
 894                 self._lookup_kw_nr = 0
 895                 self.visit_test_kw(keyword)
 896         except AttributeError:
 897             pass
 898
 899     def end_keyword(self, keyword):
 900         """Called when keyword ends. Default implementation does nothing.
 901
 902         :param keyword: Keyword to process.
 903         :type keyword: Keyword
 904         :returns: Nothing.
 905         """
 906         pass
 907
 908     def visit_test_kw(self, test_kw):
 909         """Implements traversing through the test keyword and its child
 910         keywords.
 911
 912         :param test_kw: Keyword to process.
 913         :type test_kw: Keyword
 914         :returns: Nothing.
 915         """
 916         for keyword in test_kw.keywords:
 917             if self.start_test_kw(keyword) is not False:
 918                 self.visit_test_kw(keyword)
 919                 self.end_test_kw(keyword)
 920
 921     def start_test_kw(self, test_kw):
 922         """Called when test keyword starts. Default implementation does
 923         nothing.
 924
 925         :param test_kw: Keyword to process.
 926         :type test_kw: Keyword
 927         :returns: Nothing.
 928         """
 929         if test_kw.name.count("Show Runtime Counters On All Duts"):
 930             self._lookup_kw_nr += 1
 931             self._show_run_lookup_nr = 0
 932             self._msg_type = "test-show-runtime"
 933         elif test_kw.name.count("Install Dpdk Test") and not self._version:
 934             self._msg_type = "dpdk-version"
 935         else:
 936             return
 937         test_kw.messages.visit(self)
 938
 939     def end_test_kw(self, test_kw):
 940         """Called when keyword ends. Default implementation does nothing.
 941
 942         :param test_kw: Keyword to process.
 943         :type test_kw: Keyword
 944         :returns: Nothing.
 945         """
 946         pass
 947
 948     def visit_setup_kw(self, setup_kw):
 949         """Implements traversing through the teardown keyword and its child
 950         keywords.
 951
 952         :param setup_kw: Keyword to process.
 953         :type setup_kw: Keyword
 954         :returns: Nothing.
 955         """
 956         for keyword in setup_kw.keywords:
 957             if self.start_setup_kw(keyword) is not False:
 958                 self.visit_setup_kw(keyword)
 959                 self.end_setup_kw(keyword)
 960
 961     def start_setup_kw(self, setup_kw):
 962         """Called when teardown keyword starts. Default implementation does
 963         nothing.
 964
 965         :param setup_kw: Keyword to process.
 966         :type setup_kw: Keyword
 967         :returns: Nothing.
 968         """
 969         if setup_kw.name.count("Show Vpp Version On All Duts") \
 970                 and not self._version:
 971             self._msg_type = "vpp-version"
 972         elif setup_kw.name.count("Set Global Variable") \
 973                 and not self._timestamp:
 974             self._msg_type = "timestamp"
 975         elif setup_kw.name.count("Setup Framework") and not self._testbed:
 976             self._msg_type = "testbed"
 977         else:
 978             return
 979         setup_kw.messages.visit(self)
 980
 981     def end_setup_kw(self, setup_kw):
 982         """Called when keyword ends. Default implementation does nothing.
 983
 984         :param setup_kw: Keyword to process.
 985         :type setup_kw: Keyword
 986         :returns: Nothing.
 987         """
 988         pass
 989
 990     def visit_teardown_kw(self, teardown_kw):
 991         """Implements traversing through the teardown keyword and its child
 992         keywords.
 993
 994         :param teardown_kw: Keyword to process.
 995         :type teardown_kw: Keyword
 996         :returns: Nothing.
 997         """
 998         for keyword in teardown_kw.keywords:
 999             if self.start_teardown_kw(keyword) is not False:
1000                 self.visit_teardown_kw(keyword)
1001                 self.end_teardown_kw(keyword)
1002
1003     def start_teardown_kw(self, teardown_kw):
1004         """Called when teardown keyword starts. Default implementation does
1005         nothing.
1006
1007         :param teardown_kw: Keyword to process.
1008         :type teardown_kw: Keyword
1009         :returns: Nothing.
1010         """
1011
1012         if teardown_kw.name.count("Show Vat History On All Duts"):
1013             self._conf_history_lookup_nr = 0
1014             self._msg_type = "teardown-vat-history"
1015             teardown_kw.messages.visit(self)
1016         elif teardown_kw.name.count("Show Papi History On All Duts"):
1017             self._conf_history_lookup_nr = 0
1018             self._msg_type = "teardown-papi-history"
1019             teardown_kw.messages.visit(self)
1020
1021     def end_teardown_kw(self, teardown_kw):
1022         """Called when keyword ends. Default implementation does nothing.
1023
1024         :param teardown_kw: Keyword to process.
1025         :type teardown_kw: Keyword
1026         :returns: Nothing.
1027         """
1028         pass
1029
1030     def visit_message(self, msg):
1031         """Implements visiting the message.
1032
1033         :param msg: Message to process.
1034         :type msg: Message
1035         :returns: Nothing.
1036         """
1037         if self.start_message(msg) is not False:
1038             self.end_message(msg)
1039
1040     def start_message(self, msg):
1041         """Called when message starts. Get required information from messages:
1042         - VPP version.
1043
1044         :param msg: Message to process.
1045         :type msg: Message
1046         :returns: Nothing.
1047         """
1048
1049         if self._msg_type:
1050             self.parse_msg[self._msg_type](msg)
1051
1052     def end_message(self, msg):
1053         """Called when message ends. Default implementation does nothing.
1054
1055         :param msg: Message to process.
1056         :type msg: Message
1057         :returns: Nothing.
1058         """
1059         pass
1060
1061
1062 class InputData:
1063     """Input data
1064
1065     The data is extracted from output.xml files generated by Jenkins jobs and
1066     stored in pandas' DataFrames.
1067
1068     The data structure:
1069     - job name
1070       - build number
1071         - metadata
1072           (as described in ExecutionChecker documentation)
1073         - suites
1074           (as described in ExecutionChecker documentation)
1075         - tests
1076           (as described in ExecutionChecker documentation)
1077     """
1078
1079     def __init__(self, spec):
1080         """Initialization.
1081
1082         :param spec: Specification.
1083         :type spec: Specification
1084         """
1085
1086         # Specification:
1087         self._cfg = spec
1088
1089         # Data store:
1090         self._input_data = pd.Series()
1091
1092     @property
1093     def data(self):
1094         """Getter - Input data.
1095
1096         :returns: Input data
1097         :rtype: pandas.Series
1098         """
1099         return self._input_data
1100
1101     def metadata(self, job, build):
1102         """Getter - metadata
1103
1104         :param job: Job which metadata we want.
1105         :param build: Build which metadata we want.
1106         :type job: str
1107         :type build: str
1108         :returns: Metadata
1109         :rtype: pandas.Series
1110         """
1111
1112         return self.data[job][build]["metadata"]
1113
1114     def suites(self, job, build):
1115         """Getter - suites
1116
1117         :param job: Job which suites we want.
1118         :param build: Build which suites we want.
1119         :type job: str
1120         :type build: str
1121         :returns: Suites.
1122         :rtype: pandas.Series
1123         """
1124
1125         return self.data[job][str(build)]["suites"]
1126
1127     def tests(self, job, build):
1128         """Getter - tests
1129
1130         :param job: Job which tests we want.
1131         :param build: Build which tests we want.
1132         :type job: str
1133         :type build: str
1134         :returns: Tests.
1135         :rtype: pandas.Series
1136         """
1137
1138         return self.data[job][build]["tests"]
1139
1140     def _parse_tests(self, job, build, log):
1141         """Process data from robot output.xml file and return JSON structured
1142         data.
1143
1144         :param job: The name of job which build output data will be processed.
1145         :param build: The build which output data will be processed.
1146         :param log: List of log messages.
1147         :type job: str
1148         :type build: dict
1149         :type log: list of tuples (severity, msg)
1150         :returns: JSON data structure.
1151         :rtype: dict
1152         """
1153
1154         metadata = {
1155             "job": job,
1156             "build": build
1157         }
1158
1159         with open(build["file-name"], 'r') as data_file:
1160             try:
1161                 result = ExecutionResult(data_file)
1162             except errors.DataError as err:
1163                 log.append(("ERROR", "Error occurred while parsing output.xml: "
1164                                      "{0}".format(err)))
1165                 return None
1166         checker = ExecutionChecker(metadata, self._cfg.mapping,
1167                                    self._cfg.ignore)
1168         result.visit(checker)
1169
1170         return checker.data
1171
1172     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1173         """Download and parse the input data file.
1174
1175         :param pid: PID of the process executing this method.
1176         :param job: Name of the Jenkins job which generated the processed input
1177             file.
1178         :param build: Information about the Jenkins build which generated the
1179             processed input file.
1180         :param repeat: Repeat the download specified number of times if not
1181             successful.
1182         :type pid: int
1183         :type job: str
1184         :type build: dict
1185         :type repeat: int
1186         """
1187
1188         logs = list()
1189
1190         logs.append(("INFO", "  Processing the job/build: {0}: {1}".
1191                      format(job, build["build"])))
1192
1193         state = "failed"
1194         success = False
1195         data = None
1196         do_repeat = repeat
1197         while do_repeat:
1198             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1199                                                    logs)
1200             if success:
1201                 break
1202             do_repeat -= 1
1203         if not success:
1204             logs.append(("ERROR", "It is not possible to download the input "
1205                                   "data file from the job '{job}', build "
1206                                   "'{build}', or it is damaged. Skipped.".
1207                          format(job=job, build=build["build"])))
1208         if success:
1209             logs.append(("INFO", "    Processing data from the build '{0}' ...".
1210                          format(build["build"])))
1211             data = self._parse_tests(job, build, logs)
1212             if data is None:
1213                 logs.append(("ERROR", "Input data file from the job '{job}', "
1214                                       "build '{build}' is damaged. Skipped.".
1215                              format(job=job, build=build["build"])))
1216             else:
1217                 state = "processed"
1218
1219             try:
1220                 remove(build["file-name"])
1221             except OSError as err:
1222                 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
1223                              format(build["file-name"], repr(err))))
1224
1225         # If the time-period is defined in the specification file, remove all
1226         # files which are outside the time period.
1227         timeperiod = self._cfg.input.get("time-period", None)
1228         if timeperiod and data:
1229             now = dt.utcnow()
1230             timeperiod = timedelta(int(timeperiod))
1231             metadata = data.get("metadata", None)
1232             if metadata:
1233                 generated = metadata.get("generated", None)
1234                 if generated:
1235                     generated = dt.strptime(generated, "%Y%m%d %H:%M")
1236                     if (now - generated) > timeperiod:
1237                         # Remove the data and the file:
1238                         state = "removed"
1239                         data = None
1240                         logs.append(
1241                             ("INFO",
1242                              "    The build {job}/{build} is outdated, will be "
1243                              "removed".format(job=job, build=build["build"])))
1244         logs.append(("INFO", "  Done."))
1245
1246         for level, line in logs:
1247             if level == "INFO":
1248                 logging.info(line)
1249             elif level == "ERROR":
1250                 logging.error(line)
1251             elif level == "DEBUG":
1252                 logging.debug(line)
1253             elif level == "CRITICAL":
1254                 logging.critical(line)
1255             elif level == "WARNING":
1256                 logging.warning(line)
1257
1258         return {"data": data, "state": state, "job": job, "build": build}
1259
1260     def download_and_parse_data(self, repeat=1):
1261         """Download the input data files, parse input data from input files and
1262         store in pandas' Series.
1263
1264         :param repeat: Repeat the download specified number of times if not
1265             successful.
1266         :type repeat: int
1267         """
1268
1269         logging.info("Downloading and parsing input files ...")
1270
1271         for job, builds in self._cfg.builds.items():
1272             for build in builds:
1273
1274                 result = self._download_and_parse_build(job, build, repeat)
1275                 build_nr = result["build"]["build"]
1276
1277                 if result["data"]:
1278                     data = result["data"]
1279                     build_data = pd.Series({
1280                         "metadata": pd.Series(
1281                             data["metadata"].values(),
1282                             index=data["metadata"].keys()),
1283                         "suites": pd.Series(data["suites"].values(),
1284                                             index=data["suites"].keys()),
1285                         "tests": pd.Series(data["tests"].values(),
1286                                            index=data["tests"].keys())})
1287
1288                     if self._input_data.get(job, None) is None:
1289                         self._input_data[job] = pd.Series()
1290                     self._input_data[job][str(build_nr)] = build_data
1291
1292                     self._cfg.set_input_file_name(
1293                         job, build_nr, result["build"]["file-name"])
1294
1295                 self._cfg.set_input_state(job, build_nr, result["state"])
1296
1297                 logging.info("Memory allocation: {0:,d}MB".format(
1298                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000))
1299
1300         logging.info("Done.")
1301
1302     @staticmethod
1303     def _end_of_tag(tag_filter, start=0, closer="'"):
1304         """Return the index of character in the string which is the end of tag.
1305
1306         :param tag_filter: The string where the end of tag is being searched.
1307         :param start: The index where the searching is stated.
1308         :param closer: The character which is the tag closer.
1309         :type tag_filter: str
1310         :type start: int
1311         :type closer: str
1312         :returns: The index of the tag closer.
1313         :rtype: int
1314         """
1315
1316         try:
1317             idx_opener = tag_filter.index(closer, start)
1318             return tag_filter.index(closer, idx_opener + 1)
1319         except ValueError:
1320             return None
1321
1322     @staticmethod
1323     def _condition(tag_filter):
1324         """Create a conditional statement from the given tag filter.
1325
1326         :param tag_filter: Filter based on tags from the element specification.
1327         :type tag_filter: str
1328         :returns: Conditional statement which can be evaluated.
1329         :rtype: str
1330         """
1331
1332         index = 0
1333         while True:
1334             index = InputData._end_of_tag(tag_filter, index)
1335             if index is None:
1336                 return tag_filter
1337             index += 1
1338             tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
1339
1340     def filter_data(self, element, params=None, data=None, data_set="tests",
1341                     continue_on_error=False):
1342         """Filter required data from the given jobs and builds.
1343
1344         The output data structure is:
1345
1346         - job 1
1347           - build 1
1348             - test (or suite) 1 ID:
1349               - param 1
1350               - param 2
1351               ...
1352               - param n
1353             ...
1354             - test (or suite) n ID:
1355             ...
1356           ...
1357           - build n
1358         ...
1359         - job n
1360
1361         :param element: Element which will use the filtered data.
1362         :param params: Parameters which will be included in the output. If None,
1363             all parameters are included.
1364         :param data: If not None, this data is used instead of data specified
1365             in the element.
1366         :param data_set: The set of data to be filtered: tests, suites,
1367             metadata.
1368         :param continue_on_error: Continue if there is error while reading the
1369             data. The Item will be empty then
1370         :type element: pandas.Series
1371         :type params: list
1372         :type data: dict
1373         :type data_set: str
1374         :type continue_on_error: bool
1375         :returns: Filtered data.
1376         :rtype pandas.Series
1377         """
1378
1379         try:
1380             if element["filter"] in ("all", "template"):
1381                 cond = "True"
1382             else:
1383                 cond = InputData._condition(element["filter"])
1384             logging.debug("   Filter: {0}".format(cond))
1385         except KeyError:
1386             logging.error("  No filter defined.")
1387             return None
1388
1389         if params is None:
1390             params = element.get("parameters", None)
1391             if params:
1392                 params.append("type")
1393
1394         data_to_filter = data if data else element["data"]
1395         data = pd.Series()
1396         try:
1397             for job, builds in data_to_filter.items():
1398                 data[job] = pd.Series()
1399                 for build in builds:
1400                     data[job][str(build)] = pd.Series()
1401                     try:
1402                         data_iter = self.data[job][str(build)][data_set].\
1403                             iteritems()
1404                     except KeyError:
1405                         if continue_on_error:
1406                             continue
1407                         else:
1408                             return None
1409                     for test_ID, test_data in data_iter:
1410                         if eval(cond, {"tags": test_data.get("tags", "")}):
1411                             data[job][str(build)][test_ID] = pd.Series()
1412                             if params is None:
1413                                 for param, val in test_data.items():
1414                                     data[job][str(build)][test_ID][param] = val
1415                             else:
1416                                 for param in params:
1417                                     try:
1418                                         data[job][str(build)][test_ID][param] =\
1419                                             test_data[param]
1420                                     except KeyError:
1421                                         data[job][str(build)][test_ID][param] =\
1422                                             "No Data"
1423             return data
1424
1425         except (KeyError, IndexError, ValueError) as err:
1426             logging.error("   Missing mandatory parameter in the element "
1427                           "specification: {0}".format(err))
1428             return None
1429         except AttributeError:
1430             return None
1431         except SyntaxError:
1432             logging.error("   The filter '{0}' is not correct. Check if all "
1433                           "tags are enclosed by apostrophes.".format(cond))
1434             return None
1435
1436     def filter_tests_by_name(self, element, params=None, data_set="tests",
1437                              continue_on_error=False):
1438         """Filter required data from the given jobs and builds.
1439
1440         The output data structure is:
1441
1442         - job 1
1443           - build 1
1444             - test (or suite) 1 ID:
1445               - param 1
1446               - param 2
1447               ...
1448               - param n
1449             ...
1450             - test (or suite) n ID:
1451             ...
1452           ...
1453           - build n
1454         ...
1455         - job n
1456
1457         :param element: Element which will use the filtered data.
1458         :param params: Parameters which will be included in the output. If None,
1459         all parameters are included.
1460         :param data_set: The set of data to be filtered: tests, suites,
1461         metadata.
1462         :param continue_on_error: Continue if there is error while reading the
1463         data. The Item will be empty then
1464         :type element: pandas.Series
1465         :type params: list
1466         :type data_set: str
1467         :type continue_on_error: bool
1468         :returns: Filtered data.
1469         :rtype pandas.Series
1470         """
1471
1472         include = element.get("include", None)
1473         if not include:
1474             logging.warning("No tests to include, skipping the element.")
1475             return None
1476
1477         if params is None:
1478             params = element.get("parameters", None)
1479             if params:
1480                 params.append("type")
1481
1482         data = pd.Series()
1483         try:
1484             for job, builds in element["data"].items():
1485                 data[job] = pd.Series()
1486                 for build in builds:
1487                     data[job][str(build)] = pd.Series()
1488                     for test in include:
1489                         try:
1490                             reg_ex = re.compile(str(test).lower())
1491                             for test_ID in self.data[job][str(build)]\
1492                                     [data_set].keys():
1493                                 if re.match(reg_ex, str(test_ID).lower()):
1494                                     test_data = self.data[job][str(build)]\
1495                                         [data_set][test_ID]
1496                                     data[job][str(build)][test_ID] = pd.Series()
1497                                     if params is None:
1498                                         for param, val in test_data.items():
1499                                             data[job][str(build)][test_ID]\
1500                                                 [param] = val
1501                                     else:
1502                                         for param in params:
1503                                             try:
1504                                                 data[job][str(build)][test_ID]\
1505                                                     [param] = test_data[param]
1506                                             except KeyError:
1507                                                 data[job][str(build)][test_ID]\
1508                                                     [param] = "No Data"
1509                         except KeyError as err:
1510                             logging.error("{err!r}".format(err=err))
1511                             if continue_on_error:
1512                                 continue
1513                             else:
1514                                 return None
1515             return data
1516
1517         except (KeyError, IndexError, ValueError) as err:
1518             logging.error("Missing mandatory parameter in the element "
1519                           "specification: {err!r}".format(err=err))
1520             return None
1521         except AttributeError as err:
1522             logging.error("{err!r}".format(err=err))
1523             return None
1524
1525
1526     @staticmethod
1527     def merge_data(data):
1528         """Merge data from more jobs and builds to a simple data structure.
1529
1530         The output data structure is:
1531
1532         - test (suite) 1 ID:
1533           - param 1
1534           - param 2
1535           ...
1536           - param n
1537         ...
1538         - test (suite) n ID:
1539         ...
1540
1541         :param data: Data to merge.
1542         :type data: pandas.Series
1543         :returns: Merged data.
1544         :rtype: pandas.Series
1545         """
1546
1547         logging.info("    Merging data ...")
1548
1549         merged_data = pd.Series()
1550         for _, builds in data.iteritems():
1551             for _, item in builds.iteritems():
1552                 for ID, item_data in item.iteritems():
1553                     merged_data[ID] = item_data
1554
1555         return merged_data