resources/tools/presentation/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import re
  23 import resource
  24 import pandas as pd
  25 import logging
  26 import prettytable
  27
  28 from robot.api import ExecutionResult, ResultVisitor
  29 from robot import errors
  30 from collections import OrderedDict
  31 from string import replace
  32 from os import remove
  33 from os.path import join
  34 from datetime import datetime as dt
  35 from datetime import timedelta
  36 from json import loads
  37 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
  38
  39 from input_data_files import download_and_unzip_data_file
  40
  41
  42 # Separator used in file names
  43 SEPARATOR = "__"
  44
  45
  46 class ExecutionChecker(ResultVisitor):
  47     """Class to traverse through the test suite structure.
  48
  49     The functionality implemented in this class generates a json structure:
  50
  51     Performance tests:
  52
  53     {
  54         "metadata": {
  55             "generated": "Timestamp",
  56             "version": "SUT version",
  57             "job": "Jenkins job name",
  58             "build": "Information about the build"
  59         },
  60         "suites": {
  61             "Suite long name 1": {
  62                 "name": Suite name,
  63                 "doc": "Suite 1 documentation",
  64                 "parent": "Suite 1 parent",
  65                 "level": "Level of the suite in the suite hierarchy"
  66             }
  67             "Suite long name N": {
  68                 "name": Suite name,
  69                 "doc": "Suite N documentation",
  70                 "parent": "Suite 2 parent",
  71                 "level": "Level of the suite in the suite hierarchy"
  72             }
  73         }
  74         "tests": {
  75             # NDRPDR tests:
  76             "ID": {
  77                 "name": "Test name",
  78                 "parent": "Name of the parent of the test",
  79                 "doc": "Test documentation",
  80                 "msg": "Test message",
  81                 "conf-history": "DUT1 and DUT2 VAT History",
  82                 "show-run": "Show Run",
  83                 "tags": ["tag 1", "tag 2", "tag n"],
  84                 "type": "NDRPDR",
  85                 "status": "PASS" | "FAIL",
  86                 "throughput": {
  87                     "NDR": {
  88                         "LOWER": float,
  89                         "UPPER": float
  90                     },
  91                     "PDR": {
  92                         "LOWER": float,
  93                         "UPPER": float
  94                     }
  95                 },
  96                 "latency": {
  97                     "NDR": {
  98                         "direction1": {
  99                             "min": float,
 100                             "avg": float,
 101                             "max": float
 102                         },
 103                         "direction2": {
 104                             "min": float,
 105                             "avg": float,
 106                             "max": float
 107                         }
 108                     },
 109                     "PDR": {
 110                         "direction1": {
 111                             "min": float,
 112                             "avg": float,
 113                             "max": float
 114                         },
 115                         "direction2": {
 116                             "min": float,
 117                             "avg": float,
 118                             "max": float
 119                         }
 120                     }
 121                 }
 122             }
 123
 124             # TCP tests:
 125             "ID": {
 126                 "name": "Test name",
 127                 "parent": "Name of the parent of the test",
 128                 "doc": "Test documentation",
 129                 "msg": "Test message",
 130                 "tags": ["tag 1", "tag 2", "tag n"],
 131                 "type": "TCP",
 132                 "status": "PASS" | "FAIL",
 133                 "result": int
 134             }
 135
 136             # MRR, BMRR tests:
 137             "ID": {
 138                 "name": "Test name",
 139                 "parent": "Name of the parent of the test",
 140                 "doc": "Test documentation",
 141                 "msg": "Test message",
 142                 "tags": ["tag 1", "tag 2", "tag n"],
 143                 "type": "MRR" | "BMRR",
 144                 "status": "PASS" | "FAIL",
 145                 "result": {
 146                     "receive-rate": AvgStdevMetadata,
 147                 }
 148             }
 149
 150             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 151             # NDRPDRDISC tests:
 152             "ID": {
 153                 "name": "Test name",
 154                 "parent": "Name of the parent of the test",
 155                 "doc": "Test documentation",
 156                 "msg": "Test message",
 157                 "tags": ["tag 1", "tag 2", "tag n"],
 158                 "type": "PDR" | "NDR",
 159                 "status": "PASS" | "FAIL",
 160                 "throughput": {  # Only type: "PDR" | "NDR"
 161                     "value": int,
 162                     "unit": "pps" | "bps" | "percentage"
 163                 },
 164                 "latency": {  # Only type: "PDR" | "NDR"
 165                     "direction1": {
 166                         "100": {
 167                             "min": int,
 168                             "avg": int,
 169                             "max": int
 170                         },
 171                         "50": {  # Only for NDR
 172                             "min": int,
 173                             "avg": int,
 174                             "max": int
 175                         },
 176                         "10": {  # Only for NDR
 177                             "min": int,
 178                             "avg": int,
 179                             "max": int
 180                         }
 181                     },
 182                     "direction2": {
 183                         "100": {
 184                             "min": int,
 185                             "avg": int,
 186                             "max": int
 187                         },
 188                         "50": {  # Only for NDR
 189                             "min": int,
 190                             "avg": int,
 191                             "max": int
 192                         },
 193                         "10": {  # Only for NDR
 194                             "min": int,
 195                             "avg": int,
 196                             "max": int
 197                         }
 198                     }
 199                 },
 200                 "lossTolerance": "lossTolerance",  # Only type: "PDR"
 201                 "conf-history": "DUT1 and DUT2 VAT History"
 202                 "show-run": "Show Run"
 203             },
 204             "ID" {
 205                 # next test
 206             }
 207         }
 208     }
 209
 210
 211     Functional tests:
 212
 213     {
 214         "metadata": {  # Optional
 215             "version": "VPP version",
 216             "job": "Jenkins job name",
 217             "build": "Information about the build"
 218         },
 219         "suites": {
 220             "Suite name 1": {
 221                 "doc": "Suite 1 documentation",
 222                 "parent": "Suite 1 parent",
 223                 "level": "Level of the suite in the suite hierarchy"
 224             }
 225             "Suite name N": {
 226                 "doc": "Suite N documentation",
 227                 "parent": "Suite 2 parent",
 228                 "level": "Level of the suite in the suite hierarchy"
 229             }
 230         }
 231         "tests": {
 232             "ID": {
 233                 "name": "Test name",
 234                 "parent": "Name of the parent of the test",
 235                 "doc": "Test documentation"
 236                 "msg": "Test message"
 237                 "tags": ["tag 1", "tag 2", "tag n"],
 238                 "conf-history": "DUT1 and DUT2 VAT History"
 239                 "show-run": "Show Run"
 240                 "status": "PASS" | "FAIL"
 241             },
 242             "ID" {
 243                 # next test
 244             }
 245         }
 246     }
 247
 248     .. note:: ID is the lowercase full path to the test.
 249     """
 250
 251     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 252     REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
 253
 254     REGEX_PLR_RATE = re.compile(r'PLRsearch lower bound::\s(\d+.\d+).*\n'
 255                                 r'PLRsearch upper bound::\s(\d+.\d+)')
 256
 257     REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 258                                    r'NDR_UPPER:\s(\d+.\d+).*\n'
 259                                    r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 260                                    r'PDR_UPPER:\s(\d+.\d+)')
 261
 262     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 263     REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
 264                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 265                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
 266                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 267                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
 268                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 269                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
 270
 271     REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
 272                                r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 273                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
 274
 275     REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 276                                   r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
 277
 278     REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
 279                                  r'[\D\d]*')
 280
 281     REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|"
 282                                    r"VPP Version:\s*|VPP version:\s*)(.*)")
 283
 284     REGEX_VERSION_DPDK = re.compile(r"(DPDK version:\s*|DPDK Version:\s*)(.*)")
 285
 286     REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
 287
 288     REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 289                            r'tx\s(\d*),\srx\s(\d*)')
 290
 291     REGEX_BMRR = re.compile(r'Maximum Receive Rate trial results'
 292                             r' in packets per second: \[(.*)\]')
 293
 294     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 295
 296     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 297
 298     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 299
 300     REGEX_TC_NUMBER = re.compile(r'tc[0-9]{2}-')
 301
 302     def __init__(self, metadata, mapping, ignore):
 303         """Initialisation.
 304
 305         :param metadata: Key-value pairs to be included in "metadata" part of
 306             JSON structure.
 307         :param mapping: Mapping of the old names of test cases to the new
 308             (actual) one.
 309         :param ignore: List of TCs to be ignored.
 310         :type metadata: dict
 311         :type mapping: dict
 312         :type ignore: list
 313         """
 314
 315         # Type of message to parse out from the test messages
 316         self._msg_type = None
 317
 318         # VPP version
 319         self._version = None
 320
 321         # Timestamp
 322         self._timestamp = None
 323
 324         # Testbed. The testbed is identified by TG node IP address.
 325         self._testbed = None
 326
 327         # Mapping of TCs long names
 328         self._mapping = mapping
 329
 330         # Ignore list
 331         self._ignore = ignore
 332
 333         # Number of VAT History messages found:
 334         # 0 - no message
 335         # 1 - VAT History of DUT1
 336         # 2 - VAT History of DUT2
 337         self._lookup_kw_nr = 0
 338         self._conf_history_lookup_nr = 0
 339
 340         # Number of Show Running messages found
 341         # 0 - no message
 342         # 1 - Show run message found
 343         self._show_run_lookup_nr = 0
 344
 345         # Test ID of currently processed test- the lowercase full path to the
 346         # test
 347         self._test_ID = None
 348
 349         # The main data structure
 350         self._data = {
 351             "metadata": OrderedDict(),
 352             "suites": OrderedDict(),
 353             "tests": OrderedDict()
 354         }
 355
 356         # Save the provided metadata
 357         for key, val in metadata.items():
 358             self._data["metadata"][key] = val
 359
 360         # Dictionary defining the methods used to parse different types of
 361         # messages
 362         self.parse_msg = {
 363             "timestamp": self._get_timestamp,
 364             "vpp-version": self._get_vpp_version,
 365             "dpdk-version": self._get_dpdk_version,
 366             "teardown-vat-history": self._get_vat_history,
 367             "teardown-papi-history": self._get_papi_history,
 368             "test-show-runtime": self._get_show_run,
 369             "testbed": self._get_testbed
 370         }
 371
 372     @property
 373     def data(self):
 374         """Getter - Data parsed from the XML file.
 375
 376         :returns: Data parsed from the XML file.
 377         :rtype: dict
 378         """
 379         return self._data
 380
 381     def _get_testbed(self, msg):
 382         """Called when extraction of testbed IP is required.
 383         The testbed is identified by TG node IP address.
 384
 385         :param msg: Message to process.
 386         :type msg: Message
 387         :returns: Nothing.
 388         """
 389
 390         if msg.message.count("Setup of TG node"):
 391             reg_tg_ip = re.compile(
 392                 r'Setup of TG node (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}) done')
 393             try:
 394                 self._testbed = str(re.search(reg_tg_ip, msg.message).group(1))
 395             except (KeyError, ValueError, IndexError, AttributeError):
 396                 pass
 397             finally:
 398                 self._data["metadata"]["testbed"] = self._testbed
 399                 self._msg_type = None
 400
 401     def _get_vpp_version(self, msg):
 402         """Called when extraction of VPP version is required.
 403
 404         :param msg: Message to process.
 405         :type msg: Message
 406         :returns: Nothing.
 407         """
 408
 409         if msg.message.count("return STDOUT Version:") or \
 410             msg.message.count("VPP Version:") or \
 411             msg.message.count("VPP version:"):
 412             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 413                                 group(2))
 414             self._data["metadata"]["version"] = self._version
 415             self._msg_type = None
 416
 417     def _get_dpdk_version(self, msg):
 418         """Called when extraction of DPDK version is required.
 419
 420         :param msg: Message to process.
 421         :type msg: Message
 422         :returns: Nothing.
 423         """
 424
 425         if msg.message.count("DPDK Version:"):
 426             try:
 427                 self._version = str(re.search(
 428                     self.REGEX_VERSION_DPDK, msg.message). group(2))
 429                 self._data["metadata"]["version"] = self._version
 430             except IndexError:
 431                 pass
 432             finally:
 433                 self._msg_type = None
 434
 435     def _get_timestamp(self, msg):
 436         """Called when extraction of timestamp is required.
 437
 438         :param msg: Message to process.
 439         :type msg: Message
 440         :returns: Nothing.
 441         """
 442
 443         self._timestamp = msg.timestamp[:14]
 444         self._data["metadata"]["generated"] = self._timestamp
 445         self._msg_type = None
 446
 447     def _get_vat_history(self, msg):
 448         """Called when extraction of VAT command history is required.
 449
 450         :param msg: Message to process.
 451         :type msg: Message
 452         :returns: Nothing.
 453         """
 454         if msg.message.count("VAT command history:"):
 455             self._conf_history_lookup_nr += 1
 456             if self._conf_history_lookup_nr == 1:
 457                 self._data["tests"][self._test_ID]["conf-history"] = str()
 458             else:
 459                 self._msg_type = None
 460             text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
 461                           "VAT command history:", "", msg.message, count=1). \
 462                 replace("\n\n", "\n").replace('\n', ' |br| ').\
 463                 replace('\r', '').replace('"', "'")
 464
 465             self._data["tests"][self._test_ID]["conf-history"] += " |br| "
 466             self._data["tests"][self._test_ID]["conf-history"] += \
 467                 "**DUT" + str(self._conf_history_lookup_nr) + ":** " + text
 468
 469     def _get_papi_history(self, msg):
 470         """Called when extraction of PAPI command history is required.
 471
 472         :param msg: Message to process.
 473         :type msg: Message
 474         :returns: Nothing.
 475         """
 476         if msg.message.count("PAPI command history:"):
 477             self._conf_history_lookup_nr += 1
 478             if self._conf_history_lookup_nr == 1:
 479                 self._data["tests"][self._test_ID]["conf-history"] = str()
 480             else:
 481                 self._msg_type = None
 482             text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
 483                           "PAPI command history:", "", msg.message, count=1). \
 484                 replace("\n\n", "\n").replace('\n', ' |br| ').\
 485                 replace('\r', '').replace('"', "'")
 486
 487             self._data["tests"][self._test_ID]["conf-history"] += " |br| "
 488             self._data["tests"][self._test_ID]["conf-history"] += \
 489                 "**DUT" + str(self._conf_history_lookup_nr) + ":** " + text
 490
 491     def _get_show_run(self, msg):
 492         """Called when extraction of VPP operational data (output of CLI command
 493         Show Runtime) is required.
 494
 495         :param msg: Message to process.
 496         :type msg: Message
 497         :returns: Nothing.
 498         """
 499         if msg.message.count("Runtime:"):
 500             self._show_run_lookup_nr += 1
 501             if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
 502                 self._data["tests"][self._test_ID]["show-run"] = str()
 503             if self._lookup_kw_nr > 1:
 504                 self._msg_type = None
 505             if self._show_run_lookup_nr == 1:
 506                 message = str(msg.message).replace(' ', '').replace('\n', '').\
 507                     replace("'", '"').replace('b"', '"').replace('u"', '"')[8:]
 508                 runtime = loads(message)
 509                 try:
 510                     threads_nr = len(runtime[0]["clocks"])
 511                 except (IndexError, KeyError):
 512                     return
 513                 tbl_hdr = ["Name", "Calls", "Vectors", "Suspends", "Clocks"]
 514                 table = [[tbl_hdr, ] for _ in range(threads_nr)]
 515                 for item in runtime:
 516                     for idx in range(threads_nr):
 517                         table[idx].append([
 518                             item["name"],
 519                             item["calls"][idx],
 520                             item["vectors"][idx],
 521                             item["suspends"][idx],
 522                             item["clocks"][idx]
 523                         ])
 524                 text = ""
 525                 for idx in range(threads_nr):
 526                     text += "Thread {idx} ".format(idx=idx)
 527                     text += "vpp_main\n" if idx == 0 else \
 528                         "vpp_wk_{idx}\n".format(idx=idx-1)
 529                     txt_table = None
 530                     for row in table[idx]:
 531                         if txt_table is None:
 532                             txt_table = prettytable.PrettyTable(row)
 533                         else:
 534                             if any(row[1:]):
 535                                 txt_table.add_row(row)
 536                     txt_table.set_style(prettytable.MSWORD_FRIENDLY)
 537                     txt_table.align["Name"] = "l"
 538                     txt_table.align["Calls"] = "r"
 539                     txt_table.align["Vectors"] = "r"
 540                     txt_table.align["Suspends"] = "r"
 541                     txt_table.align["Clocks"] = "r"
 542
 543                     text += txt_table.get_string(sortby="Name") + '\n'
 544
 545                 text = text.replace('\n', ' |br| ').replace('\r', '').\
 546                     replace('"', "'")
 547                 try:
 548                     self._data["tests"][self._test_ID]["show-run"] += " |br| "
 549                     self._data["tests"][self._test_ID]["show-run"] += \
 550                         "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
 551                 except KeyError:
 552                     pass
 553
 554     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 555     def _get_latency(self, msg, test_type):
 556         """Get the latency data from the test message.
 557
 558         :param msg: Message to be parsed.
 559         :param test_type: Type of the test - NDR or PDR.
 560         :type msg: str
 561         :type test_type: str
 562         :returns: Latencies parsed from the message.
 563         :rtype: dict
 564         """
 565
 566         if test_type == "NDR":
 567             groups = re.search(self.REGEX_LAT_NDR, msg)
 568             groups_range = range(1, 7)
 569         elif test_type == "PDR":
 570             groups = re.search(self.REGEX_LAT_PDR, msg)
 571             groups_range = range(1, 3)
 572         else:
 573             return {}
 574
 575         latencies = list()
 576         for idx in groups_range:
 577             try:
 578                 lat = [int(item) for item in str(groups.group(idx)).split('/')]
 579             except (AttributeError, ValueError):
 580                 lat = [-1, -1, -1]
 581             latencies.append(lat)
 582
 583         keys = ("min", "avg", "max")
 584         latency = {
 585             "direction1": {
 586             },
 587             "direction2": {
 588             }
 589         }
 590
 591         latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
 592         latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
 593         if test_type == "NDR":
 594             latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
 595             latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
 596             latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
 597             latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
 598
 599         return latency
 600
 601     def _get_ndrpdr_throughput(self, msg):
 602         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 603         message.
 604
 605         :param msg: The test message to be parsed.
 606         :type msg: str
 607         :returns: Parsed data as a dict and the status (PASS/FAIL).
 608         :rtype: tuple(dict, str)
 609         """
 610
 611         throughput = {
 612             "NDR": {"LOWER": -1.0, "UPPER": -1.0},
 613             "PDR": {"LOWER": -1.0, "UPPER": -1.0}
 614         }
 615         status = "FAIL"
 616         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 617
 618         if groups is not None:
 619             try:
 620                 throughput["NDR"]["LOWER"] = float(groups.group(1))
 621                 throughput["NDR"]["UPPER"] = float(groups.group(2))
 622                 throughput["PDR"]["LOWER"] = float(groups.group(3))
 623                 throughput["PDR"]["UPPER"] = float(groups.group(4))
 624                 status = "PASS"
 625             except (IndexError, ValueError):
 626                 pass
 627
 628         return throughput, status
 629
 630     def _get_plr_throughput(self, msg):
 631         """Get PLRsearch lower bound and PLRsearch upper bound from the test
 632         message.
 633
 634         :param msg: The test message to be parsed.
 635         :type msg: str
 636         :returns: Parsed data as a dict and the status (PASS/FAIL).
 637         :rtype: tuple(dict, str)
 638         """
 639
 640         throughput = {
 641             "LOWER": -1.0,
 642             "UPPER": -1.0
 643         }
 644         status = "FAIL"
 645         groups = re.search(self.REGEX_PLR_RATE, msg)
 646
 647         if groups is not None:
 648             try:
 649                 throughput["LOWER"] = float(groups.group(1))
 650                 throughput["UPPER"] = float(groups.group(2))
 651                 status = "PASS"
 652             except (IndexError, ValueError):
 653                 pass
 654
 655         return throughput, status
 656
 657     def _get_ndrpdr_latency(self, msg):
 658         """Get LATENCY from the test message.
 659
 660         :param msg: The test message to be parsed.
 661         :type msg: str
 662         :returns: Parsed data as a dict and the status (PASS/FAIL).
 663         :rtype: tuple(dict, str)
 664         """
 665
 666         latency = {
 667             "NDR": {
 668                 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
 669                 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
 670             },
 671             "PDR": {
 672                 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
 673                 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
 674             }
 675         }
 676         status = "FAIL"
 677         groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 678
 679         if groups is not None:
 680             keys = ("min", "avg", "max")
 681             try:
 682                 latency["NDR"]["direction1"] = dict(
 683                     zip(keys, [float(l) for l in groups.group(1).split('/')]))
 684                 latency["NDR"]["direction2"] = dict(
 685                     zip(keys, [float(l) for l in groups.group(2).split('/')]))
 686                 latency["PDR"]["direction1"] = dict(
 687                     zip(keys, [float(l) for l in groups.group(3).split('/')]))
 688                 latency["PDR"]["direction2"] = dict(
 689                     zip(keys, [float(l) for l in groups.group(4).split('/')]))
 690                 status = "PASS"
 691             except (IndexError, ValueError):
 692                 pass
 693
 694         return latency, status
 695
 696     def visit_suite(self, suite):
 697         """Implements traversing through the suite and its direct children.
 698
 699         :param suite: Suite to process.
 700         :type suite: Suite
 701         :returns: Nothing.
 702         """
 703         if self.start_suite(suite) is not False:
 704             suite.suites.visit(self)
 705             suite.tests.visit(self)
 706             self.end_suite(suite)
 707
 708     def start_suite(self, suite):
 709         """Called when suite starts.
 710
 711         :param suite: Suite to process.
 712         :type suite: Suite
 713         :returns: Nothing.
 714         """
 715
 716         try:
 717             parent_name = suite.parent.name
 718         except AttributeError:
 719             return
 720
 721         doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
 722             replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
 723         doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
 724
 725         self._data["suites"][suite.longname.lower().replace('"', "'").
 726             replace(" ", "_")] = {
 727                 "name": suite.name.lower(),
 728                 "doc": doc_str,
 729                 "parent": parent_name,
 730                 "level": len(suite.longname.split("."))
 731             }
 732
 733         suite.keywords.visit(self)
 734
 735     def end_suite(self, suite):
 736         """Called when suite ends.
 737
 738         :param suite: Suite to process.
 739         :type suite: Suite
 740         :returns: Nothing.
 741         """
 742         pass
 743
 744     def visit_test(self, test):
 745         """Implements traversing through the test.
 746
 747         :param test: Test to process.
 748         :type test: Test
 749         :returns: Nothing.
 750         """
 751         if self.start_test(test) is not False:
 752             test.keywords.visit(self)
 753             self.end_test(test)
 754
 755     def start_test(self, test):
 756         """Called when test starts.
 757
 758         :param test: Test to process.
 759         :type test: Test
 760         :returns: Nothing.
 761         """
 762
 763         longname_orig = test.longname.lower()
 764
 765         # Check the ignore list
 766         if longname_orig in self._ignore:
 767             return
 768
 769         tags = [str(tag) for tag in test.tags]
 770         test_result = dict()
 771
 772         # Change the TC long name and name if defined in the mapping table
 773         longname = self._mapping.get(longname_orig, None)
 774         if longname is not None:
 775             name = longname.split('.')[-1]
 776             logging.debug("{0}\n{1}\n{2}\n{3}".format(
 777                 self._data["metadata"], longname_orig, longname, name))
 778         else:
 779             longname = longname_orig
 780             name = test.name.lower()
 781
 782         # Remove TC number from the TC long name (backward compatibility):
 783         self._test_ID = re.sub(self.REGEX_TC_NUMBER, "", longname)
 784         # Remove TC number from the TC name (not needed):
 785         test_result["name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
 786
 787         test_result["parent"] = test.parent.name.lower()
 788         test_result["tags"] = tags
 789         doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
 790             replace('\r', '').replace('[', ' |br| [')
 791         test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
 792         test_result["msg"] = test.message.replace('\n', ' |br| '). \
 793             replace('\r', '').replace('"', "'")
 794         test_result["type"] = "FUNC"
 795         test_result["status"] = test.status
 796
 797         if "PERFTEST" in tags:
 798             # Replace info about cores (e.g. -1c-) with the info about threads
 799             # and cores (e.g. -1t1c-) in the long test case names and in the
 800             # test case names if necessary.
 801             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_ID)
 802             if not groups:
 803                 tag_count = 0
 804                 for tag in test_result["tags"]:
 805                     groups = re.search(self.REGEX_TC_TAG, tag)
 806                     if groups:
 807                         tag_count += 1
 808                         tag_tc = tag
 809
 810                 if tag_count == 1:
 811                     self._test_ID = re.sub(self.REGEX_TC_NAME_NEW,
 812                                            "-{0}-".format(tag_tc.lower()),
 813                                            self._test_ID,
 814                                            count=1)
 815                     test_result["name"] = re.sub(self.REGEX_TC_NAME_NEW,
 816                                                  "-{0}-".format(tag_tc.lower()),
 817                                                  test_result["name"],
 818                                                  count=1)
 819                 else:
 820                     test_result["status"] = "FAIL"
 821                     self._data["tests"][self._test_ID] = test_result
 822                     logging.debug("The test '{0}' has no or more than one "
 823                                   "multi-threading tags.".format(self._test_ID))
 824                     logging.debug("Tags: {0}".format(test_result["tags"]))
 825                     return
 826
 827         if test.status == "PASS" and ("NDRPDRDISC" in tags or
 828                                       "NDRPDR" in tags or
 829                                       "SOAK" in tags or
 830                                       "TCP" in tags or
 831                                       "MRR" in tags or
 832                                       "BMRR" in tags):
 833             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 834             if "NDRDISC" in tags:
 835                 test_result["type"] = "NDR"
 836             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 837             elif "PDRDISC" in tags:
 838                 test_result["type"] = "PDR"
 839             elif "NDRPDR" in tags:
 840                 test_result["type"] = "NDRPDR"
 841             elif "SOAK" in tags:
 842                 test_result["type"] = "SOAK"
 843             elif "TCP" in tags:
 844                 test_result["type"] = "TCP"
 845             elif "MRR" in tags:
 846                 test_result["type"] = "MRR"
 847             elif "FRMOBL" in tags or "BMRR" in tags:
 848                 test_result["type"] = "BMRR"
 849             else:
 850                 test_result["status"] = "FAIL"
 851                 self._data["tests"][self._test_ID] = test_result
 852                 return
 853
 854             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 855             if test_result["type"] in ("NDR", "PDR"):
 856                 try:
 857                     rate_value = str(re.search(
 858                         self.REGEX_RATE, test.message).group(1))
 859                 except AttributeError:
 860                     rate_value = "-1"
 861                 try:
 862                     rate_unit = str(re.search(
 863                         self.REGEX_RATE, test.message).group(2))
 864                 except AttributeError:
 865                     rate_unit = "-1"
 866
 867                 test_result["throughput"] = dict()
 868                 test_result["throughput"]["value"] = \
 869                     int(rate_value.split('.')[0])
 870                 test_result["throughput"]["unit"] = rate_unit
 871                 test_result["latency"] = \
 872                     self._get_latency(test.message, test_result["type"])
 873                 if test_result["type"] == "PDR":
 874                     test_result["lossTolerance"] = str(re.search(
 875                         self.REGEX_TOLERANCE, test.message).group(1))
 876
 877             elif test_result["type"] in ("NDRPDR", ):
 878                 test_result["throughput"], test_result["status"] = \
 879                     self._get_ndrpdr_throughput(test.message)
 880                 test_result["latency"], test_result["status"] = \
 881                     self._get_ndrpdr_latency(test.message)
 882
 883             elif test_result["type"] in ("SOAK", ):
 884                 test_result["throughput"], test_result["status"] = \
 885                     self._get_plr_throughput(test.message)
 886
 887             elif test_result["type"] in ("TCP", ):
 888                 groups = re.search(self.REGEX_TCP, test.message)
 889                 test_result["result"] = int(groups.group(2))
 890
 891             elif test_result["type"] in ("MRR", "BMRR"):
 892                 test_result["result"] = dict()
 893                 groups = re.search(self.REGEX_BMRR, test.message)
 894                 if groups is not None:
 895                     items_str = groups.group(1)
 896                     items_float = [float(item.strip()) for item
 897                                    in items_str.split(",")]
 898                     metadata = AvgStdevMetadataFactory.from_data(items_float)
 899                     # Next two lines have been introduced in CSIT-1179,
 900                     # to be removed in CSIT-1180.
 901                     metadata.size = 1
 902                     metadata.stdev = 0.0
 903                     test_result["result"]["receive-rate"] = metadata
 904                 else:
 905                     groups = re.search(self.REGEX_MRR, test.message)
 906                     test_result["result"]["receive-rate"] = \
 907                         AvgStdevMetadataFactory.from_data([
 908                             float(groups.group(3)) / float(groups.group(1)), ])
 909
 910         self._data["tests"][self._test_ID] = test_result
 911
 912     def end_test(self, test):
 913         """Called when test ends.
 914
 915         :param test: Test to process.
 916         :type test: Test
 917         :returns: Nothing.
 918         """
 919         pass
 920
 921     def visit_keyword(self, keyword):
 922         """Implements traversing through the keyword and its child keywords.
 923
 924         :param keyword: Keyword to process.
 925         :type keyword: Keyword
 926         :returns: Nothing.
 927         """
 928         if self.start_keyword(keyword) is not False:
 929             self.end_keyword(keyword)
 930
 931     def start_keyword(self, keyword):
 932         """Called when keyword starts. Default implementation does nothing.
 933
 934         :param keyword: Keyword to process.
 935         :type keyword: Keyword
 936         :returns: Nothing.
 937         """
 938         try:
 939             if keyword.type == "setup":
 940                 self.visit_setup_kw(keyword)
 941             elif keyword.type == "teardown":
 942                 self._lookup_kw_nr = 0
 943                 self.visit_teardown_kw(keyword)
 944             else:
 945                 self._lookup_kw_nr = 0
 946                 self.visit_test_kw(keyword)
 947         except AttributeError:
 948             pass
 949
 950     def end_keyword(self, keyword):
 951         """Called when keyword ends. Default implementation does nothing.
 952
 953         :param keyword: Keyword to process.
 954         :type keyword: Keyword
 955         :returns: Nothing.
 956         """
 957         pass
 958
 959     def visit_test_kw(self, test_kw):
 960         """Implements traversing through the test keyword and its child
 961         keywords.
 962
 963         :param test_kw: Keyword to process.
 964         :type test_kw: Keyword
 965         :returns: Nothing.
 966         """
 967         for keyword in test_kw.keywords:
 968             if self.start_test_kw(keyword) is not False:
 969                 self.visit_test_kw(keyword)
 970                 self.end_test_kw(keyword)
 971
 972     def start_test_kw(self, test_kw):
 973         """Called when test keyword starts. Default implementation does
 974         nothing.
 975
 976         :param test_kw: Keyword to process.
 977         :type test_kw: Keyword
 978         :returns: Nothing.
 979         """
 980         if test_kw.name.count("Show Runtime Counters On All Duts"):
 981             self._lookup_kw_nr += 1
 982             self._show_run_lookup_nr = 0
 983             self._msg_type = "test-show-runtime"
 984         elif test_kw.name.count("Install Dpdk Test") and not self._version:
 985             self._msg_type = "dpdk-version"
 986         else:
 987             return
 988         test_kw.messages.visit(self)
 989
 990     def end_test_kw(self, test_kw):
 991         """Called when keyword ends. Default implementation does nothing.
 992
 993         :param test_kw: Keyword to process.
 994         :type test_kw: Keyword
 995         :returns: Nothing.
 996         """
 997         pass
 998
 999     def visit_setup_kw(self, setup_kw):
1000         """Implements traversing through the teardown keyword and its child
1001         keywords.
1002
1003         :param setup_kw: Keyword to process.
1004         :type setup_kw: Keyword
1005         :returns: Nothing.
1006         """
1007         for keyword in setup_kw.keywords:
1008             if self.start_setup_kw(keyword) is not False:
1009                 self.visit_setup_kw(keyword)
1010                 self.end_setup_kw(keyword)
1011
1012     def start_setup_kw(self, setup_kw):
1013         """Called when teardown keyword starts. Default implementation does
1014         nothing.
1015
1016         :param setup_kw: Keyword to process.
1017         :type setup_kw: Keyword
1018         :returns: Nothing.
1019         """
1020         if setup_kw.name.count("Show Vpp Version On All Duts") \
1021                 and not self._version:
1022             self._msg_type = "vpp-version"
1023         elif setup_kw.name.count("Set Global Variable") \
1024                 and not self._timestamp:
1025             self._msg_type = "timestamp"
1026         elif setup_kw.name.count("Setup Framework") and not self._testbed:
1027             self._msg_type = "testbed"
1028         else:
1029             return
1030         setup_kw.messages.visit(self)
1031
1032     def end_setup_kw(self, setup_kw):
1033         """Called when keyword ends. Default implementation does nothing.
1034
1035         :param setup_kw: Keyword to process.
1036         :type setup_kw: Keyword
1037         :returns: Nothing.
1038         """
1039         pass
1040
1041     def visit_teardown_kw(self, teardown_kw):
1042         """Implements traversing through the teardown keyword and its child
1043         keywords.
1044
1045         :param teardown_kw: Keyword to process.
1046         :type teardown_kw: Keyword
1047         :returns: Nothing.
1048         """
1049         for keyword in teardown_kw.keywords:
1050             if self.start_teardown_kw(keyword) is not False:
1051                 self.visit_teardown_kw(keyword)
1052                 self.end_teardown_kw(keyword)
1053
1054     def start_teardown_kw(self, teardown_kw):
1055         """Called when teardown keyword starts. Default implementation does
1056         nothing.
1057
1058         :param teardown_kw: Keyword to process.
1059         :type teardown_kw: Keyword
1060         :returns: Nothing.
1061         """
1062
1063         if teardown_kw.name.count("Show Vat History On All Duts"):
1064             self._conf_history_lookup_nr = 0
1065             self._msg_type = "teardown-vat-history"
1066             teardown_kw.messages.visit(self)
1067         elif teardown_kw.name.count("Show Papi History On All Duts"):
1068             self._conf_history_lookup_nr = 0
1069             self._msg_type = "teardown-papi-history"
1070             teardown_kw.messages.visit(self)
1071
1072     def end_teardown_kw(self, teardown_kw):
1073         """Called when keyword ends. Default implementation does nothing.
1074
1075         :param teardown_kw: Keyword to process.
1076         :type teardown_kw: Keyword
1077         :returns: Nothing.
1078         """
1079         pass
1080
1081     def visit_message(self, msg):
1082         """Implements visiting the message.
1083
1084         :param msg: Message to process.
1085         :type msg: Message
1086         :returns: Nothing.
1087         """
1088         if self.start_message(msg) is not False:
1089             self.end_message(msg)
1090
1091     def start_message(self, msg):
1092         """Called when message starts. Get required information from messages:
1093         - VPP version.
1094
1095         :param msg: Message to process.
1096         :type msg: Message
1097         :returns: Nothing.
1098         """
1099
1100         if self._msg_type:
1101             self.parse_msg[self._msg_type](msg)
1102
1103     def end_message(self, msg):
1104         """Called when message ends. Default implementation does nothing.
1105
1106         :param msg: Message to process.
1107         :type msg: Message
1108         :returns: Nothing.
1109         """
1110         pass
1111
1112
1113 class InputData(object):
1114     """Input data
1115
1116     The data is extracted from output.xml files generated by Jenkins jobs and
1117     stored in pandas' DataFrames.
1118
1119     The data structure:
1120     - job name
1121       - build number
1122         - metadata
1123           (as described in ExecutionChecker documentation)
1124         - suites
1125           (as described in ExecutionChecker documentation)
1126         - tests
1127           (as described in ExecutionChecker documentation)
1128     """
1129
1130     def __init__(self, spec):
1131         """Initialization.
1132
1133         :param spec: Specification.
1134         :type spec: Specification
1135         """
1136
1137         # Specification:
1138         self._cfg = spec
1139
1140         # Data store:
1141         self._input_data = pd.Series()
1142
1143     @property
1144     def data(self):
1145         """Getter - Input data.
1146
1147         :returns: Input data
1148         :rtype: pandas.Series
1149         """
1150         return self._input_data
1151
1152     def metadata(self, job, build):
1153         """Getter - metadata
1154
1155         :param job: Job which metadata we want.
1156         :param build: Build which metadata we want.
1157         :type job: str
1158         :type build: str
1159         :returns: Metadata
1160         :rtype: pandas.Series
1161         """
1162
1163         return self.data[job][build]["metadata"]
1164
1165     def suites(self, job, build):
1166         """Getter - suites
1167
1168         :param job: Job which suites we want.
1169         :param build: Build which suites we want.
1170         :type job: str
1171         :type build: str
1172         :returns: Suites.
1173         :rtype: pandas.Series
1174         """
1175
1176         return self.data[job][str(build)]["suites"]
1177
1178     def tests(self, job, build):
1179         """Getter - tests
1180
1181         :param job: Job which tests we want.
1182         :param build: Build which tests we want.
1183         :type job: str
1184         :type build: str
1185         :returns: Tests.
1186         :rtype: pandas.Series
1187         """
1188
1189         return self.data[job][build]["tests"]
1190
1191     def _parse_tests(self, job, build, log):
1192         """Process data from robot output.xml file and return JSON structured
1193         data.
1194
1195         :param job: The name of job which build output data will be processed.
1196         :param build: The build which output data will be processed.
1197         :param log: List of log messages.
1198         :type job: str
1199         :type build: dict
1200         :type log: list of tuples (severity, msg)
1201         :returns: JSON data structure.
1202         :rtype: dict
1203         """
1204
1205         metadata = {
1206             "job": job,
1207             "build": build
1208         }
1209
1210         with open(build["file-name"], 'r') as data_file:
1211             try:
1212                 result = ExecutionResult(data_file)
1213             except errors.DataError as err:
1214                 log.append(("ERROR", "Error occurred while parsing output.xml: "
1215                                      "{0}".format(err)))
1216                 return None
1217         checker = ExecutionChecker(metadata, self._cfg.mapping,
1218                                    self._cfg.ignore)
1219         result.visit(checker)
1220
1221         return checker.data
1222
1223     def _download_and_parse_build(self, job, build, repeat, pid=10000):
1224         """Download and parse the input data file.
1225
1226         :param pid: PID of the process executing this method.
1227         :param job: Name of the Jenkins job which generated the processed input
1228             file.
1229         :param build: Information about the Jenkins build which generated the
1230             processed input file.
1231         :param repeat: Repeat the download specified number of times if not
1232             successful.
1233         :type pid: int
1234         :type job: str
1235         :type build: dict
1236         :type repeat: int
1237         """
1238
1239         logs = list()
1240
1241         logs.append(("INFO", "  Processing the job/build: {0}: {1}".
1242                      format(job, build["build"])))
1243
1244         state = "failed"
1245         success = False
1246         data = None
1247         do_repeat = repeat
1248         while do_repeat:
1249             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1250                                                    logs)
1251             if success:
1252                 break
1253             do_repeat -= 1
1254         if not success:
1255             logs.append(("ERROR", "It is not possible to download the input "
1256                                   "data file from the job '{job}', build "
1257                                   "'{build}', or it is damaged. Skipped.".
1258                          format(job=job, build=build["build"])))
1259         if success:
1260             logs.append(("INFO", "    Processing data from the build '{0}' ...".
1261                          format(build["build"])))
1262             data = self._parse_tests(job, build, logs)
1263             if data is None:
1264                 logs.append(("ERROR", "Input data file from the job '{job}', "
1265                                       "build '{build}' is damaged. Skipped.".
1266                              format(job=job, build=build["build"])))
1267             else:
1268                 state = "processed"
1269
1270             try:
1271                 remove(build["file-name"])
1272             except OSError as err:
1273                 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
1274                              format(build["file-name"], repr(err))))
1275
1276         # If the time-period is defined in the specification file, remove all
1277         # files which are outside the time period.
1278         timeperiod = self._cfg.input.get("time-period", None)
1279         if timeperiod and data:
1280             now = dt.utcnow()
1281             timeperiod = timedelta(int(timeperiod))
1282             metadata = data.get("metadata", None)
1283             if metadata:
1284                 generated = metadata.get("generated", None)
1285                 if generated:
1286                     generated = dt.strptime(generated, "%Y%m%d %H:%M")
1287                     if (now - generated) > timeperiod:
1288                         # Remove the data and the file:
1289                         state = "removed"
1290                         data = None
1291                         logs.append(
1292                             ("INFO",
1293                              "    The build {job}/{build} is outdated, will be "
1294                              "removed".format(job=job, build=build["build"])))
1295                         file_name = self._cfg.input["file-name"]
1296                         full_name = join(
1297                             self._cfg.environment["paths"]["DIR[WORKING,DATA]"],
1298                             "{job}{sep}{build}{sep}{name}".format(
1299                                 job=job,
1300                                 sep=SEPARATOR,
1301                                 build=build["build"],
1302                                 name=file_name))
1303                         try:
1304                             remove(full_name)
1305                             logs.append(("INFO",
1306                                          "    The file {name} has been removed".
1307                                          format(name=full_name)))
1308                         except OSError as err:
1309                             logs.append(("ERROR",
1310                                          "Cannot remove the file '{0}': {1}".
1311                                          format(full_name, repr(err))))
1312         logs.append(("INFO", "  Done."))
1313
1314         for level, line in logs:
1315             if level == "INFO":
1316                 logging.info(line)
1317             elif level == "ERROR":
1318                 logging.error(line)
1319             elif level == "DEBUG":
1320                 logging.debug(line)
1321             elif level == "CRITICAL":
1322                 logging.critical(line)
1323             elif level == "WARNING":
1324                 logging.warning(line)
1325
1326         return {"data": data, "state": state, "job": job, "build": build}
1327
1328     def download_and_parse_data(self, repeat=1):
1329         """Download the input data files, parse input data from input files and
1330         store in pandas' Series.
1331
1332         :param repeat: Repeat the download specified number of times if not
1333             successful.
1334         :type repeat: int
1335         """
1336
1337         logging.info("Downloading and parsing input files ...")
1338
1339         for job, builds in self._cfg.builds.items():
1340             for build in builds:
1341
1342                 result = self._download_and_parse_build(job, build, repeat)
1343                 build_nr = result["build"]["build"]
1344
1345                 if result["data"]:
1346                     data = result["data"]
1347                     build_data = pd.Series({
1348                         "metadata": pd.Series(
1349                             data["metadata"].values(),
1350                             index=data["metadata"].keys()),
1351                         "suites": pd.Series(data["suites"].values(),
1352                                             index=data["suites"].keys()),
1353                         "tests": pd.Series(data["tests"].values(),
1354                                            index=data["tests"].keys())})
1355
1356                     if self._input_data.get(job, None) is None:
1357                         self._input_data[job] = pd.Series()
1358                     self._input_data[job][str(build_nr)] = build_data
1359
1360                     self._cfg.set_input_file_name(
1361                         job, build_nr, result["build"]["file-name"])
1362
1363                 self._cfg.set_input_state(job, build_nr, result["state"])
1364
1365                 logging.info("Memory allocation: {0:,d}MB".format(
1366                     resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000))
1367
1368         logging.info("Done.")
1369
1370     @staticmethod
1371     def _end_of_tag(tag_filter, start=0, closer="'"):
1372         """Return the index of character in the string which is the end of tag.
1373
1374         :param tag_filter: The string where the end of tag is being searched.
1375         :param start: The index where the searching is stated.
1376         :param closer: The character which is the tag closer.
1377         :type tag_filter: str
1378         :type start: int
1379         :type closer: str
1380         :returns: The index of the tag closer.
1381         :rtype: int
1382         """
1383
1384         try:
1385             idx_opener = tag_filter.index(closer, start)
1386             return tag_filter.index(closer, idx_opener + 1)
1387         except ValueError:
1388             return None
1389
1390     @staticmethod
1391     def _condition(tag_filter):
1392         """Create a conditional statement from the given tag filter.
1393
1394         :param tag_filter: Filter based on tags from the element specification.
1395         :type tag_filter: str
1396         :returns: Conditional statement which can be evaluated.
1397         :rtype: str
1398         """
1399
1400         index = 0
1401         while True:
1402             index = InputData._end_of_tag(tag_filter, index)
1403             if index is None:
1404                 return tag_filter
1405             index += 1
1406             tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
1407
1408     def filter_data(self, element, params=None, data_set="tests",
1409                     continue_on_error=False):
1410         """Filter required data from the given jobs and builds.
1411
1412         The output data structure is:
1413
1414         - job 1
1415           - build 1
1416             - test (or suite) 1 ID:
1417               - param 1
1418               - param 2
1419               ...
1420               - param n
1421             ...
1422             - test (or suite) n ID:
1423             ...
1424           ...
1425           - build n
1426         ...
1427         - job n
1428
1429         :param element: Element which will use the filtered data.
1430         :param params: Parameters which will be included in the output. If None,
1431         all parameters are included.
1432         :param data_set: The set of data to be filtered: tests, suites,
1433         metadata.
1434         :param continue_on_error: Continue if there is error while reading the
1435         data. The Item will be empty then
1436         :type element: pandas.Series
1437         :type params: list
1438         :type data_set: str
1439         :type continue_on_error: bool
1440         :returns: Filtered data.
1441         :rtype pandas.Series
1442         """
1443
1444         try:
1445             if element["filter"] in ("all", "template"):
1446                 cond = "True"
1447             else:
1448                 cond = InputData._condition(element["filter"])
1449             logging.debug("   Filter: {0}".format(cond))
1450         except KeyError:
1451             logging.error("  No filter defined.")
1452             return None
1453
1454         if params is None:
1455             params = element.get("parameters", None)
1456             if params:
1457                 params.append("type")
1458
1459         data = pd.Series()
1460         try:
1461             for job, builds in element["data"].items():
1462                 data[job] = pd.Series()
1463                 for build in builds:
1464                     data[job][str(build)] = pd.Series()
1465                     try:
1466                         data_iter = self.data[job][str(build)][data_set].\
1467                             iteritems()
1468                     except KeyError:
1469                         if continue_on_error:
1470                             continue
1471                         else:
1472                             return None
1473                     for test_ID, test_data in data_iter:
1474                         if eval(cond, {"tags": test_data.get("tags", "")}):
1475                             data[job][str(build)][test_ID] = pd.Series()
1476                             if params is None:
1477                                 for param, val in test_data.items():
1478                                     data[job][str(build)][test_ID][param] = val
1479                             else:
1480                                 for param in params:
1481                                     try:
1482                                         data[job][str(build)][test_ID][param] =\
1483                                             test_data[param]
1484                                     except KeyError:
1485                                         data[job][str(build)][test_ID][param] =\
1486                                             "No Data"
1487             return data
1488
1489         except (KeyError, IndexError, ValueError) as err:
1490             logging.error("   Missing mandatory parameter in the element "
1491                           "specification: {0}".format(err))
1492             return None
1493         except AttributeError:
1494             return None
1495         except SyntaxError:
1496             logging.error("   The filter '{0}' is not correct. Check if all "
1497                           "tags are enclosed by apostrophes.".format(cond))
1498             return None
1499
1500     def filter_tests_by_name(self, element, params=None, data_set="tests",
1501                              continue_on_error=False):
1502         """Filter required data from the given jobs and builds.
1503
1504         The output data structure is:
1505
1506         - job 1
1507           - build 1
1508             - test (or suite) 1 ID:
1509               - param 1
1510               - param 2
1511               ...
1512               - param n
1513             ...
1514             - test (or suite) n ID:
1515             ...
1516           ...
1517           - build n
1518         ...
1519         - job n
1520
1521         :param element: Element which will use the filtered data.
1522         :param params: Parameters which will be included in the output. If None,
1523         all parameters are included.
1524         :param data_set: The set of data to be filtered: tests, suites,
1525         metadata.
1526         :param continue_on_error: Continue if there is error while reading the
1527         data. The Item will be empty then
1528         :type element: pandas.Series
1529         :type params: list
1530         :type data_set: str
1531         :type continue_on_error: bool
1532         :returns: Filtered data.
1533         :rtype pandas.Series
1534         """
1535
1536         include = element.get("include", None)
1537         if not include:
1538             logging.warning("No tests to include, skipping the element.")
1539             return None
1540
1541         if params is None:
1542             params = element.get("parameters", None)
1543             if params:
1544                 params.append("type")
1545
1546         data = pd.Series()
1547         try:
1548             for job, builds in element["data"].items():
1549                 data[job] = pd.Series()
1550                 for build in builds:
1551                     data[job][str(build)] = pd.Series()
1552                     for test in include:
1553                         try:
1554                             reg_ex = re.compile(str(test).lower())
1555                             for test_ID in self.data[job][str(build)]\
1556                                     [data_set].keys():
1557                                 if re.match(reg_ex, str(test_ID).lower()):
1558                                     test_data = self.data[job][str(build)]\
1559                                         [data_set][test_ID]
1560                                     data[job][str(build)][test_ID] = pd.Series()
1561                                     if params is None:
1562                                         for param, val in test_data.items():
1563                                             data[job][str(build)][test_ID]\
1564                                                 [param] = val
1565                                     else:
1566                                         for param in params:
1567                                             try:
1568                                                 data[job][str(build)][test_ID]\
1569                                                     [param] = test_data[param]
1570                                             except KeyError:
1571                                                 data[job][str(build)][test_ID]\
1572                                                     [param] = "No Data"
1573                         except KeyError as err:
1574                             logging.error("{err!r}".format(err=err))
1575                             if continue_on_error:
1576                                 continue
1577                             else:
1578                                 return None
1579             return data
1580
1581         except (KeyError, IndexError, ValueError) as err:
1582             logging.error("Missing mandatory parameter in the element "
1583                           "specification: {err!r}".format(err=err))
1584             return None
1585         except AttributeError as err:
1586             logging.error("{err!r}".format(err=err))
1587             return None
1588
1589
1590     @staticmethod
1591     def merge_data(data):
1592         """Merge data from more jobs and builds to a simple data structure.
1593
1594         The output data structure is:
1595
1596         - test (suite) 1 ID:
1597           - param 1
1598           - param 2
1599           ...
1600           - param n
1601         ...
1602         - test (suite) n ID:
1603         ...
1604
1605         :param data: Data to merge.
1606         :type data: pandas.Series
1607         :returns: Merged data.
1608         :rtype: pandas.Series
1609         """
1610
1611         logging.info("    Merging data ...")
1612
1613         merged_data = pd.Series()
1614         for _, builds in data.iteritems():
1615             for _, item in builds.iteritems():
1616                 for ID, item_data in item.iteritems():
1617                     merged_data[ID] = item_data
1618
1619         return merged_data