resources/tools/presentation_new/input_data_parser.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Data pre-processing
  15
  16 - extract data from output.xml files generated by Jenkins jobs and store in
  17   pandas' Series,
  18 - provide access to the data.
  19 - filter the data using tags,
  20 """
  21
  22 import multiprocessing
  23 import os
  24 import re
  25 import pandas as pd
  26 import logging
  27
  28 from robot.api import ExecutionResult, ResultVisitor
  29 from robot import errors
  30 from collections import OrderedDict
  31 from string import replace
  32 from os import remove
  33 from os.path import join
  34 from datetime import datetime as dt
  35 from datetime import timedelta
  36 from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory
  37
  38 from input_data_files import download_and_unzip_data_file
  39 from utils import Worker
  40
  41
  42 # Separator used in file names
  43 SEPARATOR = "__"
  44
  45
  46 class ExecutionChecker(ResultVisitor):
  47     """Class to traverse through the test suite structure.
  48
  49     The functionality implemented in this class generates a json structure:
  50
  51     Performance tests:
  52
  53     {
  54         "metadata": {
  55             "generated": "Timestamp",
  56             "version": "SUT version",
  57             "job": "Jenkins job name",
  58             "build": "Information about the build"
  59         },
  60         "suites": {
  61             "Suite long name 1": {
  62                 "name": Suite name,
  63                 "doc": "Suite 1 documentation",
  64                 "parent": "Suite 1 parent",
  65                 "level": "Level of the suite in the suite hierarchy"
  66             }
  67             "Suite long name N": {
  68                 "name": Suite name,
  69                 "doc": "Suite N documentation",
  70                 "parent": "Suite 2 parent",
  71                 "level": "Level of the suite in the suite hierarchy"
  72             }
  73         }
  74         "tests": {
  75             # NDRPDR tests:
  76             "ID": {
  77                 "name": "Test name",
  78                 "parent": "Name of the parent of the test",
  79                 "doc": "Test documentation",
  80                 "msg": "Test message",
  81                 "vat-history": "DUT1 and DUT2 VAT History",
  82                 "show-run": "Show Run",
  83                 "tags": ["tag 1", "tag 2", "tag n"],
  84                 "type": "NDRPDR",
  85                 "status": "PASS" | "FAIL",
  86                 "throughput": {
  87                     "NDR": {
  88                         "LOWER": float,
  89                         "UPPER": float
  90                     },
  91                     "PDR": {
  92                         "LOWER": float,
  93                         "UPPER": float
  94                     }
  95                 },
  96                 "latency": {
  97                     "NDR": {
  98                         "direction1": {
  99                             "min": float,
 100                             "avg": float,
 101                             "max": float
 102                         },
 103                         "direction2": {
 104                             "min": float,
 105                             "avg": float,
 106                             "max": float
 107                         }
 108                     },
 109                     "PDR": {
 110                         "direction1": {
 111                             "min": float,
 112                             "avg": float,
 113                             "max": float
 114                         },
 115                         "direction2": {
 116                             "min": float,
 117                             "avg": float,
 118                             "max": float
 119                         }
 120                     }
 121                 }
 122             }
 123
 124             # TCP tests:
 125             "ID": {
 126                 "name": "Test name",
 127                 "parent": "Name of the parent of the test",
 128                 "doc": "Test documentation",
 129                 "msg": "Test message",
 130                 "tags": ["tag 1", "tag 2", "tag n"],
 131                 "type": "TCP",
 132                 "status": "PASS" | "FAIL",
 133                 "result": int
 134             }
 135
 136             # MRR, BMRR tests:
 137             "ID": {
 138                 "name": "Test name",
 139                 "parent": "Name of the parent of the test",
 140                 "doc": "Test documentation",
 141                 "msg": "Test message",
 142                 "tags": ["tag 1", "tag 2", "tag n"],
 143                 "type": "MRR" | "BMRR",
 144                 "status": "PASS" | "FAIL",
 145                 "result": {
 146                     "receive-rate": AvgStdevMetadata,
 147                 }
 148             }
 149
 150             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 151             # NDRPDRDISC tests:
 152             "ID": {
 153                 "name": "Test name",
 154                 "parent": "Name of the parent of the test",
 155                 "doc": "Test documentation",
 156                 "msg": "Test message",
 157                 "tags": ["tag 1", "tag 2", "tag n"],
 158                 "type": "PDR" | "NDR",
 159                 "status": "PASS" | "FAIL",
 160                 "throughput": {  # Only type: "PDR" | "NDR"
 161                     "value": int,
 162                     "unit": "pps" | "bps" | "percentage"
 163                 },
 164                 "latency": {  # Only type: "PDR" | "NDR"
 165                     "direction1": {
 166                         "100": {
 167                             "min": int,
 168                             "avg": int,
 169                             "max": int
 170                         },
 171                         "50": {  # Only for NDR
 172                             "min": int,
 173                             "avg": int,
 174                             "max": int
 175                         },
 176                         "10": {  # Only for NDR
 177                             "min": int,
 178                             "avg": int,
 179                             "max": int
 180                         }
 181                     },
 182                     "direction2": {
 183                         "100": {
 184                             "min": int,
 185                             "avg": int,
 186                             "max": int
 187                         },
 188                         "50": {  # Only for NDR
 189                             "min": int,
 190                             "avg": int,
 191                             "max": int
 192                         },
 193                         "10": {  # Only for NDR
 194                             "min": int,
 195                             "avg": int,
 196                             "max": int
 197                         }
 198                     }
 199                 },
 200                 "lossTolerance": "lossTolerance",  # Only type: "PDR"
 201                 "vat-history": "DUT1 and DUT2 VAT History"
 202                 "show-run": "Show Run"
 203             },
 204             "ID" {
 205                 # next test
 206             }
 207         }
 208     }
 209
 210
 211     Functional tests:
 212
 213     {
 214         "metadata": {  # Optional
 215             "version": "VPP version",
 216             "job": "Jenkins job name",
 217             "build": "Information about the build"
 218         },
 219         "suites": {
 220             "Suite name 1": {
 221                 "doc": "Suite 1 documentation",
 222                 "parent": "Suite 1 parent",
 223                 "level": "Level of the suite in the suite hierarchy"
 224             }
 225             "Suite name N": {
 226                 "doc": "Suite N documentation",
 227                 "parent": "Suite 2 parent",
 228                 "level": "Level of the suite in the suite hierarchy"
 229             }
 230         }
 231         "tests": {
 232             "ID": {
 233                 "name": "Test name",
 234                 "parent": "Name of the parent of the test",
 235                 "doc": "Test documentation"
 236                 "msg": "Test message"
 237                 "tags": ["tag 1", "tag 2", "tag n"],
 238                 "vat-history": "DUT1 and DUT2 VAT History"
 239                 "show-run": "Show Run"
 240                 "status": "PASS" | "FAIL"
 241             },
 242             "ID" {
 243                 # next test
 244             }
 245         }
 246     }
 247
 248     .. note:: ID is the lowercase full path to the test.
 249     """
 250
 251     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 252     REGEX_RATE = re.compile(r'^[\D\d]*FINAL_RATE:\s(\d+\.\d+)\s(\w+)')
 253
 254     REGEX_NDRPDR_RATE = re.compile(r'NDR_LOWER:\s(\d+.\d+).*\n.*\n'
 255                                    r'NDR_UPPER:\s(\d+.\d+).*\n'
 256                                    r'PDR_LOWER:\s(\d+.\d+).*\n.*\n'
 257                                    r'PDR_UPPER:\s(\d+.\d+)')
 258
 259     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 260     REGEX_LAT_NDR = re.compile(r'^[\D\d]*'
 261                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 262                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
 263                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 264                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]\s\n'
 265                                r'LAT_\d+%NDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 266                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\]')
 267
 268     REGEX_LAT_PDR = re.compile(r'^[\D\d]*'
 269                                r'LAT_\d+%PDR:\s\[\'(-?\d+/-?\d+/-?\d+)\','
 270                                r'\s\'(-?\d+/-?\d+/-?\d+)\'\][\D\d]*')
 271
 272     REGEX_NDRPDR_LAT = re.compile(r'LATENCY.*\[\'(.*)\', \'(.*)\'\]\s\n.*\n.*\n'
 273                                   r'LATENCY.*\[\'(.*)\', \'(.*)\'\]')
 274
 275     REGEX_TOLERANCE = re.compile(r'^[\D\d]*LOSS_ACCEPTANCE:\s(\d*\.\d*)\s'
 276                                  r'[\D\d]*')
 277
 278     REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*)(.*)")
 279
 280     REGEX_VERSION_DPDK = re.compile(r"(return STDOUT testpmd)([\d\D\n]*)"
 281                                     r"(RTE Version: 'DPDK )(.*)(')")
 282
 283     REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$')
 284
 285     REGEX_MRR = re.compile(r'MaxReceivedRate_Results\s\[pkts/(\d*)sec\]:\s'
 286                            r'tx\s(\d*),\srx\s(\d*)')
 287
 288     REGEX_BMRR = re.compile(r'Maximum Receive Rate trial results'
 289                             r' in packets per second: \[(.*)\]')
 290
 291     REGEX_TC_TAG = re.compile(r'\d+[tT]\d+[cC]')
 292
 293     REGEX_TC_NAME_OLD = re.compile(r'-\d+[tT]\d+[cC]-')
 294
 295     REGEX_TC_NAME_NEW = re.compile(r'-\d+[cC]-')
 296
 297     REGEX_TC_NUMBER = re.compile(r'tc[0-9]{2}-')
 298
 299     def __init__(self, metadata, mapping, ignore):
 300         """Initialisation.
 301
 302         :param metadata: Key-value pairs to be included in "metadata" part of
 303             JSON structure.
 304         :param mapping: Mapping of the old names of test cases to the new
 305             (actual) one.
 306         :param ignore: List of TCs to be ignored.
 307         :type metadata: dict
 308         :type mapping: dict
 309         :type ignore: list
 310         """
 311
 312         # Type of message to parse out from the test messages
 313         self._msg_type = None
 314
 315         # VPP version
 316         self._version = None
 317
 318         # Timestamp
 319         self._timestamp = None
 320
 321         # Mapping of TCs long names
 322         self._mapping = mapping
 323
 324         # Ignore list
 325         self._ignore = ignore
 326
 327         # Number of VAT History messages found:
 328         # 0 - no message
 329         # 1 - VAT History of DUT1
 330         # 2 - VAT History of DUT2
 331         self._lookup_kw_nr = 0
 332         self._vat_history_lookup_nr = 0
 333
 334         # Number of Show Running messages found
 335         # 0 - no message
 336         # 1 - Show run message found
 337         self._show_run_lookup_nr = 0
 338
 339         # Test ID of currently processed test- the lowercase full path to the
 340         # test
 341         self._test_ID = None
 342
 343         # The main data structure
 344         self._data = {
 345             "metadata": OrderedDict(),
 346             "suites": OrderedDict(),
 347             "tests": OrderedDict()
 348         }
 349
 350         # Save the provided metadata
 351         for key, val in metadata.items():
 352             self._data["metadata"][key] = val
 353
 354         # Dictionary defining the methods used to parse different types of
 355         # messages
 356         self.parse_msg = {
 357             "timestamp": self._get_timestamp,
 358             "vpp-version": self._get_vpp_version,
 359             "dpdk-version": self._get_dpdk_version,
 360             "teardown-vat-history": self._get_vat_history,
 361             "test-show-runtime": self._get_show_run
 362         }
 363
 364     @property
 365     def data(self):
 366         """Getter - Data parsed from the XML file.
 367
 368         :returns: Data parsed from the XML file.
 369         :rtype: dict
 370         """
 371         return self._data
 372
 373     def _get_vpp_version(self, msg):
 374         """Called when extraction of VPP version is required.
 375
 376         :param msg: Message to process.
 377         :type msg: Message
 378         :returns: Nothing.
 379         """
 380
 381         if msg.message.count("return STDOUT Version:"):
 382             self._version = str(re.search(self.REGEX_VERSION_VPP, msg.message).
 383                                 group(2))
 384             self._data["metadata"]["version"] = self._version
 385             self._msg_type = None
 386
 387     def _get_dpdk_version(self, msg):
 388         """Called when extraction of DPDK version is required.
 389
 390         :param msg: Message to process.
 391         :type msg: Message
 392         :returns: Nothing.
 393         """
 394
 395         if msg.message.count("return STDOUT testpmd"):
 396             try:
 397                 self._version = str(re.search(
 398                     self.REGEX_VERSION_DPDK, msg.message). group(4))
 399                 self._data["metadata"]["version"] = self._version
 400             except IndexError:
 401                 pass
 402             finally:
 403                 self._msg_type = None
 404
 405     def _get_timestamp(self, msg):
 406         """Called when extraction of timestamp is required.
 407
 408         :param msg: Message to process.
 409         :type msg: Message
 410         :returns: Nothing.
 411         """
 412
 413         self._timestamp = msg.timestamp[:14]
 414         self._data["metadata"]["generated"] = self._timestamp
 415         self._msg_type = None
 416
 417     def _get_vat_history(self, msg):
 418         """Called when extraction of VAT command history is required.
 419
 420         :param msg: Message to process.
 421         :type msg: Message
 422         :returns: Nothing.
 423         """
 424         if msg.message.count("VAT command history:"):
 425             self._vat_history_lookup_nr += 1
 426             if self._vat_history_lookup_nr == 1:
 427                 self._data["tests"][self._test_ID]["vat-history"] = str()
 428             else:
 429                 self._msg_type = None
 430             text = re.sub("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3} "
 431                           "VAT command history:", "", msg.message, count=1). \
 432                 replace("\n\n", "\n").replace('\n', ' |br| ').\
 433                 replace('\r', '').replace('"', "'")
 434
 435             self._data["tests"][self._test_ID]["vat-history"] += " |br| "
 436             self._data["tests"][self._test_ID]["vat-history"] += \
 437                 "**DUT" + str(self._vat_history_lookup_nr) + ":** " + text
 438
 439     def _get_show_run(self, msg):
 440         """Called when extraction of VPP operational data (output of CLI command
 441         Show Runtime) is required.
 442
 443         :param msg: Message to process.
 444         :type msg: Message
 445         :returns: Nothing.
 446         """
 447         if msg.message.count("return STDOUT Thread "):
 448             self._show_run_lookup_nr += 1
 449             if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1:
 450                 self._data["tests"][self._test_ID]["show-run"] = str()
 451             if self._lookup_kw_nr > 1:
 452                 self._msg_type = None
 453             if self._show_run_lookup_nr == 1:
 454                 text = msg.message.replace("vat# ", "").\
 455                     replace("return STDOUT ", "").replace("\n\n", "\n").\
 456                     replace('\n', ' |br| ').\
 457                     replace('\r', '').replace('"', "'")
 458                 try:
 459                     self._data["tests"][self._test_ID]["show-run"] += " |br| "
 460                     self._data["tests"][self._test_ID]["show-run"] += \
 461                         "**DUT" + str(self._lookup_kw_nr) + ":** |br| " + text
 462                 except KeyError:
 463                     pass
 464
 465     # TODO: Remove when definitely no NDRPDRDISC tests are used:
 466     def _get_latency(self, msg, test_type):
 467         """Get the latency data from the test message.
 468
 469         :param msg: Message to be parsed.
 470         :param test_type: Type of the test - NDR or PDR.
 471         :type msg: str
 472         :type test_type: str
 473         :returns: Latencies parsed from the message.
 474         :rtype: dict
 475         """
 476
 477         if test_type == "NDR":
 478             groups = re.search(self.REGEX_LAT_NDR, msg)
 479             groups_range = range(1, 7)
 480         elif test_type == "PDR":
 481             groups = re.search(self.REGEX_LAT_PDR, msg)
 482             groups_range = range(1, 3)
 483         else:
 484             return {}
 485
 486         latencies = list()
 487         for idx in groups_range:
 488             try:
 489                 lat = [int(item) for item in str(groups.group(idx)).split('/')]
 490             except (AttributeError, ValueError):
 491                 lat = [-1, -1, -1]
 492             latencies.append(lat)
 493
 494         keys = ("min", "avg", "max")
 495         latency = {
 496             "direction1": {
 497             },
 498             "direction2": {
 499             }
 500         }
 501
 502         latency["direction1"]["100"] = dict(zip(keys, latencies[0]))
 503         latency["direction2"]["100"] = dict(zip(keys, latencies[1]))
 504         if test_type == "NDR":
 505             latency["direction1"]["50"] = dict(zip(keys, latencies[2]))
 506             latency["direction2"]["50"] = dict(zip(keys, latencies[3]))
 507             latency["direction1"]["10"] = dict(zip(keys, latencies[4]))
 508             latency["direction2"]["10"] = dict(zip(keys, latencies[5]))
 509
 510         return latency
 511
 512     def _get_ndrpdr_throughput(self, msg):
 513         """Get NDR_LOWER, NDR_UPPER, PDR_LOWER and PDR_UPPER from the test
 514         message.
 515
 516         :param msg: The test message to be parsed.
 517         :type msg: str
 518         :returns: Parsed data as a dict and the status (PASS/FAIL).
 519         :rtype: tuple(dict, str)
 520         """
 521
 522         throughput = {
 523             "NDR": {"LOWER": -1.0, "UPPER": -1.0},
 524             "PDR": {"LOWER": -1.0, "UPPER": -1.0}
 525         }
 526         status = "FAIL"
 527         groups = re.search(self.REGEX_NDRPDR_RATE, msg)
 528
 529         if groups is not None:
 530             try:
 531                 throughput["NDR"]["LOWER"] = float(groups.group(1))
 532                 throughput["NDR"]["UPPER"] = float(groups.group(2))
 533                 throughput["PDR"]["LOWER"] = float(groups.group(3))
 534                 throughput["PDR"]["UPPER"] = float(groups.group(4))
 535                 status = "PASS"
 536             except (IndexError, ValueError):
 537                 pass
 538
 539         return throughput, status
 540
 541     def _get_ndrpdr_latency(self, msg):
 542         """Get LATENCY from the test message.
 543
 544         :param msg: The test message to be parsed.
 545         :type msg: str
 546         :returns: Parsed data as a dict and the status (PASS/FAIL).
 547         :rtype: tuple(dict, str)
 548         """
 549
 550         latency = {
 551             "NDR": {
 552                 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
 553                 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
 554             },
 555             "PDR": {
 556                 "direction1": {"min": -1.0, "avg": -1.0, "max": -1.0},
 557                 "direction2": {"min": -1.0, "avg": -1.0, "max": -1.0}
 558             }
 559         }
 560         status = "FAIL"
 561         groups = re.search(self.REGEX_NDRPDR_LAT, msg)
 562
 563         if groups is not None:
 564             keys = ("min", "avg", "max")
 565             try:
 566                 latency["NDR"]["direction1"] = dict(
 567                     zip(keys, [float(l) for l in groups.group(1).split('/')]))
 568                 latency["NDR"]["direction2"] = dict(
 569                     zip(keys, [float(l) for l in groups.group(2).split('/')]))
 570                 latency["PDR"]["direction1"] = dict(
 571                     zip(keys, [float(l) for l in groups.group(3).split('/')]))
 572                 latency["PDR"]["direction2"] = dict(
 573                     zip(keys, [float(l) for l in groups.group(4).split('/')]))
 574                 status = "PASS"
 575             except (IndexError, ValueError):
 576                 pass
 577
 578         return latency, status
 579
 580     def visit_suite(self, suite):
 581         """Implements traversing through the suite and its direct children.
 582
 583         :param suite: Suite to process.
 584         :type suite: Suite
 585         :returns: Nothing.
 586         """
 587         if self.start_suite(suite) is not False:
 588             suite.suites.visit(self)
 589             suite.tests.visit(self)
 590             self.end_suite(suite)
 591
 592     def start_suite(self, suite):
 593         """Called when suite starts.
 594
 595         :param suite: Suite to process.
 596         :type suite: Suite
 597         :returns: Nothing.
 598         """
 599
 600         try:
 601             parent_name = suite.parent.name
 602         except AttributeError:
 603             return
 604
 605         doc_str = suite.doc.replace('"', "'").replace('\n', ' ').\
 606             replace('\r', '').replace('*[', ' |br| *[').replace("*", "**")
 607         doc_str = replace(doc_str, ' |br| *[', '*[', maxreplace=1)
 608
 609         self._data["suites"][suite.longname.lower().replace('"', "'").
 610             replace(" ", "_")] = {
 611                 "name": suite.name.lower(),
 612                 "doc": doc_str,
 613                 "parent": parent_name,
 614                 "level": len(suite.longname.split("."))
 615             }
 616
 617         suite.keywords.visit(self)
 618
 619     def end_suite(self, suite):
 620         """Called when suite ends.
 621
 622         :param suite: Suite to process.
 623         :type suite: Suite
 624         :returns: Nothing.
 625         """
 626         pass
 627
 628     def visit_test(self, test):
 629         """Implements traversing through the test.
 630
 631         :param test: Test to process.
 632         :type test: Test
 633         :returns: Nothing.
 634         """
 635         if self.start_test(test) is not False:
 636             test.keywords.visit(self)
 637             self.end_test(test)
 638
 639     def start_test(self, test):
 640         """Called when test starts.
 641
 642         :param test: Test to process.
 643         :type test: Test
 644         :returns: Nothing.
 645         """
 646
 647         longname_orig = test.longname.lower()
 648
 649         # Check the ignore list
 650         if longname_orig in self._ignore:
 651             return
 652
 653         tags = [str(tag) for tag in test.tags]
 654         test_result = dict()
 655
 656         # Change the TC long name and name if defined in the mapping table
 657         longname = self._mapping.get(longname_orig, None)
 658         if longname is not None:
 659             name = longname.split('.')[-1]
 660             logging.debug("{0}\n{1}\n{2}\n{3}".format(
 661                 self._data["metadata"], longname_orig, longname, name))
 662         else:
 663             longname = longname_orig
 664             name = test.name.lower()
 665
 666         # Remove TC number from the TC long name (backward compatibility):
 667         self._test_ID = re.sub(self.REGEX_TC_NUMBER, "", longname)
 668         # Remove TC number from the TC name (not needed):
 669         test_result["name"] = re.sub(self.REGEX_TC_NUMBER, "", name)
 670
 671         test_result["parent"] = test.parent.name.lower()
 672         test_result["tags"] = tags
 673         doc_str = test.doc.replace('"', "'").replace('\n', ' '). \
 674             replace('\r', '').replace('[', ' |br| [')
 675         test_result["doc"] = replace(doc_str, ' |br| [', '[', maxreplace=1)
 676         test_result["msg"] = test.message.replace('\n', ' |br| '). \
 677             replace('\r', '').replace('"', "'")
 678         test_result["type"] = "FUNC"
 679         test_result["status"] = test.status
 680
 681         if "PERFTEST" in tags:
 682             # Replace info about cores (e.g. -1c-) with the info about threads
 683             # and cores (e.g. -1t1c-) in the long test case names and in the
 684             # test case names if necessary.
 685             groups = re.search(self.REGEX_TC_NAME_OLD, self._test_ID)
 686             if not groups:
 687                 tag_count = 0
 688                 for tag in test_result["tags"]:
 689                     groups = re.search(self.REGEX_TC_TAG, tag)
 690                     if groups:
 691                         tag_count += 1
 692                         tag_tc = tag
 693
 694                 if tag_count == 1:
 695                     self._test_ID = re.sub(self.REGEX_TC_NAME_NEW,
 696                                            "-{0}-".format(tag_tc.lower()),
 697                                            self._test_ID,
 698                                            count=1)
 699                     test_result["name"] = re.sub(self.REGEX_TC_NAME_NEW,
 700                                                  "-{0}-".format(tag_tc.lower()),
 701                                                  test_result["name"],
 702                                                  count=1)
 703                 else:
 704                     test_result["status"] = "FAIL"
 705                     self._data["tests"][self._test_ID] = test_result
 706                     logging.debug("The test '{0}' has no or more than one "
 707                                   "multi-threading tags.".format(self._test_ID))
 708                     logging.debug("Tags: {0}".format(test_result["tags"]))
 709                     return
 710
 711         if test.status == "PASS" and ("NDRPDRDISC" in tags or
 712                                       "NDRPDR" in tags or
 713                                       "TCP" in tags or
 714                                       "MRR" in tags or
 715                                       "BMRR" in tags):
 716             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 717             if "NDRDISC" in tags:
 718                 test_result["type"] = "NDR"
 719             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 720             elif "PDRDISC" in tags:
 721                 test_result["type"] = "PDR"
 722             elif "NDRPDR" in tags:
 723                 test_result["type"] = "NDRPDR"
 724             elif "TCP" in tags:
 725                 test_result["type"] = "TCP"
 726             elif "MRR" in tags:
 727                 test_result["type"] = "MRR"
 728             elif "FRMOBL" in tags or "BMRR" in tags:
 729                 test_result["type"] = "BMRR"
 730             else:
 731                 test_result["status"] = "FAIL"
 732                 self._data["tests"][self._test_ID] = test_result
 733                 return
 734
 735             # TODO: Remove when definitely no NDRPDRDISC tests are used:
 736             if test_result["type"] in ("NDR", "PDR"):
 737                 try:
 738                     rate_value = str(re.search(
 739                         self.REGEX_RATE, test.message).group(1))
 740                 except AttributeError:
 741                     rate_value = "-1"
 742                 try:
 743                     rate_unit = str(re.search(
 744                         self.REGEX_RATE, test.message).group(2))
 745                 except AttributeError:
 746                     rate_unit = "-1"
 747
 748                 test_result["throughput"] = dict()
 749                 test_result["throughput"]["value"] = \
 750                     int(rate_value.split('.')[0])
 751                 test_result["throughput"]["unit"] = rate_unit
 752                 test_result["latency"] = \
 753                     self._get_latency(test.message, test_result["type"])
 754                 if test_result["type"] == "PDR":
 755                     test_result["lossTolerance"] = str(re.search(
 756                         self.REGEX_TOLERANCE, test.message).group(1))
 757
 758             elif test_result["type"] in ("NDRPDR", ):
 759                 test_result["throughput"], test_result["status"] = \
 760                     self._get_ndrpdr_throughput(test.message)
 761                 test_result["latency"], test_result["status"] = \
 762                     self._get_ndrpdr_latency(test.message)
 763
 764             elif test_result["type"] in ("TCP", ):
 765                 groups = re.search(self.REGEX_TCP, test.message)
 766                 test_result["result"] = int(groups.group(2))
 767
 768             elif test_result["type"] in ("MRR", "BMRR"):
 769                 test_result["result"] = dict()
 770                 groups = re.search(self.REGEX_BMRR, test.message)
 771                 if groups is not None:
 772                     items_str = groups.group(1)
 773                     items_float = [float(item.strip()) for item
 774                                    in items_str.split(",")]
 775                     metadata = AvgStdevMetadataFactory.from_data(items_float)
 776                     test_result["result"]["receive-rate"] = metadata
 777                 else:
 778                     groups = re.search(self.REGEX_MRR, test.message)
 779                     test_result["result"]["receive-rate"] = \
 780                         AvgStdevMetadataFactory.from_data([
 781                             float(groups.group(3)) / float(groups.group(1)), ])
 782
 783         self._data["tests"][self._test_ID] = test_result
 784
 785     def end_test(self, test):
 786         """Called when test ends.
 787
 788         :param test: Test to process.
 789         :type test: Test
 790         :returns: Nothing.
 791         """
 792         pass
 793
 794     def visit_keyword(self, keyword):
 795         """Implements traversing through the keyword and its child keywords.
 796
 797         :param keyword: Keyword to process.
 798         :type keyword: Keyword
 799         :returns: Nothing.
 800         """
 801         if self.start_keyword(keyword) is not False:
 802             self.end_keyword(keyword)
 803
 804     def start_keyword(self, keyword):
 805         """Called when keyword starts. Default implementation does nothing.
 806
 807         :param keyword: Keyword to process.
 808         :type keyword: Keyword
 809         :returns: Nothing.
 810         """
 811         try:
 812             if keyword.type == "setup":
 813                 self.visit_setup_kw(keyword)
 814             elif keyword.type == "teardown":
 815                 self._lookup_kw_nr = 0
 816                 self.visit_teardown_kw(keyword)
 817             else:
 818                 self._lookup_kw_nr = 0
 819                 self.visit_test_kw(keyword)
 820         except AttributeError:
 821             pass
 822
 823     def end_keyword(self, keyword):
 824         """Called when keyword ends. Default implementation does nothing.
 825
 826         :param keyword: Keyword to process.
 827         :type keyword: Keyword
 828         :returns: Nothing.
 829         """
 830         pass
 831
 832     def visit_test_kw(self, test_kw):
 833         """Implements traversing through the test keyword and its child
 834         keywords.
 835
 836         :param test_kw: Keyword to process.
 837         :type test_kw: Keyword
 838         :returns: Nothing.
 839         """
 840         for keyword in test_kw.keywords:
 841             if self.start_test_kw(keyword) is not False:
 842                 self.visit_test_kw(keyword)
 843                 self.end_test_kw(keyword)
 844
 845     def start_test_kw(self, test_kw):
 846         """Called when test keyword starts. Default implementation does
 847         nothing.
 848
 849         :param test_kw: Keyword to process.
 850         :type test_kw: Keyword
 851         :returns: Nothing.
 852         """
 853         if test_kw.name.count("Show Runtime Counters On All Duts"):
 854             self._lookup_kw_nr += 1
 855             self._show_run_lookup_nr = 0
 856             self._msg_type = "test-show-runtime"
 857         elif test_kw.name.count("Start The L2fwd Test") and not self._version:
 858             self._msg_type = "dpdk-version"
 859         else:
 860             return
 861         test_kw.messages.visit(self)
 862
 863     def end_test_kw(self, test_kw):
 864         """Called when keyword ends. Default implementation does nothing.
 865
 866         :param test_kw: Keyword to process.
 867         :type test_kw: Keyword
 868         :returns: Nothing.
 869         """
 870         pass
 871
 872     def visit_setup_kw(self, setup_kw):
 873         """Implements traversing through the teardown keyword and its child
 874         keywords.
 875
 876         :param setup_kw: Keyword to process.
 877         :type setup_kw: Keyword
 878         :returns: Nothing.
 879         """
 880         for keyword in setup_kw.keywords:
 881             if self.start_setup_kw(keyword) is not False:
 882                 self.visit_setup_kw(keyword)
 883                 self.end_setup_kw(keyword)
 884
 885     def start_setup_kw(self, setup_kw):
 886         """Called when teardown keyword starts. Default implementation does
 887         nothing.
 888
 889         :param setup_kw: Keyword to process.
 890         :type setup_kw: Keyword
 891         :returns: Nothing.
 892         """
 893         if setup_kw.name.count("Show Vpp Version On All Duts") \
 894                 and not self._version:
 895             self._msg_type = "vpp-version"
 896
 897         elif setup_kw.name.count("Setup performance global Variables") \
 898                 and not self._timestamp:
 899             self._msg_type = "timestamp"
 900         else:
 901             return
 902         setup_kw.messages.visit(self)
 903
 904     def end_setup_kw(self, setup_kw):
 905         """Called when keyword ends. Default implementation does nothing.
 906
 907         :param setup_kw: Keyword to process.
 908         :type setup_kw: Keyword
 909         :returns: Nothing.
 910         """
 911         pass
 912
 913     def visit_teardown_kw(self, teardown_kw):
 914         """Implements traversing through the teardown keyword and its child
 915         keywords.
 916
 917         :param teardown_kw: Keyword to process.
 918         :type teardown_kw: Keyword
 919         :returns: Nothing.
 920         """
 921         for keyword in teardown_kw.keywords:
 922             if self.start_teardown_kw(keyword) is not False:
 923                 self.visit_teardown_kw(keyword)
 924                 self.end_teardown_kw(keyword)
 925
 926     def start_teardown_kw(self, teardown_kw):
 927         """Called when teardown keyword starts. Default implementation does
 928         nothing.
 929
 930         :param teardown_kw: Keyword to process.
 931         :type teardown_kw: Keyword
 932         :returns: Nothing.
 933         """
 934
 935         if teardown_kw.name.count("Show Vat History On All Duts"):
 936             self._vat_history_lookup_nr = 0
 937             self._msg_type = "teardown-vat-history"
 938             teardown_kw.messages.visit(self)
 939
 940     def end_teardown_kw(self, teardown_kw):
 941         """Called when keyword ends. Default implementation does nothing.
 942
 943         :param teardown_kw: Keyword to process.
 944         :type teardown_kw: Keyword
 945         :returns: Nothing.
 946         """
 947         pass
 948
 949     def visit_message(self, msg):
 950         """Implements visiting the message.
 951
 952         :param msg: Message to process.
 953         :type msg: Message
 954         :returns: Nothing.
 955         """
 956         if self.start_message(msg) is not False:
 957             self.end_message(msg)
 958
 959     def start_message(self, msg):
 960         """Called when message starts. Get required information from messages:
 961         - VPP version.
 962
 963         :param msg: Message to process.
 964         :type msg: Message
 965         :returns: Nothing.
 966         """
 967
 968         if self._msg_type:
 969             self.parse_msg[self._msg_type](msg)
 970
 971     def end_message(self, msg):
 972         """Called when message ends. Default implementation does nothing.
 973
 974         :param msg: Message to process.
 975         :type msg: Message
 976         :returns: Nothing.
 977         """
 978         pass
 979
 980
 981 class InputData(object):
 982     """Input data
 983
 984     The data is extracted from output.xml files generated by Jenkins jobs and
 985     stored in pandas' DataFrames.
 986
 987     The data structure:
 988     - job name
 989       - build number
 990         - metadata
 991           (as described in ExecutionChecker documentation)
 992         - suites
 993           (as described in ExecutionChecker documentation)
 994         - tests
 995           (as described in ExecutionChecker documentation)
 996     """
 997
 998     def __init__(self, spec):
 999         """Initialization.
1000
1001         :param spec: Specification.
1002         :type spec: Specification
1003         """
1004
1005         # Specification:
1006         self._cfg = spec
1007
1008         # Data store:
1009         self._input_data = pd.Series()
1010
1011     @property
1012     def data(self):
1013         """Getter - Input data.
1014
1015         :returns: Input data
1016         :rtype: pandas.Series
1017         """
1018         return self._input_data
1019
1020     def metadata(self, job, build):
1021         """Getter - metadata
1022
1023         :param job: Job which metadata we want.
1024         :param build: Build which metadata we want.
1025         :type job: str
1026         :type build: str
1027         :returns: Metadata
1028         :rtype: pandas.Series
1029         """
1030
1031         return self.data[job][build]["metadata"]
1032
1033     def suites(self, job, build):
1034         """Getter - suites
1035
1036         :param job: Job which suites we want.
1037         :param build: Build which suites we want.
1038         :type job: str
1039         :type build: str
1040         :returns: Suites.
1041         :rtype: pandas.Series
1042         """
1043
1044         return self.data[job][str(build)]["suites"]
1045
1046     def tests(self, job, build):
1047         """Getter - tests
1048
1049         :param job: Job which tests we want.
1050         :param build: Build which tests we want.
1051         :type job: str
1052         :type build: str
1053         :returns: Tests.
1054         :rtype: pandas.Series
1055         """
1056
1057         return self.data[job][build]["tests"]
1058
1059     def _parse_tests(self, job, build, log):
1060         """Process data from robot output.xml file and return JSON structured
1061         data.
1062
1063         :param job: The name of job which build output data will be processed.
1064         :param build: The build which output data will be processed.
1065         :param log: List of log messages.
1066         :type job: str
1067         :type build: dict
1068         :type log: list of tuples (severity, msg)
1069         :returns: JSON data structure.
1070         :rtype: dict
1071         """
1072
1073         metadata = {
1074             "job": job,
1075             "build": build
1076         }
1077
1078         with open(build["file-name"], 'r') as data_file:
1079             try:
1080                 result = ExecutionResult(data_file)
1081             except errors.DataError as err:
1082                 log.append(("ERROR", "Error occurred while parsing output.xml: "
1083                                      "{0}".format(err)))
1084                 return None
1085         checker = ExecutionChecker(metadata, self._cfg.mapping,
1086                                    self._cfg.ignore)
1087         result.visit(checker)
1088
1089         return checker.data
1090
1091     def _download_and_parse_build(self, pid, data_queue, job, build, repeat):
1092         """Download and parse the input data file.
1093
1094         :param pid: PID of the process executing this method.
1095         :param data_queue: Shared memory between processes. Queue which keeps
1096             the result data. This data is then read by the main process and used
1097             in further processing.
1098         :param job: Name of the Jenkins job which generated the processed input
1099             file.
1100         :param build: Information about the Jenkins build which generated the
1101             processed input file.
1102         :param repeat: Repeat the download specified number of times if not
1103             successful.
1104         :type pid: int
1105         :type data_queue: multiprocessing.Manager().Queue()
1106         :type job: str
1107         :type build: dict
1108         :type repeat: int
1109         """
1110
1111         logs = list()
1112
1113         logging.info("  Processing the job/build: {0}: {1}".
1114                      format(job, build["build"]))
1115
1116         logs.append(("INFO", "  Processing the job/build: {0}: {1}".
1117                      format(job, build["build"])))
1118
1119         state = "failed"
1120         success = False
1121         data = None
1122         do_repeat = repeat
1123         while do_repeat:
1124             success = download_and_unzip_data_file(self._cfg, job, build, pid,
1125                                                    logs)
1126             if success:
1127                 break
1128             do_repeat -= 1
1129         if not success:
1130             logs.append(("ERROR", "It is not possible to download the input "
1131                                   "data file from the job '{job}', build "
1132                                   "'{build}', or it is damaged. Skipped.".
1133                          format(job=job, build=build["build"])))
1134         if success:
1135             logs.append(("INFO", "  Processing data from the build '{0}' ...".
1136                          format(build["build"])))
1137             data = self._parse_tests(job, build, logs)
1138             if data is None:
1139                 logs.append(("ERROR", "Input data file from the job '{job}', "
1140                                       "build '{build}' is damaged. Skipped.".
1141                              format(job=job, build=build["build"])))
1142             else:
1143                 state = "processed"
1144
1145             try:
1146                 remove(build["file-name"])
1147             except OSError as err:
1148                 logs.append(("ERROR", "Cannot remove the file '{0}': {1}".
1149                              format(build["file-name"], repr(err))))
1150
1151         # If the time-period is defined in the specification file, remove all
1152         # files which are outside the time period.
1153         timeperiod = self._cfg.input.get("time-period", None)
1154         if timeperiod and data:
1155             now = dt.utcnow()
1156             timeperiod = timedelta(int(timeperiod))
1157             metadata = data.get("metadata", None)
1158             if metadata:
1159                 generated = metadata.get("generated", None)
1160                 if generated:
1161                     generated = dt.strptime(generated, "%Y%m%d %H:%M")
1162                     if (now - generated) > timeperiod:
1163                         # Remove the data and the file:
1164                         state = "removed"
1165                         data = None
1166                         logs.append(
1167                             ("INFO",
1168                              "    The build {job}/{build} is outdated, will be "
1169                              "removed".format(job=job, build=build["build"])))
1170                         file_name = self._cfg.input["file-name"]
1171                         full_name = join(
1172                             self._cfg.environment["paths"]["DIR[WORKING,DATA]"],
1173                             "{job}{sep}{build}{sep}{name}".
1174                                 format(job=job,
1175                                        sep=SEPARATOR,
1176                                        build=build["build"],
1177                                        name=file_name))
1178                         try:
1179                             remove(full_name)
1180                             logs.append(("INFO",
1181                                          "    The file {name} has been removed".
1182                                          format(name=full_name)))
1183                         except OSError as err:
1184                             logs.append(("ERROR",
1185                                         "Cannot remove the file '{0}': {1}".
1186                                         format(full_name, repr(err))))
1187
1188         logs.append(("INFO", "  Done."))
1189
1190         result = {
1191             "data": data,
1192             "state": state,
1193             "job": job,
1194             "build": build,
1195             "logs": logs
1196         }
1197         data_queue.put(result)
1198
1199     def download_and_parse_data(self, repeat=1):
1200         """Download the input data files, parse input data from input files and
1201         store in pandas' Series.
1202
1203         :param repeat: Repeat the download specified number of times if not
1204             successful.
1205         :type repeat: int
1206         """
1207
1208         logging.info("Downloading and parsing input files ...")
1209
1210         work_queue = multiprocessing.JoinableQueue()
1211         manager = multiprocessing.Manager()
1212         data_queue = manager.Queue()
1213         cpus = multiprocessing.cpu_count()
1214
1215         workers = list()
1216         for cpu in range(cpus):
1217             worker = Worker(work_queue,
1218                             data_queue,
1219                             self._download_and_parse_build)
1220             worker.daemon = True
1221             worker.start()
1222             workers.append(worker)
1223             os.system("taskset -p -c {0} {1} > /dev/null 2>&1".
1224                       format(cpu, worker.pid))
1225
1226         for job, builds in self._cfg.builds.items():
1227             for build in builds:
1228                 work_queue.put((job, build, repeat))
1229
1230         work_queue.join()
1231
1232         logging.info("Done.")
1233
1234         while not data_queue.empty():
1235             result = data_queue.get()
1236
1237             job = result["job"]
1238             build_nr = result["build"]["build"]
1239
1240             if result["data"]:
1241                 data = result["data"]
1242                 build_data = pd.Series({
1243                     "metadata": pd.Series(data["metadata"].values(),
1244                                           index=data["metadata"].keys()),
1245                     "suites": pd.Series(data["suites"].values(),
1246                                         index=data["suites"].keys()),
1247                     "tests": pd.Series(data["tests"].values(),
1248                                        index=data["tests"].keys())})
1249
1250                 if self._input_data.get(job, None) is None:
1251                     self._input_data[job] = pd.Series()
1252                 self._input_data[job][str(build_nr)] = build_data
1253
1254                 self._cfg.set_input_file_name(job, build_nr,
1255                                               result["build"]["file-name"])
1256
1257             self._cfg.set_input_state(job, build_nr, result["state"])
1258
1259             for item in result["logs"]:
1260                 if item[0] == "INFO":
1261                     logging.info(item[1])
1262                 elif item[0] == "ERROR":
1263                     logging.error(item[1])
1264                 elif item[0] == "DEBUG":
1265                     logging.debug(item[1])
1266                 elif item[0] == "CRITICAL":
1267                     logging.critical(item[1])
1268                 elif item[0] == "WARNING":
1269                     logging.warning(item[1])
1270
1271         del data_queue
1272
1273         # Terminate all workers
1274         for worker in workers:
1275             worker.terminate()
1276             worker.join()
1277
1278         logging.info("Done.")
1279
1280     @staticmethod
1281     def _end_of_tag(tag_filter, start=0, closer="'"):
1282         """Return the index of character in the string which is the end of tag.
1283
1284         :param tag_filter: The string where the end of tag is being searched.
1285         :param start: The index where the searching is stated.
1286         :param closer: The character which is the tag closer.
1287         :type tag_filter: str
1288         :type start: int
1289         :type closer: str
1290         :returns: The index of the tag closer.
1291         :rtype: int
1292         """
1293
1294         try:
1295             idx_opener = tag_filter.index(closer, start)
1296             return tag_filter.index(closer, idx_opener + 1)
1297         except ValueError:
1298             return None
1299
1300     @staticmethod
1301     def _condition(tag_filter):
1302         """Create a conditional statement from the given tag filter.
1303
1304         :param tag_filter: Filter based on tags from the element specification.
1305         :type tag_filter: str
1306         :returns: Conditional statement which can be evaluated.
1307         :rtype: str
1308         """
1309
1310         index = 0
1311         while True:
1312             index = InputData._end_of_tag(tag_filter, index)
1313             if index is None:
1314                 return tag_filter
1315             index += 1
1316             tag_filter = tag_filter[:index] + " in tags" + tag_filter[index:]
1317
1318     def filter_data(self, element, params=None, data_set="tests",
1319                     continue_on_error=False):
1320         """Filter required data from the given jobs and builds.
1321
1322         The output data structure is:
1323
1324         - job 1
1325           - build 1
1326             - test (or suite) 1 ID:
1327               - param 1
1328               - param 2
1329               ...
1330               - param n
1331             ...
1332             - test (or suite) n ID:
1333             ...
1334           ...
1335           - build n
1336         ...
1337         - job n
1338
1339         :param element: Element which will use the filtered data.
1340         :param params: Parameters which will be included in the output. If None,
1341         all parameters are included.
1342         :param data_set: The set of data to be filtered: tests, suites,
1343         metadata.
1344         :param continue_on_error: Continue if there is error while reading the
1345         data. The Item will be empty then
1346         :type element: pandas.Series
1347         :type params: list
1348         :type data_set: str
1349         :type continue_on_error: bool
1350         :returns: Filtered data.
1351         :rtype pandas.Series
1352         """
1353
1354         try:
1355             if element["filter"] in ("all", "template"):
1356                 cond = "True"
1357             else:
1358                 cond = InputData._condition(element["filter"])
1359             logging.debug("   Filter: {0}".format(cond))
1360         except KeyError:
1361             logging.error("  No filter defined.")
1362             return None
1363
1364         if params is None:
1365             params = element.get("parameters", None)
1366             if params:
1367                 params.append("type")
1368
1369         data = pd.Series()
1370         try:
1371             for job, builds in element["data"].items():
1372                 data[job] = pd.Series()
1373                 for build in builds:
1374                     data[job][str(build)] = pd.Series()
1375                     try:
1376                         data_iter = self.data[job][str(build)][data_set].\
1377                             iteritems()
1378                     except KeyError:
1379                         if continue_on_error:
1380                             continue
1381                         else:
1382                             return None
1383                     for test_ID, test_data in data_iter:
1384                         if eval(cond, {"tags": test_data.get("tags", "")}):
1385                             data[job][str(build)][test_ID] = pd.Series()
1386                             if params is None:
1387                                 for param, val in test_data.items():
1388                                     data[job][str(build)][test_ID][param] = val
1389                             else:
1390                                 for param in params:
1391                                     try:
1392                                         data[job][str(build)][test_ID][param] =\
1393                                             test_data[param]
1394                                     except KeyError:
1395                                         data[job][str(build)][test_ID][param] =\
1396                                             "No Data"
1397             return data
1398
1399         except (KeyError, IndexError, ValueError) as err:
1400             logging.error("   Missing mandatory parameter in the element "
1401                           "specification: {0}".format(err))
1402             return None
1403         except AttributeError:
1404             return None
1405         except SyntaxError:
1406             logging.error("   The filter '{0}' is not correct. Check if all "
1407                           "tags are enclosed by apostrophes.".format(cond))
1408             return None
1409
1410     @staticmethod
1411     def merge_data(data):
1412         """Merge data from more jobs and builds to a simple data structure.
1413
1414         The output data structure is:
1415
1416         - test (suite) 1 ID:
1417           - param 1
1418           - param 2
1419           ...
1420           - param n
1421         ...
1422         - test (suite) n ID:
1423         ...
1424
1425         :param data: Data to merge.
1426         :type data: pandas.Series
1427         :returns: Merged data.
1428         :rtype: pandas.Series
1429         """
1430
1431         logging.info("    Merging data ...")
1432
1433         merged_data = pd.Series()
1434         for _, builds in data.iteritems():
1435             for _, item in builds.iteritems():
1436                 for ID, item_data in item.iteritems():
1437                     merged_data[ID] = item_data
1438
1439         return merged_data