resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import prettytable
  21 import pandas as pd
  22
  23 from string import replace
  24 from collections import OrderedDict
  25 from numpy import nan, isnan
  26 from xml.etree import ElementTree as ET
  27
  28 from errors import PresentationError
  29 from utils import mean, stdev, relative_change, remove_outliers,\
  30     split_outliers, classify_anomalies
  31
  32
  33 def generate_tables(spec, data):
  34     """Generate all tables specified in the specification file.
  35
  36     :param spec: Specification read from the specification file.
  37     :param data: Data to process.
  38     :type spec: Specification
  39     :type data: InputData
  40     """
  41
  42     logging.info("Generating the tables ...")
  43     for table in spec.tables:
  44         try:
  45             eval(table["algorithm"])(table, data)
  46         except NameError as err:
  47             logging.error("Probably algorithm '{alg}' is not defined: {err}".
  48                           format(alg=table["algorithm"], err=repr(err))
  49     logging.info("Done.")
  50
  51
  52 def table_details(table, input_data):
  53     """Generate the table(s) with algorithm: table_detailed_test_results
  54     specified in the specification file.
  55
  56     :param table: Table to generate.
  57     :param input_data: Data to process.
  58     :type table: pandas.Series
  59     :type input_data: InputData
  60     """
  61
  62     logging.info("  Generating the table {0} ...".
  63                  format(table.get("title", "")))
  64
  65     # Transform the data
  66     logging.info("    Creating the data set for the {0} '{1}'.".
  67                  format(table.get("type", ""), table.get("title", "")))
  68     data = input_data.filter_data(table)
  69
  70     # Prepare the header of the tables
  71     header = list()
  72     for column in table["columns"]:
  73         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  74
  75     # Generate the data for the table according to the model in the table
  76     # specification
  77     job = table["data"].keys()[0]
  78     build = str(table["data"][job][0])
  79     try:
  80         suites = input_data.suites(job, build)
  81     except KeyError:
  82         logging.error("    No data available. The table will not be generated.")
  83         return
  84
  85     for suite_longname, suite in suites.iteritems():
  86         # Generate data
  87         suite_name = suite["name"]
  88         table_lst = list()
  89         for test in data[job][build].keys():
  90             if data[job][build][test]["parent"] in suite_name:
  91                 row_lst = list()
  92                 for column in table["columns"]:
  93                     try:
  94                         col_data = str(data[job][build][test][column["data"].
  95                                        split(" ")[1]]).replace('"', '""')
  96                         if column["data"].split(" ")[1] in ("vat-history",
  97                                                             "show-run"):
  98                             col_data = replace(col_data, " |br| ", "",
  99                                                maxreplace=1)
 100                             col_data = " |prein| {0} |preout| ".\
 101                                 format(col_data[:-5])
 102                         row_lst.append('"{0}"'.format(col_data))
 103                     except KeyError:
 104                         row_lst.append("No data")
 105                 table_lst.append(row_lst)
 106
 107         # Write the data to file
 108         if table_lst:
 109             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 110                                             table["output-file-ext"])
 111             logging.info("      Writing file: '{}'".format(file_name))
 112             with open(file_name, "w") as file_handler:
 113                 file_handler.write(",".join(header) + "\n")
 114                 for item in table_lst:
 115                     file_handler.write(",".join(item) + "\n")
 116
 117     logging.info("  Done.")
 118
 119
 120 def table_merged_details(table, input_data):
 121     """Generate the table(s) with algorithm: table_merged_details
 122     specified in the specification file.
 123
 124     :param table: Table to generate.
 125     :param input_data: Data to process.
 126     :type table: pandas.Series
 127     :type input_data: InputData
 128     """
 129
 130     logging.info("  Generating the table {0} ...".
 131                  format(table.get("title", "")))
 132
 133     # Transform the data
 134     logging.info("    Creating the data set for the {0} '{1}'.".
 135                  format(table.get("type", ""), table.get("title", "")))
 136     data = input_data.filter_data(table)
 137     data = input_data.merge_data(data)
 138     data.sort_index(inplace=True)
 139
 140     logging.info("    Creating the data set for the {0} '{1}'.".
 141                  format(table.get("type", ""), table.get("title", "")))
 142     suites = input_data.filter_data(table, data_set="suites")
 143     suites = input_data.merge_data(suites)
 144
 145     # Prepare the header of the tables
 146     header = list()
 147     for column in table["columns"]:
 148         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 149
 150     for _, suite in suites.iteritems():
 151         # Generate data
 152         suite_name = suite["name"]
 153         table_lst = list()
 154         for test in data.keys():
 155             if data[test]["parent"] in suite_name:
 156                 row_lst = list()
 157                 for column in table["columns"]:
 158                     try:
 159                         col_data = str(data[test][column["data"].
 160                                        split(" ")[1]]).replace('"', '""')
 161                         if column["data"].split(" ")[1] in ("vat-history",
 162                                                             "show-run"):
 163                             col_data = replace(col_data, " |br| ", "",
 164                                                maxreplace=1)
 165                             col_data = " |prein| {0} |preout| ".\
 166                                 format(col_data[:-5])
 167                         row_lst.append('"{0}"'.format(col_data))
 168                     except KeyError:
 169                         row_lst.append("No data")
 170                 table_lst.append(row_lst)
 171
 172         # Write the data to file
 173         if table_lst:
 174             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 175                                             table["output-file-ext"])
 176             logging.info("      Writing file: '{}'".format(file_name))
 177             with open(file_name, "w") as file_handler:
 178                 file_handler.write(",".join(header) + "\n")
 179                 for item in table_lst:
 180                     file_handler.write(",".join(item) + "\n")
 181
 182     logging.info("  Done.")
 183
 184
 185 def table_performance_improvements(table, input_data):
 186     """Generate the table(s) with algorithm: table_performance_improvements
 187     specified in the specification file.
 188
 189     :param table: Table to generate.
 190     :param input_data: Data to process.
 191     :type table: pandas.Series
 192     :type input_data: InputData
 193     """
 194
 195     def _write_line_to_file(file_handler, data):
 196         """Write a line to the .csv file.
 197
 198         :param file_handler: File handler for the csv file. It must be open for
 199          writing text.
 200         :param data: Item to be written to the file.
 201         :type file_handler: BinaryIO
 202         :type data: list
 203         """
 204
 205         line_lst = list()
 206         for item in data:
 207             if isinstance(item["data"], str):
 208                 # Remove -?drdisc from the end
 209                 if item["data"].endswith("drdisc"):
 210                     item["data"] = item["data"][:-8]
 211                 line_lst.append(item["data"])
 212             elif isinstance(item["data"], float):
 213                 line_lst.append("{:.1f}".format(item["data"]))
 214             elif item["data"] is None:
 215                 line_lst.append("")
 216         file_handler.write(",".join(line_lst) + "\n")
 217
 218     logging.info("  Generating the table {0} ...".
 219                  format(table.get("title", "")))
 220
 221     # Read the template
 222     file_name = table.get("template", None)
 223     if file_name:
 224         try:
 225             tmpl = _read_csv_template(file_name)
 226         except PresentationError:
 227             logging.error("  The template '{0}' does not exist. Skipping the "
 228                           "table.".format(file_name))
 229             return None
 230     else:
 231         logging.error("The template is not defined. Skipping the table.")
 232         return None
 233
 234     # Transform the data
 235     logging.info("    Creating the data set for the {0} '{1}'.".
 236                  format(table.get("type", ""), table.get("title", "")))
 237     data = input_data.filter_data(table)
 238
 239     # Prepare the header of the tables
 240     header = list()
 241     for column in table["columns"]:
 242         header.append(column["title"])
 243
 244     # Generate the data for the table according to the model in the table
 245     # specification
 246     tbl_lst = list()
 247     for tmpl_item in tmpl:
 248         tbl_item = list()
 249         for column in table["columns"]:
 250             cmd = column["data"].split(" ")[0]
 251             args = column["data"].split(" ")[1:]
 252             if cmd == "template":
 253                 try:
 254                     val = float(tmpl_item[int(args[0])])
 255                 except ValueError:
 256                     val = tmpl_item[int(args[0])]
 257                 tbl_item.append({"data": val})
 258             elif cmd == "data":
 259                 jobs = args[0:-1]
 260                 operation = args[-1]
 261                 data_lst = list()
 262                 for job in jobs:
 263                     for build in data[job]:
 264                         try:
 265                             data_lst.append(float(build[tmpl_item[0]]
 266                                                   ["throughput"]["value"]))
 267                         except (KeyError, TypeError):
 268                             # No data, ignore
 269                             continue
 270                 if data_lst:
 271                     tbl_item.append({"data": (eval(operation)(data_lst)) /
 272                                              1000000})
 273                 else:
 274                     tbl_item.append({"data": None})
 275             elif cmd == "operation":
 276                 operation = args[0]
 277                 try:
 278                     nr1 = float(tbl_item[int(args[1])]["data"])
 279                     nr2 = float(tbl_item[int(args[2])]["data"])
 280                     if nr1 and nr2:
 281                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
 282                     else:
 283                         tbl_item.append({"data": None})
 284                 except (IndexError, ValueError, TypeError):
 285                     logging.error("No data for {0}".format(tbl_item[0]["data"]))
 286                     tbl_item.append({"data": None})
 287                     continue
 288             else:
 289                 logging.error("Not supported command {0}. Skipping the table.".
 290                               format(cmd))
 291                 return None
 292         tbl_lst.append(tbl_item)
 293
 294     # Sort the table according to the relative change
 295     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
 296
 297     # Create the tables and write them to the files
 298     file_names = [
 299         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
 300         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
 301         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
 302         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
 303     ]
 304
 305     for file_name in file_names:
 306         logging.info("    Writing the file '{0}'".format(file_name))
 307         with open(file_name, "w") as file_handler:
 308             file_handler.write(",".join(header) + "\n")
 309             for item in tbl_lst:
 310                 if isinstance(item[-1]["data"], float):
 311                     rel_change = round(item[-1]["data"], 1)
 312                 else:
 313                     rel_change = item[-1]["data"]
 314                 if "ndr_top" in file_name \
 315                         and "ndr" in item[0]["data"] \
 316                         and rel_change >= 10.0:
 317                     _write_line_to_file(file_handler, item)
 318                 elif "pdr_top" in file_name \
 319                         and "pdr" in item[0]["data"] \
 320                         and rel_change >= 10.0:
 321                     _write_line_to_file(file_handler, item)
 322                 elif "ndr_low" in file_name \
 323                         and "ndr" in item[0]["data"] \
 324                         and rel_change < 10.0:
 325                     _write_line_to_file(file_handler, item)
 326                 elif "pdr_low" in file_name \
 327                         and "pdr" in item[0]["data"] \
 328                         and rel_change < 10.0:
 329                     _write_line_to_file(file_handler, item)
 330
 331     logging.info("  Done.")
 332
 333
 334 def _read_csv_template(file_name):
 335     """Read the template from a .csv file.
 336
 337     :param file_name: Name / full path / relative path of the file to read.
 338     :type file_name: str
 339     :returns: Data from the template as list (lines) of lists (items on line).
 340     :rtype: list
 341     :raises: PresentationError if it is not possible to read the file.
 342     """
 343
 344     try:
 345         with open(file_name, 'r') as csv_file:
 346             tmpl_data = list()
 347             for line in csv_file:
 348                 tmpl_data.append(line[:-1].split(","))
 349         return tmpl_data
 350     except IOError as err:
 351         raise PresentationError(str(err), level="ERROR")
 352
 353
 354 def table_performance_comparison(table, input_data):
 355     """Generate the table(s) with algorithm: table_performance_comparison
 356     specified in the specification file.
 357
 358     :param table: Table to generate.
 359     :param input_data: Data to process.
 360     :type table: pandas.Series
 361     :type input_data: InputData
 362     """
 363
 364     logging.info("  Generating the table {0} ...".
 365                  format(table.get("title", "")))
 366
 367     # Transform the data
 368     logging.info("    Creating the data set for the {0} '{1}'.".
 369                  format(table.get("type", ""), table.get("title", "")))
 370     data = input_data.filter_data(table, continue_on_error=True)
 371
 372     # Prepare the header of the tables
 373     try:
 374         header = ["Test case", ]
 375
 376         history = table.get("history", None)
 377         if history:
 378             for item in history:
 379                 header.extend(
 380                     ["{0} Throughput [Mpps]".format(item["title"]),
 381                      "{0} Stdev [Mpps]".format(item["title"])])
 382         header.extend(
 383             ["{0} Throughput [Mpps]".format(table["reference"]["title"]),
 384              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 385              "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 386              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 387              "Change [%]"])
 388         header_str = ",".join(header) + "\n"
 389     except (AttributeError, KeyError) as err:
 390         logging.error("The model is invalid, missing parameter: {0}".
 391                       format(err))
 392         return
 393
 394     # Prepare data to the table:
 395     tbl_dict = dict()
 396     for job, builds in table["reference"]["data"].items():
 397         for build in builds:
 398             for tst_name, tst_data in data[job][str(build)].iteritems():
 399                 if tbl_dict.get(tst_name, None) is None:
 400                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 401                                             "-".join(tst_data["name"].
 402                                                      split("-")[1:]))
 403                     tbl_dict[tst_name] = {"name": name,
 404                                           "ref-data": list(),
 405                                           "cmp-data": list()}
 406                 try:
 407                     tbl_dict[tst_name]["ref-data"].\
 408                         append(tst_data["throughput"]["value"])
 409                 except TypeError:
 410                     pass  # No data in output.xml for this test
 411
 412     for job, builds in table["compare"]["data"].items():
 413         for build in builds:
 414             for tst_name, tst_data in data[job][str(build)].iteritems():
 415                 try:
 416                     tbl_dict[tst_name]["cmp-data"].\
 417                         append(tst_data["throughput"]["value"])
 418                 except KeyError:
 419                     pass
 420                 except TypeError:
 421                     tbl_dict.pop(tst_name, None)
 422     if history:
 423         for item in history:
 424             for job, builds in item["data"].items():
 425                 for build in builds:
 426                     for tst_name, tst_data in data[job][str(build)].iteritems():
 427                         if tbl_dict.get(tst_name, None) is None:
 428                             continue
 429                         if tbl_dict[tst_name].get("history", None) is None:
 430                             tbl_dict[tst_name]["history"] = OrderedDict()
 431                         if tbl_dict[tst_name]["history"].get(item["title"],
 432                                                              None) is None:
 433                             tbl_dict[tst_name]["history"][item["title"]] = \
 434                                 list()
 435                         try:
 436                             tbl_dict[tst_name]["history"][item["title"]].\
 437                                 append(tst_data["throughput"]["value"])
 438                         except (TypeError, KeyError):
 439                             pass
 440
 441     tbl_lst = list()
 442     for tst_name in tbl_dict.keys():
 443         item = [tbl_dict[tst_name]["name"], ]
 444         if history:
 445             if tbl_dict[tst_name].get("history", None) is not None:
 446                 for hist_data in tbl_dict[tst_name]["history"].values():
 447                     if hist_data:
 448                         data_t = remove_outliers(
 449                             hist_data, outlier_const=table["outlier-const"])
 450                         if data_t:
 451                             item.append(round(mean(data_t) / 1000000, 2))
 452                             item.append(round(stdev(data_t) / 1000000, 2))
 453                         else:
 454                             item.extend([None, None])
 455                     else:
 456                         item.extend([None, None])
 457             else:
 458                 item.extend([None, None])
 459         if tbl_dict[tst_name]["ref-data"]:
 460             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
 461                                      outlier_const=table["outlier-const"])
 462             # TODO: Specify window size.
 463             if data_t:
 464                 item.append(round(mean(data_t) / 1000000, 2))
 465                 item.append(round(stdev(data_t) / 1000000, 2))
 466             else:
 467                 item.extend([None, None])
 468         else:
 469             item.extend([None, None])
 470         if tbl_dict[tst_name]["cmp-data"]:
 471             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
 472                                      outlier_const=table["outlier-const"])
 473             # TODO: Specify window size.
 474             if data_t:
 475                 item.append(round(mean(data_t) / 1000000, 2))
 476                 item.append(round(stdev(data_t) / 1000000, 2))
 477             else:
 478                 item.extend([None, None])
 479         else:
 480             item.extend([None, None])
 481         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
 482             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 483         if len(item) == len(header):
 484             tbl_lst.append(item)
 485
 486     # Sort the table according to the relative change
 487     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 488
 489     # Generate tables:
 490     # All tests in csv:
 491     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
 492                                                table["output-file-ext"]),
 493                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
 494                                                table["output-file-ext"]),
 495                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
 496                                                table["output-file-ext"]),
 497                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 498                                                table["output-file-ext"]),
 499                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
 500                                                table["output-file-ext"]),
 501                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
 502                                                table["output-file-ext"])
 503                  ]
 504     for file_name in tbl_names:
 505         logging.info("      Writing file: '{0}'".format(file_name))
 506         with open(file_name, "w") as file_handler:
 507             file_handler.write(header_str)
 508             for test in tbl_lst:
 509                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
 510                         file_name.split("-")[-2] in test[0]):  # cores
 511                     test[0] = "-".join(test[0].split("-")[:-1])
 512                     file_handler.write(",".join([str(item) for item in test]) +
 513                                        "\n")
 514
 515     # All tests in txt:
 516     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
 517                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
 518                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
 519                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
 520                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
 521                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
 522                      ]
 523
 524     for i, txt_name in enumerate(tbl_names_txt):
 525         txt_table = None
 526         logging.info("      Writing file: '{0}'".format(txt_name))
 527         with open(tbl_names[i], 'rb') as csv_file:
 528             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 529             for row in csv_content:
 530                 if txt_table is None:
 531                     txt_table = prettytable.PrettyTable(row)
 532                 else:
 533                     txt_table.add_row(row)
 534             txt_table.align["Test case"] = "l"
 535         with open(txt_name, "w") as txt_file:
 536             txt_file.write(str(txt_table))
 537
 538     # Selected tests in csv:
 539     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
 540                                                table["output-file-ext"])
 541     with open(input_file, "r") as in_file:
 542         lines = list()
 543         for line in in_file:
 544             lines.append(line)
 545
 546     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
 547                                                table["output-file-ext"])
 548     logging.info("      Writing file: '{0}'".format(output_file))
 549     with open(output_file, "w") as out_file:
 550         out_file.write(header_str)
 551         for i, line in enumerate(lines[1:]):
 552             if i == table["nr-of-tests-shown"]:
 553                 break
 554             out_file.write(line)
 555
 556     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
 557                                                   table["output-file-ext"])
 558     logging.info("      Writing file: '{0}'".format(output_file))
 559     with open(output_file, "w") as out_file:
 560         out_file.write(header_str)
 561         for i, line in enumerate(lines[-1:0:-1]):
 562             if i == table["nr-of-tests-shown"]:
 563                 break
 564             out_file.write(line)
 565
 566     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 567                                                table["output-file-ext"])
 568     with open(input_file, "r") as in_file:
 569         lines = list()
 570         for line in in_file:
 571             lines.append(line)
 572
 573     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
 574                                                table["output-file-ext"])
 575     logging.info("      Writing file: '{0}'".format(output_file))
 576     with open(output_file, "w") as out_file:
 577         out_file.write(header_str)
 578         for i, line in enumerate(lines[1:]):
 579             if i == table["nr-of-tests-shown"]:
 580                 break
 581             out_file.write(line)
 582
 583     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
 584                                                   table["output-file-ext"])
 585     logging.info("      Writing file: '{0}'".format(output_file))
 586     with open(output_file, "w") as out_file:
 587         out_file.write(header_str)
 588         for i, line in enumerate(lines[-1:0:-1]):
 589             if i == table["nr-of-tests-shown"]:
 590                 break
 591             out_file.write(line)
 592
 593
 594 def table_performance_comparison_mrr(table, input_data):
 595     """Generate the table(s) with algorithm: table_performance_comparison_mrr
 596     specified in the specification file.
 597
 598     :param table: Table to generate.
 599     :param input_data: Data to process.
 600     :type table: pandas.Series
 601     :type input_data: InputData
 602     """
 603
 604     logging.info("  Generating the table {0} ...".
 605                  format(table.get("title", "")))
 606
 607     # Transform the data
 608     logging.info("    Creating the data set for the {0} '{1}'.".
 609                  format(table.get("type", ""), table.get("title", "")))
 610     data = input_data.filter_data(table, continue_on_error=True)
 611
 612     # Prepare the header of the tables
 613     try:
 614         header = ["Test case",
 615                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 616                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 617                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 618                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 619                   "Change [%]"]
 620         header_str = ",".join(header) + "\n"
 621     except (AttributeError, KeyError) as err:
 622         logging.error("The model is invalid, missing parameter: {0}".
 623                       format(err))
 624         return
 625
 626     # Prepare data to the table:
 627     tbl_dict = dict()
 628     for job, builds in table["reference"]["data"].items():
 629         for build in builds:
 630             for tst_name, tst_data in data[job][str(build)].iteritems():
 631                 if tbl_dict.get(tst_name, None) is None:
 632                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 633                                             "-".join(tst_data["name"].
 634                                                      split("-")[1:]))
 635                     tbl_dict[tst_name] = {"name": name,
 636                                           "ref-data": list(),
 637                                           "cmp-data": list()}
 638                 try:
 639                     tbl_dict[tst_name]["ref-data"].\
 640                         append(tst_data["result"]["throughput"])
 641                 except TypeError:
 642                     pass  # No data in output.xml for this test
 643
 644     for job, builds in table["compare"]["data"].items():
 645         for build in builds:
 646             for tst_name, tst_data in data[job][str(build)].iteritems():
 647                 try:
 648                     tbl_dict[tst_name]["cmp-data"].\
 649                         append(tst_data["result"]["throughput"])
 650                 except KeyError:
 651                     pass
 652                 except TypeError:
 653                     tbl_dict.pop(tst_name, None)
 654
 655     tbl_lst = list()
 656     for tst_name in tbl_dict.keys():
 657         item = [tbl_dict[tst_name]["name"], ]
 658         if tbl_dict[tst_name]["ref-data"]:
 659             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
 660                                      outlier_const=table["outlier-const"])
 661             # TODO: Specify window size.
 662             if data_t:
 663                 item.append(round(mean(data_t) / 1000000, 2))
 664                 item.append(round(stdev(data_t) / 1000000, 2))
 665             else:
 666                 item.extend([None, None])
 667         else:
 668             item.extend([None, None])
 669         if tbl_dict[tst_name]["cmp-data"]:
 670             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
 671                                      outlier_const=table["outlier-const"])
 672             # TODO: Specify window size.
 673             if data_t:
 674                 item.append(round(mean(data_t) / 1000000, 2))
 675                 item.append(round(stdev(data_t) / 1000000, 2))
 676             else:
 677                 item.extend([None, None])
 678         else:
 679             item.extend([None, None])
 680         if item[1] is not None and item[3] is not None and item[1] != 0:
 681             item.append(int(relative_change(float(item[1]), float(item[3]))))
 682         if len(item) == 6:
 683             tbl_lst.append(item)
 684
 685     # Sort the table according to the relative change
 686     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 687
 688     # Generate tables:
 689     # All tests in csv:
 690     tbl_names = ["{0}-1t1c-full{1}".format(table["output-file"],
 691                                            table["output-file-ext"]),
 692                  "{0}-2t2c-full{1}".format(table["output-file"],
 693                                            table["output-file-ext"]),
 694                  "{0}-4t4c-full{1}".format(table["output-file"],
 695                                            table["output-file-ext"])
 696                  ]
 697     for file_name in tbl_names:
 698         logging.info("      Writing file: '{0}'".format(file_name))
 699         with open(file_name, "w") as file_handler:
 700             file_handler.write(header_str)
 701             for test in tbl_lst:
 702                 if file_name.split("-")[-2] in test[0]:  # cores
 703                     test[0] = "-".join(test[0].split("-")[:-1])
 704                     file_handler.write(",".join([str(item) for item in test]) +
 705                                        "\n")
 706
 707     # All tests in txt:
 708     tbl_names_txt = ["{0}-1t1c-full.txt".format(table["output-file"]),
 709                      "{0}-2t2c-full.txt".format(table["output-file"]),
 710                      "{0}-4t4c-full.txt".format(table["output-file"])
 711                      ]
 712
 713     for i, txt_name in enumerate(tbl_names_txt):
 714         txt_table = None
 715         logging.info("      Writing file: '{0}'".format(txt_name))
 716         with open(tbl_names[i], 'rb') as csv_file:
 717             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 718             for row in csv_content:
 719                 if txt_table is None:
 720                     txt_table = prettytable.PrettyTable(row)
 721                 else:
 722                     txt_table.add_row(row)
 723             txt_table.align["Test case"] = "l"
 724         with open(txt_name, "w") as txt_file:
 725             txt_file.write(str(txt_table))
 726
 727
 728 def table_performance_trending_dashboard(table, input_data):
 729     """Generate the table(s) with algorithm: table_performance_comparison
 730     specified in the specification file.
 731
 732     :param table: Table to generate.
 733     :param input_data: Data to process.
 734     :type table: pandas.Series
 735     :type input_data: InputData
 736     """
 737
 738     logging.info("  Generating the table {0} ...".
 739                  format(table.get("title", "")))
 740
 741     # Transform the data
 742     logging.info("    Creating the data set for the {0} '{1}'.".
 743                  format(table.get("type", ""), table.get("title", "")))
 744     data = input_data.filter_data(table, continue_on_error=True)
 745
 746     # Prepare the header of the tables
 747     header = ["Test Case",
 748               "Trend [Mpps]",
 749               "Short-Term Change [%]",
 750               "Long-Term Change [%]",
 751               "Regressions [#]",
 752               "Progressions [#]",
 753               "Outliers [#]"
 754               ]
 755     header_str = ",".join(header) + "\n"
 756
 757     # Prepare data to the table:
 758     tbl_dict = dict()
 759     for job, builds in table["data"].items():
 760         for build in builds:
 761             for tst_name, tst_data in data[job][str(build)].iteritems():
 762                 if tst_name.lower() in table["ignore-list"]:
 763                     continue
 764                 if tbl_dict.get(tst_name, None) is None:
 765                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 766                                             "-".join(tst_data["name"].
 767                                                      split("-")[1:]))
 768                     tbl_dict[tst_name] = {"name": name,
 769                                           "data": OrderedDict()}
 770                 try:
 771                     tbl_dict[tst_name]["data"][str(build)] =  \
 772                         tst_data["result"]["throughput"]
 773                 except (TypeError, KeyError):
 774                     pass  # No data in output.xml for this test
 775
 776     tbl_lst = list()
 777     for tst_name in tbl_dict.keys():
 778         if len(tbl_dict[tst_name]["data"]) < 3:
 779             continue
 780
 781         pd_data = pd.Series(tbl_dict[tst_name]["data"])
 782         data_t, _ = split_outliers(pd_data, outlier_const=1.5,
 783                                    window=table["window"])
 784         last_key = data_t.keys()[-1]
 785         win_size = min(data_t.size, table["window"])
 786         win_first_idx = data_t.size - win_size
 787         key_14 = data_t.keys()[win_first_idx]
 788         long_win_size = min(data_t.size, table["long-trend-window"])
 789         median_t = data_t.rolling(window=win_size, min_periods=2).median()
 790         median_first_idx = median_t.size - long_win_size
 791         try:
 792             max_median = max(
 793                 [x for x in median_t.values[median_first_idx:-win_size]
 794                  if not isnan(x)])
 795         except ValueError:
 796             max_median = nan
 797         try:
 798             last_median_t = median_t[last_key]
 799         except KeyError:
 800             last_median_t = nan
 801         try:
 802             median_t_14 = median_t[key_14]
 803         except KeyError:
 804             median_t_14 = nan
 805
 806         if isnan(last_median_t) or isnan(median_t_14) or median_t_14 == 0.0:
 807             rel_change_last = nan
 808         else:
 809             rel_change_last = round(
 810                 ((last_median_t - median_t_14) / median_t_14) * 100, 2)
 811
 812         if isnan(max_median) or isnan(last_median_t) or max_median == 0.0:
 813             rel_change_long = nan
 814         else:
 815             rel_change_long = round(
 816                 ((last_median_t - max_median) / max_median) * 100, 2)
 817
 818         # Classification list:
 819         classification_lst = classify_anomalies(data_t, window=14)
 820
 821         if classification_lst:
 822             if isnan(rel_change_last) and isnan(rel_change_long):
 823                 continue
 824             tbl_lst.append(
 825                 [tbl_dict[tst_name]["name"],
 826                  '-' if isnan(last_median_t) else
 827                  round(last_median_t / 1000000, 2),
 828                  '-' if isnan(rel_change_last) else rel_change_last,
 829                  '-' if isnan(rel_change_long) else rel_change_long,
 830                  classification_lst[win_first_idx:].count("regression"),
 831                  classification_lst[win_first_idx:].count("progression"),
 832                  classification_lst[win_first_idx:].count("outlier")])
 833
 834     tbl_lst.sort(key=lambda rel: rel[0])
 835
 836     tbl_sorted = list()
 837     for nrr in range(table["window"], -1, -1):
 838         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
 839         for nrp in range(table["window"], -1, -1):
 840             tbl_pro = [item for item in tbl_reg if item[5] == nrp]
 841             for nro in range(table["window"], -1, -1):
 842                 tbl_out = [item for item in tbl_pro if item[6] == nro]
 843                 tbl_out.sort(key=lambda rel: rel[2])
 844                 tbl_sorted.extend(tbl_out)
 845
 846     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
 847
 848     logging.info("      Writing file: '{0}'".format(file_name))
 849     with open(file_name, "w") as file_handler:
 850         file_handler.write(header_str)
 851         for test in tbl_sorted:
 852             file_handler.write(",".join([str(item) for item in test]) + '\n')
 853
 854     txt_file_name = "{0}.txt".format(table["output-file"])
 855     txt_table = None
 856     logging.info("      Writing file: '{0}'".format(txt_file_name))
 857     with open(file_name, 'rb') as csv_file:
 858         csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 859         for row in csv_content:
 860             if txt_table is None:
 861                 txt_table = prettytable.PrettyTable(row)
 862             else:
 863                 txt_table.add_row(row)
 864         txt_table.align["Test case"] = "l"
 865     with open(txt_file_name, "w") as txt_file:
 866         txt_file.write(str(txt_table))
 867
 868
 869 def table_performance_trending_dashboard_html(table, input_data):
 870     """Generate the table(s) with algorithm:
 871     table_performance_trending_dashboard_html specified in the specification
 872     file.
 873
 874     :param table: Table to generate.
 875     :param input_data: Data to process.
 876     :type table: pandas.Series
 877     :type input_data: InputData
 878     """
 879
 880     logging.info("  Generating the table {0} ...".
 881                  format(table.get("title", "")))
 882
 883     try:
 884         with open(table["input-file"], 'rb') as csv_file:
 885             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 886             csv_lst = [item for item in csv_content]
 887     except KeyError:
 888         logging.warning("The input file is not defined.")
 889         return
 890     except csv.Error as err:
 891         logging.warning("Not possible to process the file '{0}'.\n{1}".
 892                         format(table["input-file"], err))
 893         return
 894
 895     # Table:
 896     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
 897
 898     # Table header:
 899     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
 900     for idx, item in enumerate(csv_lst[0]):
 901         alignment = "left" if idx == 0 else "center"
 902         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
 903         th.text = item
 904
 905     # Rows:
 906     colors = {"regression": ("#ffcccc", "#ff9999"),
 907               "progression": ("#c6ecc6", "#9fdf9f"),
 908               "outlier": ("#e6e6e6", "#cccccc"),
 909               "normal": ("#e9f1fb", "#d4e4f7")}
 910     for r_idx, row in enumerate(csv_lst[1:]):
 911         if int(row[4]):
 912             color = "regression"
 913         elif int(row[5]):
 914             color = "progression"
 915         elif int(row[6]):
 916             color = "outlier"
 917         else:
 918             color = "normal"
 919         background = colors[color][r_idx % 2]
 920         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
 921
 922         # Columns:
 923         for c_idx, item in enumerate(row):
 924             alignment = "left" if c_idx == 0 else "center"
 925             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
 926             # Name:
 927             url = "../trending/"
 928             file_name = ""
 929             anchor = "#"
 930             feature = ""
 931             if c_idx == 0:
 932                 if "memif" in item:
 933                     file_name = "container_memif.html"
 934
 935                 elif "srv6" in item:
 936                     file_name = "srv6.html"
 937
 938                 elif "vhost" in item:
 939                     if "l2xcbase" in item or "l2bdbasemaclrn" in item:
 940                         file_name = "vm_vhost_l2.html"
 941                     elif "ip4base" in item:
 942                         file_name = "vm_vhost_ip4.html"
 943
 944                 elif "ipsec" in item:
 945                     file_name = "ipsec.html"
 946
 947                 elif "ethip4lispip" in item or "ethip4vxlan" in item:
 948                     file_name = "ip4_tunnels.html"
 949
 950                 elif "ip4base" in item or "ip4scale" in item:
 951                     file_name = "ip4.html"
 952                     if "iacl" in item or "snat" in item or "cop" in item:
 953                         feature = "-features"
 954
 955                 elif "ip6base" in item or "ip6scale" in item:
 956                     file_name = "ip6.html"
 957
 958                 elif "l2xcbase" in item or "l2xcscale" in item \
 959                         or "l2bdbasemaclrn" in item or "l2bdscale" in item \
 960                         or "l2dbbasemaclrn" in item or "l2dbscale" in item:
 961                     file_name = "l2.html"
 962                     if "iacl" in item:
 963                         feature = "-features"
 964
 965                 if "x520" in item:
 966                     anchor += "x520-"
 967                 elif "x710" in item:
 968                     anchor += "x710-"
 969                 elif "xl710" in item:
 970                     anchor += "xl710-"
 971
 972                 if "64b" in item:
 973                     anchor += "64b-"
 974                 elif "78b" in item:
 975                     anchor += "78b-"
 976                 elif "imix" in item:
 977                     anchor += "imix-"
 978                 elif "9000b" in item:
 979                     anchor += "9000b-"
 980                 elif "1518" in item:
 981                     anchor += "1518b-"
 982
 983                 if "1t1c" in item:
 984                     anchor += "1t1c"
 985                 elif "2t2c" in item:
 986                     anchor += "2t2c"
 987                 elif "4t4c" in item:
 988                     anchor += "4t4c"
 989
 990                 url = url + file_name + anchor + feature
 991
 992                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
 993                 ref.text = item
 994
 995             if c_idx > 0:
 996                 td.text = item
 997
 998     try:
 999         with open(table["output-file"], 'w') as html_file:
1000             logging.info("      Writing file: '{0}'".
1001                          format(table["output-file"]))
1002             html_file.write(".. raw:: html\n\n\t")
1003             html_file.write(ET.tostring(dashboard))
1004             html_file.write("\n\t<p><br><br></p>\n")
1005     except KeyError:
1006         logging.warning("The output file is not defined.")
1007         return