resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2018 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import prettytable
  21 import pandas as pd
  22
  23 from string import replace
  24 from collections import OrderedDict
  25 from numpy import nan, isnan
  26 from xml.etree import ElementTree as ET
  27
  28 from errors import PresentationError
  29 from utils import mean, stdev, relative_change, remove_outliers, split_outliers
  30
  31
  32 def generate_tables(spec, data):
  33     """Generate all tables specified in the specification file.
  34
  35     :param spec: Specification read from the specification file.
  36     :param data: Data to process.
  37     :type spec: Specification
  38     :type data: InputData
  39     """
  40
  41     logging.info("Generating the tables ...")
  42     for table in spec.tables:
  43         try:
  44             eval(table["algorithm"])(table, data)
  45         except NameError:
  46             logging.error("The algorithm '{0}' is not defined.".
  47                           format(table["algorithm"]))
  48     logging.info("Done.")
  49
  50
  51 def table_details(table, input_data):
  52     """Generate the table(s) with algorithm: table_detailed_test_results
  53     specified in the specification file.
  54
  55     :param table: Table to generate.
  56     :param input_data: Data to process.
  57     :type table: pandas.Series
  58     :type input_data: InputData
  59     """
  60
  61     logging.info("  Generating the table {0} ...".
  62                  format(table.get("title", "")))
  63
  64     # Transform the data
  65     logging.info("    Creating the data set for the {0} '{1}'.".
  66                  format(table.get("type", ""), table.get("title", "")))
  67     data = input_data.filter_data(table)
  68
  69     # Prepare the header of the tables
  70     header = list()
  71     for column in table["columns"]:
  72         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  73
  74     # Generate the data for the table according to the model in the table
  75     # specification
  76     job = table["data"].keys()[0]
  77     build = str(table["data"][job][0])
  78     try:
  79         suites = input_data.suites(job, build)
  80     except KeyError:
  81         logging.error("    No data available. The table will not be generated.")
  82         return
  83
  84     for suite_longname, suite in suites.iteritems():
  85         # Generate data
  86         suite_name = suite["name"]
  87         table_lst = list()
  88         for test in data[job][build].keys():
  89             if data[job][build][test]["parent"] in suite_name:
  90                 row_lst = list()
  91                 for column in table["columns"]:
  92                     try:
  93                         col_data = str(data[job][build][test][column["data"].
  94                                        split(" ")[1]]).replace('"', '""')
  95                         if column["data"].split(" ")[1] in ("vat-history",
  96                                                             "show-run"):
  97                             col_data = replace(col_data, " |br| ", "",
  98                                                maxreplace=1)
  99                             col_data = " |prein| {0} |preout| ".\
 100                                 format(col_data[:-5])
 101                         row_lst.append('"{0}"'.format(col_data))
 102                     except KeyError:
 103                         row_lst.append("No data")
 104                 table_lst.append(row_lst)
 105
 106         # Write the data to file
 107         if table_lst:
 108             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 109                                             table["output-file-ext"])
 110             logging.info("      Writing file: '{}'".format(file_name))
 111             with open(file_name, "w") as file_handler:
 112                 file_handler.write(",".join(header) + "\n")
 113                 for item in table_lst:
 114                     file_handler.write(",".join(item) + "\n")
 115
 116     logging.info("  Done.")
 117
 118
 119 def table_merged_details(table, input_data):
 120     """Generate the table(s) with algorithm: table_merged_details
 121     specified in the specification file.
 122
 123     :param table: Table to generate.
 124     :param input_data: Data to process.
 125     :type table: pandas.Series
 126     :type input_data: InputData
 127     """
 128
 129     logging.info("  Generating the table {0} ...".
 130                  format(table.get("title", "")))
 131
 132     # Transform the data
 133     logging.info("    Creating the data set for the {0} '{1}'.".
 134                  format(table.get("type", ""), table.get("title", "")))
 135     data = input_data.filter_data(table)
 136     data = input_data.merge_data(data)
 137     data.sort_index(inplace=True)
 138
 139     logging.info("    Creating the data set for the {0} '{1}'.".
 140                  format(table.get("type", ""), table.get("title", "")))
 141     suites = input_data.filter_data(table, data_set="suites")
 142     suites = input_data.merge_data(suites)
 143
 144     # Prepare the header of the tables
 145     header = list()
 146     for column in table["columns"]:
 147         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 148
 149     for _, suite in suites.iteritems():
 150         # Generate data
 151         suite_name = suite["name"]
 152         table_lst = list()
 153         for test in data.keys():
 154             if data[test]["parent"] in suite_name:
 155                 row_lst = list()
 156                 for column in table["columns"]:
 157                     try:
 158                         col_data = str(data[test][column["data"].
 159                                        split(" ")[1]]).replace('"', '""')
 160                         if column["data"].split(" ")[1] in ("vat-history",
 161                                                             "show-run"):
 162                             col_data = replace(col_data, " |br| ", "",
 163                                                maxreplace=1)
 164                             col_data = " |prein| {0} |preout| ".\
 165                                 format(col_data[:-5])
 166                         row_lst.append('"{0}"'.format(col_data))
 167                     except KeyError:
 168                         row_lst.append("No data")
 169                 table_lst.append(row_lst)
 170
 171         # Write the data to file
 172         if table_lst:
 173             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 174                                             table["output-file-ext"])
 175             logging.info("      Writing file: '{}'".format(file_name))
 176             with open(file_name, "w") as file_handler:
 177                 file_handler.write(",".join(header) + "\n")
 178                 for item in table_lst:
 179                     file_handler.write(",".join(item) + "\n")
 180
 181     logging.info("  Done.")
 182
 183
 184 def table_performance_improvements(table, input_data):
 185     """Generate the table(s) with algorithm: table_performance_improvements
 186     specified in the specification file.
 187
 188     :param table: Table to generate.
 189     :param input_data: Data to process.
 190     :type table: pandas.Series
 191     :type input_data: InputData
 192     """
 193
 194     def _write_line_to_file(file_handler, data):
 195         """Write a line to the .csv file.
 196
 197         :param file_handler: File handler for the csv file. It must be open for
 198          writing text.
 199         :param data: Item to be written to the file.
 200         :type file_handler: BinaryIO
 201         :type data: list
 202         """
 203
 204         line_lst = list()
 205         for item in data:
 206             if isinstance(item["data"], str):
 207                 # Remove -?drdisc from the end
 208                 if item["data"].endswith("drdisc"):
 209                     item["data"] = item["data"][:-8]
 210                 line_lst.append(item["data"])
 211             elif isinstance(item["data"], float):
 212                 line_lst.append("{:.1f}".format(item["data"]))
 213             elif item["data"] is None:
 214                 line_lst.append("")
 215         file_handler.write(",".join(line_lst) + "\n")
 216
 217     logging.info("  Generating the table {0} ...".
 218                  format(table.get("title", "")))
 219
 220     # Read the template
 221     file_name = table.get("template", None)
 222     if file_name:
 223         try:
 224             tmpl = _read_csv_template(file_name)
 225         except PresentationError:
 226             logging.error("  The template '{0}' does not exist. Skipping the "
 227                           "table.".format(file_name))
 228             return None
 229     else:
 230         logging.error("The template is not defined. Skipping the table.")
 231         return None
 232
 233     # Transform the data
 234     logging.info("    Creating the data set for the {0} '{1}'.".
 235                  format(table.get("type", ""), table.get("title", "")))
 236     data = input_data.filter_data(table)
 237
 238     # Prepare the header of the tables
 239     header = list()
 240     for column in table["columns"]:
 241         header.append(column["title"])
 242
 243     # Generate the data for the table according to the model in the table
 244     # specification
 245     tbl_lst = list()
 246     for tmpl_item in tmpl:
 247         tbl_item = list()
 248         for column in table["columns"]:
 249             cmd = column["data"].split(" ")[0]
 250             args = column["data"].split(" ")[1:]
 251             if cmd == "template":
 252                 try:
 253                     val = float(tmpl_item[int(args[0])])
 254                 except ValueError:
 255                     val = tmpl_item[int(args[0])]
 256                 tbl_item.append({"data": val})
 257             elif cmd == "data":
 258                 jobs = args[0:-1]
 259                 operation = args[-1]
 260                 data_lst = list()
 261                 for job in jobs:
 262                     for build in data[job]:
 263                         try:
 264                             data_lst.append(float(build[tmpl_item[0]]
 265                                                   ["throughput"]["value"]))
 266                         except (KeyError, TypeError):
 267                             # No data, ignore
 268                             continue
 269                 if data_lst:
 270                     tbl_item.append({"data": (eval(operation)(data_lst)) /
 271                                              1000000})
 272                 else:
 273                     tbl_item.append({"data": None})
 274             elif cmd == "operation":
 275                 operation = args[0]
 276                 try:
 277                     nr1 = float(tbl_item[int(args[1])]["data"])
 278                     nr2 = float(tbl_item[int(args[2])]["data"])
 279                     if nr1 and nr2:
 280                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
 281                     else:
 282                         tbl_item.append({"data": None})
 283                 except (IndexError, ValueError, TypeError):
 284                     logging.error("No data for {0}".format(tbl_item[0]["data"]))
 285                     tbl_item.append({"data": None})
 286                     continue
 287             else:
 288                 logging.error("Not supported command {0}. Skipping the table.".
 289                               format(cmd))
 290                 return None
 291         tbl_lst.append(tbl_item)
 292
 293     # Sort the table according to the relative change
 294     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
 295
 296     # Create the tables and write them to the files
 297     file_names = [
 298         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
 299         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
 300         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
 301         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
 302     ]
 303
 304     for file_name in file_names:
 305         logging.info("    Writing the file '{0}'".format(file_name))
 306         with open(file_name, "w") as file_handler:
 307             file_handler.write(",".join(header) + "\n")
 308             for item in tbl_lst:
 309                 if isinstance(item[-1]["data"], float):
 310                     rel_change = round(item[-1]["data"], 1)
 311                 else:
 312                     rel_change = item[-1]["data"]
 313                 if "ndr_top" in file_name \
 314                         and "ndr" in item[0]["data"] \
 315                         and rel_change >= 10.0:
 316                     _write_line_to_file(file_handler, item)
 317                 elif "pdr_top" in file_name \
 318                         and "pdr" in item[0]["data"] \
 319                         and rel_change >= 10.0:
 320                     _write_line_to_file(file_handler, item)
 321                 elif "ndr_low" in file_name \
 322                         and "ndr" in item[0]["data"] \
 323                         and rel_change < 10.0:
 324                     _write_line_to_file(file_handler, item)
 325                 elif "pdr_low" in file_name \
 326                         and "pdr" in item[0]["data"] \
 327                         and rel_change < 10.0:
 328                     _write_line_to_file(file_handler, item)
 329
 330     logging.info("  Done.")
 331
 332
 333 def _read_csv_template(file_name):
 334     """Read the template from a .csv file.
 335
 336     :param file_name: Name / full path / relative path of the file to read.
 337     :type file_name: str
 338     :returns: Data from the template as list (lines) of lists (items on line).
 339     :rtype: list
 340     :raises: PresentationError if it is not possible to read the file.
 341     """
 342
 343     try:
 344         with open(file_name, 'r') as csv_file:
 345             tmpl_data = list()
 346             for line in csv_file:
 347                 tmpl_data.append(line[:-1].split(","))
 348         return tmpl_data
 349     except IOError as err:
 350         raise PresentationError(str(err), level="ERROR")
 351
 352
 353 def table_performance_comparison(table, input_data):
 354     """Generate the table(s) with algorithm: table_performance_comparison
 355     specified in the specification file.
 356
 357     :param table: Table to generate.
 358     :param input_data: Data to process.
 359     :type table: pandas.Series
 360     :type input_data: InputData
 361     """
 362
 363     logging.info("  Generating the table {0} ...".
 364                  format(table.get("title", "")))
 365
 366     # Transform the data
 367     logging.info("    Creating the data set for the {0} '{1}'.".
 368                  format(table.get("type", ""), table.get("title", "")))
 369     data = input_data.filter_data(table, continue_on_error=True)
 370
 371     # Prepare the header of the tables
 372     try:
 373         header = ["Test case", ]
 374
 375         history = table.get("history", None)
 376         if history:
 377             for item in history:
 378                 header.extend(
 379                     ["{0} Throughput [Mpps]".format(item["title"]),
 380                      "{0} Stdev [Mpps]".format(item["title"])])
 381         header.extend(
 382             ["{0} Throughput [Mpps]".format(table["reference"]["title"]),
 383              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
 384              "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 385              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
 386              "Change [%]"])
 387         header_str = ",".join(header) + "\n"
 388     except (AttributeError, KeyError) as err:
 389         logging.error("The model is invalid, missing parameter: {0}".
 390                       format(err))
 391         return
 392
 393     # Prepare data to the table:
 394     tbl_dict = dict()
 395     for job, builds in table["reference"]["data"].items():
 396         for build in builds:
 397             for tst_name, tst_data in data[job][str(build)].iteritems():
 398                 if tbl_dict.get(tst_name, None) is None:
 399                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 400                                             "-".join(tst_data["name"].
 401                                                      split("-")[1:]))
 402                     tbl_dict[tst_name] = {"name": name,
 403                                           "ref-data": list(),
 404                                           "cmp-data": list()}
 405                 try:
 406                     tbl_dict[tst_name]["ref-data"].\
 407                         append(tst_data["throughput"]["value"])
 408                 except TypeError:
 409                     pass  # No data in output.xml for this test
 410
 411     for job, builds in table["compare"]["data"].items():
 412         for build in builds:
 413             for tst_name, tst_data in data[job][str(build)].iteritems():
 414                 try:
 415                     tbl_dict[tst_name]["cmp-data"].\
 416                         append(tst_data["throughput"]["value"])
 417                 except KeyError:
 418                     pass
 419                 except TypeError:
 420                     tbl_dict.pop(tst_name, None)
 421     if history:
 422         for item in history:
 423             for job, builds in item["data"].items():
 424                 for build in builds:
 425                     for tst_name, tst_data in data[job][str(build)].iteritems():
 426                         if tbl_dict.get(tst_name, None) is None:
 427                             continue
 428                         if tbl_dict[tst_name].get("history", None) is None:
 429                             tbl_dict[tst_name]["history"] = OrderedDict()
 430                         if tbl_dict[tst_name]["history"].get(item["title"],
 431                                                              None) is None:
 432                             tbl_dict[tst_name]["history"][item["title"]] = \
 433                                 list()
 434                         try:
 435                             tbl_dict[tst_name]["history"][item["title"]].\
 436                                 append(tst_data["throughput"]["value"])
 437                         except (TypeError, KeyError):
 438                             pass
 439
 440     tbl_lst = list()
 441     for tst_name in tbl_dict.keys():
 442         item = [tbl_dict[tst_name]["name"], ]
 443         if history:
 444             if tbl_dict[tst_name].get("history", None) is not None:
 445                 for hist_data in tbl_dict[tst_name]["history"].values():
 446                     if hist_data:
 447                         data_t = remove_outliers(
 448                             hist_data, outlier_const=table["outlier-const"])
 449                         if data_t:
 450                             item.append(round(mean(data_t) / 1000000, 2))
 451                             item.append(round(stdev(data_t) / 1000000, 2))
 452                         else:
 453                             item.extend([None, None])
 454                     else:
 455                         item.extend([None, None])
 456             else:
 457                 item.extend([None, None])
 458         if tbl_dict[tst_name]["ref-data"]:
 459             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
 460                                      outlier_const=table["outlier-const"])
 461             # TODO: Specify window size.
 462             if data_t:
 463                 item.append(round(mean(data_t) / 1000000, 2))
 464                 item.append(round(stdev(data_t) / 1000000, 2))
 465             else:
 466                 item.extend([None, None])
 467         else:
 468             item.extend([None, None])
 469         if tbl_dict[tst_name]["cmp-data"]:
 470             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
 471                                      outlier_const=table["outlier-const"])
 472             # TODO: Specify window size.
 473             if data_t:
 474                 item.append(round(mean(data_t) / 1000000, 2))
 475                 item.append(round(stdev(data_t) / 1000000, 2))
 476             else:
 477                 item.extend([None, None])
 478         else:
 479             item.extend([None, None])
 480         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
 481             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
 482         if len(item) == len(header):
 483             tbl_lst.append(item)
 484
 485     # Sort the table according to the relative change
 486     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 487
 488     # Generate tables:
 489     # All tests in csv:
 490     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
 491                                                table["output-file-ext"]),
 492                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
 493                                                table["output-file-ext"]),
 494                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
 495                                                table["output-file-ext"]),
 496                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 497                                                table["output-file-ext"]),
 498                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
 499                                                table["output-file-ext"]),
 500                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
 501                                                table["output-file-ext"])
 502                  ]
 503     for file_name in tbl_names:
 504         logging.info("      Writing file: '{0}'".format(file_name))
 505         with open(file_name, "w") as file_handler:
 506             file_handler.write(header_str)
 507             for test in tbl_lst:
 508                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
 509                         file_name.split("-")[-2] in test[0]):  # cores
 510                     test[0] = "-".join(test[0].split("-")[:-1])
 511                     file_handler.write(",".join([str(item) for item in test]) +
 512                                        "\n")
 513
 514     # All tests in txt:
 515     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
 516                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
 517                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
 518                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
 519                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
 520                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
 521                      ]
 522
 523     for i, txt_name in enumerate(tbl_names_txt):
 524         txt_table = None
 525         logging.info("      Writing file: '{0}'".format(txt_name))
 526         with open(tbl_names[i], 'rb') as csv_file:
 527             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 528             for row in csv_content:
 529                 if txt_table is None:
 530                     txt_table = prettytable.PrettyTable(row)
 531                 else:
 532                     txt_table.add_row(row)
 533             txt_table.align["Test case"] = "l"
 534         with open(txt_name, "w") as txt_file:
 535             txt_file.write(str(txt_table))
 536
 537     # Selected tests in csv:
 538     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
 539                                                table["output-file-ext"])
 540     with open(input_file, "r") as in_file:
 541         lines = list()
 542         for line in in_file:
 543             lines.append(line)
 544
 545     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
 546                                                table["output-file-ext"])
 547     logging.info("      Writing file: '{0}'".format(output_file))
 548     with open(output_file, "w") as out_file:
 549         out_file.write(header_str)
 550         for i, line in enumerate(lines[1:]):
 551             if i == table["nr-of-tests-shown"]:
 552                 break
 553             out_file.write(line)
 554
 555     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
 556                                                   table["output-file-ext"])
 557     logging.info("      Writing file: '{0}'".format(output_file))
 558     with open(output_file, "w") as out_file:
 559         out_file.write(header_str)
 560         for i, line in enumerate(lines[-1:0:-1]):
 561             if i == table["nr-of-tests-shown"]:
 562                 break
 563             out_file.write(line)
 564
 565     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 566                                                table["output-file-ext"])
 567     with open(input_file, "r") as in_file:
 568         lines = list()
 569         for line in in_file:
 570             lines.append(line)
 571
 572     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
 573                                                table["output-file-ext"])
 574     logging.info("      Writing file: '{0}'".format(output_file))
 575     with open(output_file, "w") as out_file:
 576         out_file.write(header_str)
 577         for i, line in enumerate(lines[1:]):
 578             if i == table["nr-of-tests-shown"]:
 579                 break
 580             out_file.write(line)
 581
 582     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
 583                                                   table["output-file-ext"])
 584     logging.info("      Writing file: '{0}'".format(output_file))
 585     with open(output_file, "w") as out_file:
 586         out_file.write(header_str)
 587         for i, line in enumerate(lines[-1:0:-1]):
 588             if i == table["nr-of-tests-shown"]:
 589                 break
 590             out_file.write(line)
 591
 592
 593 def table_performance_comparison_mrr(table, input_data):
 594     """Generate the table(s) with algorithm: table_performance_comparison_mrr
 595     specified in the specification file.
 596
 597     :param table: Table to generate.
 598     :param input_data: Data to process.
 599     :type table: pandas.Series
 600     :type input_data: InputData
 601     """
 602
 603     logging.info("  Generating the table {0} ...".
 604                  format(table.get("title", "")))
 605
 606     # Transform the data
 607     logging.info("    Creating the data set for the {0} '{1}'.".
 608                  format(table.get("type", ""), table.get("title", "")))
 609     data = input_data.filter_data(table, continue_on_error=True)
 610
 611     # Prepare the header of the tables
 612     try:
 613         header = ["Test case",
 614                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 615                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 616                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 617                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 618                   "Change [%]"]
 619         header_str = ",".join(header) + "\n"
 620     except (AttributeError, KeyError) as err:
 621         logging.error("The model is invalid, missing parameter: {0}".
 622                       format(err))
 623         return
 624
 625     # Prepare data to the table:
 626     tbl_dict = dict()
 627     for job, builds in table["reference"]["data"].items():
 628         for build in builds:
 629             for tst_name, tst_data in data[job][str(build)].iteritems():
 630                 if tbl_dict.get(tst_name, None) is None:
 631                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 632                                             "-".join(tst_data["name"].
 633                                                      split("-")[1:]))
 634                     tbl_dict[tst_name] = {"name": name,
 635                                           "ref-data": list(),
 636                                           "cmp-data": list()}
 637                 try:
 638                     tbl_dict[tst_name]["ref-data"].\
 639                         append(tst_data["result"]["throughput"])
 640                 except TypeError:
 641                     pass  # No data in output.xml for this test
 642
 643     for job, builds in table["compare"]["data"].items():
 644         for build in builds:
 645             for tst_name, tst_data in data[job][str(build)].iteritems():
 646                 try:
 647                     tbl_dict[tst_name]["cmp-data"].\
 648                         append(tst_data["result"]["throughput"])
 649                 except KeyError:
 650                     pass
 651                 except TypeError:
 652                     tbl_dict.pop(tst_name, None)
 653
 654     tbl_lst = list()
 655     for tst_name in tbl_dict.keys():
 656         item = [tbl_dict[tst_name]["name"], ]
 657         if tbl_dict[tst_name]["ref-data"]:
 658             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
 659                                      outlier_const=table["outlier-const"])
 660             # TODO: Specify window size.
 661             if data_t:
 662                 item.append(round(mean(data_t) / 1000000, 2))
 663                 item.append(round(stdev(data_t) / 1000000, 2))
 664             else:
 665                 item.extend([None, None])
 666         else:
 667             item.extend([None, None])
 668         if tbl_dict[tst_name]["cmp-data"]:
 669             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
 670                                      outlier_const=table["outlier-const"])
 671             # TODO: Specify window size.
 672             if data_t:
 673                 item.append(round(mean(data_t) / 1000000, 2))
 674                 item.append(round(stdev(data_t) / 1000000, 2))
 675             else:
 676                 item.extend([None, None])
 677         else:
 678             item.extend([None, None])
 679         if item[1] is not None and item[3] is not None and item[1] != 0:
 680             item.append(int(relative_change(float(item[1]), float(item[3]))))
 681         if len(item) == 6:
 682             tbl_lst.append(item)
 683
 684     # Sort the table according to the relative change
 685     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 686
 687     # Generate tables:
 688     # All tests in csv:
 689     tbl_names = ["{0}-1t1c-full{1}".format(table["output-file"],
 690                                            table["output-file-ext"]),
 691                  "{0}-2t2c-full{1}".format(table["output-file"],
 692                                            table["output-file-ext"]),
 693                  "{0}-4t4c-full{1}".format(table["output-file"],
 694                                            table["output-file-ext"])
 695                  ]
 696     for file_name in tbl_names:
 697         logging.info("      Writing file: '{0}'".format(file_name))
 698         with open(file_name, "w") as file_handler:
 699             file_handler.write(header_str)
 700             for test in tbl_lst:
 701                 if file_name.split("-")[-2] in test[0]:  # cores
 702                     test[0] = "-".join(test[0].split("-")[:-1])
 703                     file_handler.write(",".join([str(item) for item in test]) +
 704                                        "\n")
 705
 706     # All tests in txt:
 707     tbl_names_txt = ["{0}-1t1c-full.txt".format(table["output-file"]),
 708                      "{0}-2t2c-full.txt".format(table["output-file"]),
 709                      "{0}-4t4c-full.txt".format(table["output-file"])
 710                      ]
 711
 712     for i, txt_name in enumerate(tbl_names_txt):
 713         txt_table = None
 714         logging.info("      Writing file: '{0}'".format(txt_name))
 715         with open(tbl_names[i], 'rb') as csv_file:
 716             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 717             for row in csv_content:
 718                 if txt_table is None:
 719                     txt_table = prettytable.PrettyTable(row)
 720                 else:
 721                     txt_table.add_row(row)
 722             txt_table.align["Test case"] = "l"
 723         with open(txt_name, "w") as txt_file:
 724             txt_file.write(str(txt_table))
 725
 726
 727 def table_performance_trending_dashboard(table, input_data):
 728     """Generate the table(s) with algorithm: table_performance_comparison
 729     specified in the specification file.
 730
 731     :param table: Table to generate.
 732     :param input_data: Data to process.
 733     :type table: pandas.Series
 734     :type input_data: InputData
 735     """
 736
 737     logging.info("  Generating the table {0} ...".
 738                  format(table.get("title", "")))
 739
 740     # Transform the data
 741     logging.info("    Creating the data set for the {0} '{1}'.".
 742                  format(table.get("type", ""), table.get("title", "")))
 743     data = input_data.filter_data(table, continue_on_error=True)
 744
 745     # Prepare the header of the tables
 746     header = ["Test Case",
 747               "Trend [Mpps]",
 748               "Short-Term Change [%]",
 749               "Long-Term Change [%]",
 750               "Regressions [#]",
 751               "Progressions [#]",
 752               "Outliers [#]"
 753               ]
 754     header_str = ",".join(header) + "\n"
 755
 756     # Prepare data to the table:
 757     tbl_dict = dict()
 758     for job, builds in table["data"].items():
 759         for build in builds:
 760             for tst_name, tst_data in data[job][str(build)].iteritems():
 761                 if tst_name.lower() in table["ignore-list"]:
 762                     continue
 763                 if tbl_dict.get(tst_name, None) is None:
 764                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 765                                             "-".join(tst_data["name"].
 766                                                      split("-")[1:]))
 767                     tbl_dict[tst_name] = {"name": name,
 768                                           "data": OrderedDict()}
 769                 try:
 770                     tbl_dict[tst_name]["data"][str(build)] =  \
 771                         tst_data["result"]["throughput"]
 772                 except (TypeError, KeyError):
 773                     pass  # No data in output.xml for this test
 774
 775     tbl_lst = list()
 776     for tst_name in tbl_dict.keys():
 777         if len(tbl_dict[tst_name]["data"]) > 2:
 778
 779             pd_data = pd.Series(tbl_dict[tst_name]["data"])
 780             data_t, _ = split_outliers(pd_data, outlier_const=1.5,
 781                                        window=table["window"])
 782             last_key = data_t.keys()[-1]
 783             win_size = min(data_t.size, table["window"])
 784             win_first_idx = data_t.size - win_size
 785             key_14 = data_t.keys()[win_first_idx]
 786             long_win_size = min(data_t.size, table["long-trend-window"])
 787             median_t = data_t.rolling(window=win_size, min_periods=2).median()
 788             stdev_t = data_t.rolling(window=win_size, min_periods=2).std()
 789             median_first_idx = median_t.size - long_win_size
 790             try:
 791                 max_median = max(
 792                     [x for x in median_t.values[median_first_idx:-win_size]
 793                      if not isnan(x)])
 794             except ValueError:
 795                 max_median = nan
 796             try:
 797                 last_median_t = median_t[last_key]
 798             except KeyError:
 799                 last_median_t = nan
 800             try:
 801                 median_t_14 = median_t[key_14]
 802             except KeyError:
 803                 median_t_14 = nan
 804
 805             # Classification list:
 806             classification_lst = list()
 807             for build_nr, value in data_t.iteritems():
 808                 if isnan(median_t[build_nr]) \
 809                         or isnan(stdev_t[build_nr]) \
 810                         or isnan(value):
 811                     classification_lst.append("outlier")
 812                 elif value < (median_t[build_nr] - 3 * stdev_t[build_nr]):
 813                     classification_lst.append("regression")
 814                 elif value > (median_t[build_nr] + 3 * stdev_t[build_nr]):
 815                     classification_lst.append("progression")
 816                 else:
 817                     classification_lst.append("normal")
 818
 819             if isnan(last_median_t) or isnan(median_t_14) or median_t_14 == 0.0:
 820                 rel_change_last = nan
 821             else:
 822                 rel_change_last = round(
 823                     ((last_median_t - median_t_14) / median_t_14) * 100, 2)
 824
 825             if isnan(max_median) or isnan(last_median_t) or max_median == 0.0:
 826                 rel_change_long = nan
 827             else:
 828                 rel_change_long = round(
 829                     ((last_median_t - max_median) / max_median) * 100, 2)
 830
 831             tbl_lst.append(
 832                 [tbl_dict[tst_name]["name"],
 833                  '-' if isnan(last_median_t) else
 834                  round(last_median_t / 1000000, 2),
 835                  '-' if isnan(rel_change_last) else rel_change_last,
 836                  '-' if isnan(rel_change_long) else rel_change_long,
 837                  classification_lst[win_first_idx:].count("regression"),
 838                  classification_lst[win_first_idx:].count("progression"),
 839                  classification_lst[win_first_idx:].count("outlier")])
 840
 841     tbl_lst.sort(key=lambda rel: rel[0])
 842
 843     tbl_sorted = list()
 844     for nrr in range(table["window"], -1, -1):
 845         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
 846         for nrp in range(table["window"], -1, -1):
 847             tbl_pro = [item for item in tbl_reg if item[5] == nrp]
 848             for nro in range(table["window"], -1, -1):
 849                 tbl_out = [item for item in tbl_pro if item[6] == nro]
 850                 tbl_out.sort(key=lambda rel: rel[2])
 851                 tbl_sorted.extend(tbl_out)
 852
 853     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
 854
 855     logging.info("      Writing file: '{0}'".format(file_name))
 856     with open(file_name, "w") as file_handler:
 857         file_handler.write(header_str)
 858         for test in tbl_sorted:
 859             file_handler.write(",".join([str(item) for item in test]) + '\n')
 860
 861     txt_file_name = "{0}.txt".format(table["output-file"])
 862     txt_table = None
 863     logging.info("      Writing file: '{0}'".format(txt_file_name))
 864     with open(file_name, 'rb') as csv_file:
 865         csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 866         for row in csv_content:
 867             if txt_table is None:
 868                 txt_table = prettytable.PrettyTable(row)
 869             else:
 870                 txt_table.add_row(row)
 871         txt_table.align["Test case"] = "l"
 872     with open(txt_file_name, "w") as txt_file:
 873         txt_file.write(str(txt_table))
 874
 875
 876 def table_performance_trending_dashboard_html(table, input_data):
 877     """Generate the table(s) with algorithm:
 878     table_performance_trending_dashboard_html specified in the specification
 879     file.
 880
 881     :param table: Table to generate.
 882     :param input_data: Data to process.
 883     :type table: pandas.Series
 884     :type input_data: InputData
 885     """
 886
 887     logging.info("  Generating the table {0} ...".
 888                  format(table.get("title", "")))
 889
 890     try:
 891         with open(table["input-file"], 'rb') as csv_file:
 892             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 893             csv_lst = [item for item in csv_content]
 894     except KeyError:
 895         logging.warning("The input file is not defined.")
 896         return
 897     except csv.Error as err:
 898         logging.warning("Not possible to process the file '{0}'.\n{1}".
 899                         format(table["input-file"], err))
 900         return
 901
 902     # Table:
 903     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
 904
 905     # Table header:
 906     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
 907     for idx, item in enumerate(csv_lst[0]):
 908         alignment = "left" if idx == 0 else "center"
 909         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
 910         th.text = item
 911
 912     # Rows:
 913     colors = {"regression": ("#ffcccc", "#ff9999"),
 914               "progression": ("#c6ecc6", "#9fdf9f"),
 915               "outlier": ("#e6e6e6", "#cccccc"),
 916               "normal": ("#e9f1fb", "#d4e4f7")}
 917     for r_idx, row in enumerate(csv_lst[1:]):
 918         if int(row[4]):
 919             color = "regression"
 920         elif int(row[5]):
 921             color = "progression"
 922         elif int(row[6]):
 923             color = "outlier"
 924         else:
 925             color = "normal"
 926         background = colors[color][r_idx % 2]
 927         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
 928
 929         # Columns:
 930         for c_idx, item in enumerate(row):
 931             alignment = "left" if c_idx == 0 else "center"
 932             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
 933             # Name:
 934             url = "../trending/"
 935             file_name = ""
 936             anchor = "#"
 937             feature = ""
 938             if c_idx == 0:
 939                 if "memif" in item:
 940                     file_name = "container_memif.html"
 941
 942                 elif "vhost" in item:
 943                     if "l2xcbase" in item or "l2bdbasemaclrn" in item:
 944                         file_name = "vm_vhost_l2.html"
 945                     elif "ip4base" in item:
 946                         file_name = "vm_vhost_ip4.html"
 947
 948                 elif "ipsec" in item:
 949                     file_name = "ipsec.html"
 950
 951                 elif "ethip4lispip" in item or "ethip4vxlan" in item:
 952                     file_name = "ip4_tunnels.html"
 953
 954                 elif "ip4base" in item or "ip4scale" in item:
 955                     file_name = "ip4.html"
 956                     if "iacl" in item or "snat" in item or "cop" in item:
 957                         feature = "-features"
 958
 959                 elif "ip6base" in item or "ip6scale" in item:
 960                     file_name = "ip6.html"
 961
 962                 elif "l2xcbase" in item or "l2xcscale" in item \
 963                         or "l2bdbasemaclrn" in item or "l2bdscale" in item \
 964                         or "l2dbbasemaclrn" in item or "l2dbscale" in item:
 965                     file_name = "l2.html"
 966                     if "iacl" in item:
 967                         feature = "-features"
 968
 969                 if "x520" in item:
 970                     anchor += "x520-"
 971                 elif "x710" in item:
 972                     anchor += "x710-"
 973                 elif "xl710" in item:
 974                     anchor += "xl710-"
 975
 976                 if "64b" in item:
 977                     anchor += "64b-"
 978                 elif "78b" in item:
 979                     anchor += "78b"
 980                 elif "imix" in item:
 981                     anchor += "imix-"
 982                 elif "9000b" in item:
 983                     anchor += "9000b-"
 984                 elif "1518" in item:
 985                     anchor += "1518b-"
 986
 987                 if "1t1c" in item:
 988                     anchor += "1t1c"
 989                 elif "2t2c" in item:
 990                     anchor += "2t2c"
 991                 elif "4t4c" in item:
 992                     anchor += "4t4c"
 993
 994                 url = url + file_name + anchor + feature
 995
 996                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
 997                 ref.text = item
 998
 999             if c_idx > 0:
1000                 td.text = item
1001
1002     try:
1003         with open(table["output-file"], 'w') as html_file:
1004             logging.info("      Writing file: '{0}'".
1005                          format(table["output-file"]))
1006             html_file.write(".. raw:: html\n\n\t")
1007             html_file.write(ET.tostring(dashboard))
1008             html_file.write("\n\t<p><br><br></p>\n")
1009     except KeyError:
1010         logging.warning("The output file is not defined.")
1011         return