resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import prettytable
  21 import pandas as pd
  22
  23 from string import replace
  24 from math import isnan
  25 from xml.etree import ElementTree as ET
  26
  27 from errors import PresentationError
  28 from utils import mean, stdev, relative_change, remove_outliers, split_outliers
  29
  30
  31 def generate_tables(spec, data):
  32     """Generate all tables specified in the specification file.
  33
  34     :param spec: Specification read from the specification file.
  35     :param data: Data to process.
  36     :type spec: Specification
  37     :type data: InputData
  38     """
  39
  40     logging.info("Generating the tables ...")
  41     for table in spec.tables:
  42         try:
  43             eval(table["algorithm"])(table, data)
  44         except NameError:
  45             logging.error("The algorithm '{0}' is not defined.".
  46                           format(table["algorithm"]))
  47     logging.info("Done.")
  48
  49
  50 def table_details(table, input_data):
  51     """Generate the table(s) with algorithm: table_detailed_test_results
  52     specified in the specification file.
  53
  54     :param table: Table to generate.
  55     :param input_data: Data to process.
  56     :type table: pandas.Series
  57     :type input_data: InputData
  58     """
  59
  60     logging.info("  Generating the table {0} ...".
  61                  format(table.get("title", "")))
  62
  63     # Transform the data
  64     data = input_data.filter_data(table)
  65
  66     # Prepare the header of the tables
  67     header = list()
  68     for column in table["columns"]:
  69         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  70
  71     # Generate the data for the table according to the model in the table
  72     # specification
  73     job = table["data"].keys()[0]
  74     build = str(table["data"][job][0])
  75     try:
  76         suites = input_data.suites(job, build)
  77     except KeyError:
  78         logging.error("    No data available. The table will not be generated.")
  79         return
  80
  81     for suite_longname, suite in suites.iteritems():
  82         # Generate data
  83         suite_name = suite["name"]
  84         table_lst = list()
  85         for test in data[job][build].keys():
  86             if data[job][build][test]["parent"] in suite_name:
  87                 row_lst = list()
  88                 for column in table["columns"]:
  89                     try:
  90                         col_data = str(data[job][build][test][column["data"].
  91                                        split(" ")[1]]).replace('"', '""')
  92                         if column["data"].split(" ")[1] in ("vat-history",
  93                                                             "show-run"):
  94                             col_data = replace(col_data, " |br| ", "",
  95                                                maxreplace=1)
  96                             col_data = " |prein| {0} |preout| ".\
  97                                 format(col_data[:-5])
  98                         row_lst.append('"{0}"'.format(col_data))
  99                     except KeyError:
 100                         row_lst.append("No data")
 101                 table_lst.append(row_lst)
 102
 103         # Write the data to file
 104         if table_lst:
 105             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 106                                             table["output-file-ext"])
 107             logging.info("      Writing file: '{}'".format(file_name))
 108             with open(file_name, "w") as file_handler:
 109                 file_handler.write(",".join(header) + "\n")
 110                 for item in table_lst:
 111                     file_handler.write(",".join(item) + "\n")
 112
 113     logging.info("  Done.")
 114
 115
 116 def table_merged_details(table, input_data):
 117     """Generate the table(s) with algorithm: table_merged_details
 118     specified in the specification file.
 119
 120     :param table: Table to generate.
 121     :param input_data: Data to process.
 122     :type table: pandas.Series
 123     :type input_data: InputData
 124     """
 125
 126     logging.info("  Generating the table {0} ...".
 127                  format(table.get("title", "")))
 128
 129     # Transform the data
 130     data = input_data.filter_data(table)
 131     data = input_data.merge_data(data)
 132     data.sort_index(inplace=True)
 133
 134     suites = input_data.filter_data(table, data_set="suites")
 135     suites = input_data.merge_data(suites)
 136
 137     # Prepare the header of the tables
 138     header = list()
 139     for column in table["columns"]:
 140         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 141
 142     for _, suite in suites.iteritems():
 143         # Generate data
 144         suite_name = suite["name"]
 145         table_lst = list()
 146         for test in data.keys():
 147             if data[test]["parent"] in suite_name:
 148                 row_lst = list()
 149                 for column in table["columns"]:
 150                     try:
 151                         col_data = str(data[test][column["data"].
 152                                        split(" ")[1]]).replace('"', '""')
 153                         if column["data"].split(" ")[1] in ("vat-history",
 154                                                             "show-run"):
 155                             col_data = replace(col_data, " |br| ", "",
 156                                                maxreplace=1)
 157                             col_data = " |prein| {0} |preout| ".\
 158                                 format(col_data[:-5])
 159                         row_lst.append('"{0}"'.format(col_data))
 160                     except KeyError:
 161                         row_lst.append("No data")
 162                 table_lst.append(row_lst)
 163
 164         # Write the data to file
 165         if table_lst:
 166             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 167                                             table["output-file-ext"])
 168             logging.info("      Writing file: '{}'".format(file_name))
 169             with open(file_name, "w") as file_handler:
 170                 file_handler.write(",".join(header) + "\n")
 171                 for item in table_lst:
 172                     file_handler.write(",".join(item) + "\n")
 173
 174     logging.info("  Done.")
 175
 176
 177 def table_performance_improvements(table, input_data):
 178     """Generate the table(s) with algorithm: table_performance_improvements
 179     specified in the specification file.
 180
 181     :param table: Table to generate.
 182     :param input_data: Data to process.
 183     :type table: pandas.Series
 184     :type input_data: InputData
 185     """
 186
 187     def _write_line_to_file(file_handler, data):
 188         """Write a line to the .csv file.
 189
 190         :param file_handler: File handler for the csv file. It must be open for
 191          writing text.
 192         :param data: Item to be written to the file.
 193         :type file_handler: BinaryIO
 194         :type data: list
 195         """
 196
 197         line_lst = list()
 198         for item in data:
 199             if isinstance(item["data"], str):
 200                 # Remove -?drdisc from the end
 201                 if item["data"].endswith("drdisc"):
 202                     item["data"] = item["data"][:-8]
 203                 line_lst.append(item["data"])
 204             elif isinstance(item["data"], float):
 205                 line_lst.append("{:.1f}".format(item["data"]))
 206             elif item["data"] is None:
 207                 line_lst.append("")
 208         file_handler.write(",".join(line_lst) + "\n")
 209
 210     logging.info("  Generating the table {0} ...".
 211                  format(table.get("title", "")))
 212
 213     # Read the template
 214     file_name = table.get("template", None)
 215     if file_name:
 216         try:
 217             tmpl = _read_csv_template(file_name)
 218         except PresentationError:
 219             logging.error("  The template '{0}' does not exist. Skipping the "
 220                           "table.".format(file_name))
 221             return None
 222     else:
 223         logging.error("The template is not defined. Skipping the table.")
 224         return None
 225
 226     # Transform the data
 227     data = input_data.filter_data(table)
 228
 229     # Prepare the header of the tables
 230     header = list()
 231     for column in table["columns"]:
 232         header.append(column["title"])
 233
 234     # Generate the data for the table according to the model in the table
 235     # specification
 236     tbl_lst = list()
 237     for tmpl_item in tmpl:
 238         tbl_item = list()
 239         for column in table["columns"]:
 240             cmd = column["data"].split(" ")[0]
 241             args = column["data"].split(" ")[1:]
 242             if cmd == "template":
 243                 try:
 244                     val = float(tmpl_item[int(args[0])])
 245                 except ValueError:
 246                     val = tmpl_item[int(args[0])]
 247                 tbl_item.append({"data": val})
 248             elif cmd == "data":
 249                 jobs = args[0:-1]
 250                 operation = args[-1]
 251                 data_lst = list()
 252                 for job in jobs:
 253                     for build in data[job]:
 254                         try:
 255                             data_lst.append(float(build[tmpl_item[0]]
 256                                                   ["throughput"]["value"]))
 257                         except (KeyError, TypeError):
 258                             # No data, ignore
 259                             continue
 260                 if data_lst:
 261                     tbl_item.append({"data": (eval(operation)(data_lst)) /
 262                                              1000000})
 263                 else:
 264                     tbl_item.append({"data": None})
 265             elif cmd == "operation":
 266                 operation = args[0]
 267                 try:
 268                     nr1 = float(tbl_item[int(args[1])]["data"])
 269                     nr2 = float(tbl_item[int(args[2])]["data"])
 270                     if nr1 and nr2:
 271                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
 272                     else:
 273                         tbl_item.append({"data": None})
 274                 except (IndexError, ValueError, TypeError):
 275                     logging.error("No data for {0}".format(tbl_item[0]["data"]))
 276                     tbl_item.append({"data": None})
 277                     continue
 278             else:
 279                 logging.error("Not supported command {0}. Skipping the table.".
 280                               format(cmd))
 281                 return None
 282         tbl_lst.append(tbl_item)
 283
 284     # Sort the table according to the relative change
 285     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
 286
 287     # Create the tables and write them to the files
 288     file_names = [
 289         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
 290         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
 291         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
 292         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
 293     ]
 294
 295     for file_name in file_names:
 296         logging.info("    Writing the file '{0}'".format(file_name))
 297         with open(file_name, "w") as file_handler:
 298             file_handler.write(",".join(header) + "\n")
 299             for item in tbl_lst:
 300                 if isinstance(item[-1]["data"], float):
 301                     rel_change = round(item[-1]["data"], 1)
 302                 else:
 303                     rel_change = item[-1]["data"]
 304                 if "ndr_top" in file_name \
 305                         and "ndr" in item[0]["data"] \
 306                         and rel_change >= 10.0:
 307                     _write_line_to_file(file_handler, item)
 308                 elif "pdr_top" in file_name \
 309                         and "pdr" in item[0]["data"] \
 310                         and rel_change >= 10.0:
 311                     _write_line_to_file(file_handler, item)
 312                 elif "ndr_low" in file_name \
 313                         and "ndr" in item[0]["data"] \
 314                         and rel_change < 10.0:
 315                     _write_line_to_file(file_handler, item)
 316                 elif "pdr_low" in file_name \
 317                         and "pdr" in item[0]["data"] \
 318                         and rel_change < 10.0:
 319                     _write_line_to_file(file_handler, item)
 320
 321     logging.info("  Done.")
 322
 323
 324 def _read_csv_template(file_name):
 325     """Read the template from a .csv file.
 326
 327     :param file_name: Name / full path / relative path of the file to read.
 328     :type file_name: str
 329     :returns: Data from the template as list (lines) of lists (items on line).
 330     :rtype: list
 331     :raises: PresentationError if it is not possible to read the file.
 332     """
 333
 334     try:
 335         with open(file_name, 'r') as csv_file:
 336             tmpl_data = list()
 337             for line in csv_file:
 338                 tmpl_data.append(line[:-1].split(","))
 339         return tmpl_data
 340     except IOError as err:
 341         raise PresentationError(str(err), level="ERROR")
 342
 343
 344 def table_performance_comparison(table, input_data):
 345     """Generate the table(s) with algorithm: table_performance_comparison
 346     specified in the specification file.
 347
 348     :param table: Table to generate.
 349     :param input_data: Data to process.
 350     :type table: pandas.Series
 351     :type input_data: InputData
 352     """
 353
 354     logging.info("  Generating the table {0} ...".
 355                  format(table.get("title", "")))
 356
 357     # Transform the data
 358     data = input_data.filter_data(table, continue_on_error=True)
 359
 360     # Prepare the header of the tables
 361     try:
 362         header = ["Test case",
 363                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 364                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 365                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 366                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 367                   "Change [%]"]
 368         header_str = ",".join(header) + "\n"
 369     except (AttributeError, KeyError) as err:
 370         logging.error("The model is invalid, missing parameter: {0}".
 371                       format(err))
 372         return
 373
 374     # Prepare data to the table:
 375     tbl_dict = dict()
 376     for job, builds in table["reference"]["data"].items():
 377         for build in builds:
 378             for tst_name, tst_data in data[job][str(build)].iteritems():
 379                 if tbl_dict.get(tst_name, None) is None:
 380                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 381                                             "-".join(tst_data["name"].
 382                                                      split("-")[1:]))
 383                     tbl_dict[tst_name] = {"name": name,
 384                                           "ref-data": list(),
 385                                           "cmp-data": list()}
 386                 try:
 387                     tbl_dict[tst_name]["ref-data"].\
 388                         append(tst_data["throughput"]["value"])
 389                 except TypeError:
 390                     pass  # No data in output.xml for this test
 391
 392     for job, builds in table["compare"]["data"].items():
 393         for build in builds:
 394             for tst_name, tst_data in data[job][str(build)].iteritems():
 395                 try:
 396                     tbl_dict[tst_name]["cmp-data"].\
 397                         append(tst_data["throughput"]["value"])
 398                 except KeyError:
 399                     pass
 400                 except TypeError:
 401                     tbl_dict.pop(tst_name, None)
 402
 403     tbl_lst = list()
 404     for tst_name in tbl_dict.keys():
 405         item = [tbl_dict[tst_name]["name"], ]
 406         if tbl_dict[tst_name]["ref-data"]:
 407             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
 408                                      outlier_const=table["outlier-const"])
 409             # TODO: Specify window size.
 410             item.append(round(mean(data_t) / 1000000, 2))
 411             item.append(round(stdev(data_t) / 1000000, 2))
 412         else:
 413             item.extend([None, None])
 414         if tbl_dict[tst_name]["cmp-data"]:
 415             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
 416                                      outlier_const=table["outlier-const"])
 417             # TODO: Specify window size.
 418             item.append(round(mean(data_t) / 1000000, 2))
 419             item.append(round(stdev(data_t) / 1000000, 2))
 420         else:
 421             item.extend([None, None])
 422         if item[1] is not None and item[3] is not None:
 423             item.append(int(relative_change(float(item[1]), float(item[3]))))
 424         if len(item) == 6:
 425             tbl_lst.append(item)
 426
 427     # Sort the table according to the relative change
 428     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 429
 430     # Generate tables:
 431     # All tests in csv:
 432     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
 433                                                table["output-file-ext"]),
 434                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
 435                                                table["output-file-ext"]),
 436                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
 437                                                table["output-file-ext"]),
 438                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 439                                                table["output-file-ext"]),
 440                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
 441                                                table["output-file-ext"]),
 442                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
 443                                                table["output-file-ext"])
 444                  ]
 445     for file_name in tbl_names:
 446         logging.info("      Writing file: '{0}'".format(file_name))
 447         with open(file_name, "w") as file_handler:
 448             file_handler.write(header_str)
 449             for test in tbl_lst:
 450                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
 451                         file_name.split("-")[-2] in test[0]):  # cores
 452                     test[0] = "-".join(test[0].split("-")[:-1])
 453                     file_handler.write(",".join([str(item) for item in test]) +
 454                                        "\n")
 455
 456     # All tests in txt:
 457     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
 458                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
 459                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
 460                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
 461                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
 462                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
 463                      ]
 464
 465     for i, txt_name in enumerate(tbl_names_txt):
 466         txt_table = None
 467         logging.info("      Writing file: '{0}'".format(txt_name))
 468         with open(tbl_names[i], 'rb') as csv_file:
 469             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 470             for row in csv_content:
 471                 if txt_table is None:
 472                     txt_table = prettytable.PrettyTable(row)
 473                 else:
 474                     txt_table.add_row(row)
 475             txt_table.align["Test case"] = "l"
 476         with open(txt_name, "w") as txt_file:
 477             txt_file.write(str(txt_table))
 478
 479     # Selected tests in csv:
 480     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
 481                                                table["output-file-ext"])
 482     with open(input_file, "r") as in_file:
 483         lines = list()
 484         for line in in_file:
 485             lines.append(line)
 486
 487     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
 488                                                table["output-file-ext"])
 489     logging.info("      Writing file: '{0}'".format(output_file))
 490     with open(output_file, "w") as out_file:
 491         out_file.write(header_str)
 492         for i, line in enumerate(lines[1:]):
 493             if i == table["nr-of-tests-shown"]:
 494                 break
 495             out_file.write(line)
 496
 497     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
 498                                                   table["output-file-ext"])
 499     logging.info("      Writing file: '{0}'".format(output_file))
 500     with open(output_file, "w") as out_file:
 501         out_file.write(header_str)
 502         for i, line in enumerate(lines[-1:0:-1]):
 503             if i == table["nr-of-tests-shown"]:
 504                 break
 505             out_file.write(line)
 506
 507     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 508                                                table["output-file-ext"])
 509     with open(input_file, "r") as in_file:
 510         lines = list()
 511         for line in in_file:
 512             lines.append(line)
 513
 514     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
 515                                                table["output-file-ext"])
 516     logging.info("      Writing file: '{0}'".format(output_file))
 517     with open(output_file, "w") as out_file:
 518         out_file.write(header_str)
 519         for i, line in enumerate(lines[1:]):
 520             if i == table["nr-of-tests-shown"]:
 521                 break
 522             out_file.write(line)
 523
 524     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
 525                                                   table["output-file-ext"])
 526     logging.info("      Writing file: '{0}'".format(output_file))
 527     with open(output_file, "w") as out_file:
 528         out_file.write(header_str)
 529         for i, line in enumerate(lines[-1:0:-1]):
 530             if i == table["nr-of-tests-shown"]:
 531                 break
 532             out_file.write(line)
 533
 534
 535 def table_performance_comparison_mrr(table, input_data):
 536     """Generate the table(s) with algorithm: table_performance_comparison_mrr
 537     specified in the specification file.
 538
 539     :param table: Table to generate.
 540     :param input_data: Data to process.
 541     :type table: pandas.Series
 542     :type input_data: InputData
 543     """
 544
 545     logging.info("  Generating the table {0} ...".
 546                  format(table.get("title", "")))
 547
 548     # Transform the data
 549     data = input_data.filter_data(table, continue_on_error=True)
 550
 551     # Prepare the header of the tables
 552     try:
 553         header = ["Test case",
 554                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 555                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 556                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 557                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 558                   "Change [%]"]
 559         header_str = ",".join(header) + "\n"
 560     except (AttributeError, KeyError) as err:
 561         logging.error("The model is invalid, missing parameter: {0}".
 562                       format(err))
 563         return
 564
 565     # Prepare data to the table:
 566     tbl_dict = dict()
 567     for job, builds in table["reference"]["data"].items():
 568         for build in builds:
 569             for tst_name, tst_data in data[job][str(build)].iteritems():
 570                 if tbl_dict.get(tst_name, None) is None:
 571                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 572                                             "-".join(tst_data["name"].
 573                                                      split("-")[1:]))
 574                     tbl_dict[tst_name] = {"name": name,
 575                                           "ref-data": list(),
 576                                           "cmp-data": list()}
 577                 try:
 578                     tbl_dict[tst_name]["ref-data"].\
 579                         append(tst_data["result"]["throughput"])
 580                 except TypeError:
 581                     pass  # No data in output.xml for this test
 582
 583     for job, builds in table["compare"]["data"].items():
 584         for build in builds:
 585             for tst_name, tst_data in data[job][str(build)].iteritems():
 586                 try:
 587                     tbl_dict[tst_name]["cmp-data"].\
 588                         append(tst_data["result"]["throughput"])
 589                 except KeyError:
 590                     pass
 591                 except TypeError:
 592                     tbl_dict.pop(tst_name, None)
 593
 594     tbl_lst = list()
 595     for tst_name in tbl_dict.keys():
 596         item = [tbl_dict[tst_name]["name"], ]
 597         if tbl_dict[tst_name]["ref-data"]:
 598             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
 599                                      outlier_const=table["outlier-const"])
 600             # TODO: Specify window size.
 601             item.append(round(mean(data_t) / 1000000, 2))
 602             item.append(round(stdev(data_t) / 1000000, 2))
 603         else:
 604             item.extend([None, None])
 605         if tbl_dict[tst_name]["cmp-data"]:
 606             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
 607                                      outlier_const=table["outlier-const"])
 608             # TODO: Specify window size.
 609             item.append(round(mean(data_t) / 1000000, 2))
 610             item.append(round(stdev(data_t) / 1000000, 2))
 611         else:
 612             item.extend([None, None])
 613         if item[1] is not None and item[3] is not None and item[1] != 0:
 614             item.append(int(relative_change(float(item[1]), float(item[3]))))
 615         if len(item) == 6:
 616             tbl_lst.append(item)
 617
 618     # Sort the table according to the relative change
 619     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 620
 621     # Generate tables:
 622     # All tests in csv:
 623     tbl_names = ["{0}-1t1c-full{1}".format(table["output-file"],
 624                                            table["output-file-ext"]),
 625                  "{0}-2t2c-full{1}".format(table["output-file"],
 626                                            table["output-file-ext"]),
 627                  "{0}-4t4c-full{1}".format(table["output-file"],
 628                                            table["output-file-ext"])
 629                  ]
 630     for file_name in tbl_names:
 631         logging.info("      Writing file: '{0}'".format(file_name))
 632         with open(file_name, "w") as file_handler:
 633             file_handler.write(header_str)
 634             for test in tbl_lst:
 635                 if file_name.split("-")[-2] in test[0]:  # cores
 636                     test[0] = "-".join(test[0].split("-")[:-1])
 637                     file_handler.write(",".join([str(item) for item in test]) +
 638                                        "\n")
 639
 640     # All tests in txt:
 641     tbl_names_txt = ["{0}-1t1c-full.txt".format(table["output-file"]),
 642                      "{0}-2t2c-full.txt".format(table["output-file"]),
 643                      "{0}-4t4c-full.txt".format(table["output-file"])
 644                      ]
 645
 646     for i, txt_name in enumerate(tbl_names_txt):
 647         txt_table = None
 648         logging.info("      Writing file: '{0}'".format(txt_name))
 649         with open(tbl_names[i], 'rb') as csv_file:
 650             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 651             for row in csv_content:
 652                 if txt_table is None:
 653                     txt_table = prettytable.PrettyTable(row)
 654                 else:
 655                     txt_table.add_row(row)
 656             txt_table.align["Test case"] = "l"
 657         with open(txt_name, "w") as txt_file:
 658             txt_file.write(str(txt_table))
 659
 660
 661 def table_performance_trending_dashboard(table, input_data):
 662     """Generate the table(s) with algorithm: table_performance_comparison
 663     specified in the specification file.
 664
 665     :param table: Table to generate.
 666     :param input_data: Data to process.
 667     :type table: pandas.Series
 668     :type input_data: InputData
 669     """
 670
 671     logging.info("  Generating the table {0} ...".
 672                  format(table.get("title", "")))
 673
 674     # Transform the data
 675     data = input_data.filter_data(table, continue_on_error=True)
 676
 677     # Prepare the header of the tables
 678     header = ["Test Case",
 679               "Throughput Trend [Mpps]",
 680               "Long Trend Compliance",
 681               "Trend Compliance",
 682               "Top Anomaly [Mpps]",
 683               "Change [%]",
 684               "Outliers [Number]"
 685               ]
 686     header_str = ",".join(header) + "\n"
 687
 688     # Prepare data to the table:
 689     tbl_dict = dict()
 690     for job, builds in table["data"].items():
 691         for build in builds:
 692             for tst_name, tst_data in data[job][str(build)].iteritems():
 693                 if tbl_dict.get(tst_name, None) is None:
 694                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 695                                             "-".join(tst_data["name"].
 696                                                      split("-")[1:]))
 697                     tbl_dict[tst_name] = {"name": name,
 698                                           "data": dict()}
 699                 try:
 700                     tbl_dict[tst_name]["data"][str(build)] =  \
 701                         tst_data["result"]["throughput"]
 702                 except (TypeError, KeyError):
 703                     pass  # No data in output.xml for this test
 704
 705     tbl_lst = list()
 706     for tst_name in tbl_dict.keys():
 707         if len(tbl_dict[tst_name]["data"]) > 2:
 708
 709             pd_data = pd.Series(tbl_dict[tst_name]["data"])
 710             win_size = min(pd_data.size, table["window"])
 711             # Test name:
 712             name = tbl_dict[tst_name]["name"]
 713
 714             median = pd_data.rolling(window=win_size, min_periods=2).median()
 715             median_idx = pd_data.size - table["long-trend-window"]
 716             median_idx = 0 if median_idx < 0 else median_idx
 717             max_median = max(median.values[median_idx:])
 718             trimmed_data, _ = split_outliers(pd_data, outlier_const=1.5,
 719                                              window=win_size)
 720             stdev_t = pd_data.rolling(window=win_size, min_periods=2).std()
 721
 722             rel_change_lst = [None, ]
 723             classification_lst = [None, ]
 724             median_lst = [None, ]
 725             sample_lst = [None, ]
 726             first = True
 727             for build_nr, value in pd_data.iteritems():
 728                 if first:
 729                     first = False
 730                     continue
 731                 # Relative changes list:
 732                 if not isnan(value) \
 733                         and not isnan(median[build_nr]) \
 734                         and median[build_nr] != 0:
 735                     rel_change_lst.append(round(
 736                         relative_change(float(median[build_nr]), float(value)),
 737                         2))
 738                 else:
 739                     rel_change_lst.append(None)
 740
 741                 # Classification list:
 742                 if isnan(trimmed_data[build_nr]) \
 743                         or isnan(median[build_nr]) \
 744                         or isnan(stdev_t[build_nr]) \
 745                         or isnan(value):
 746                     classification_lst.append("outlier")
 747                 elif value < (median[build_nr] - 3 * stdev_t[build_nr]):
 748                     classification_lst.append("regression")
 749                 elif value > (median[build_nr] + 3 * stdev_t[build_nr]):
 750                     classification_lst.append("progression")
 751                 else:
 752                     classification_lst.append("normal")
 753                 sample_lst.append(value)
 754                 median_lst.append(median[build_nr])
 755
 756             last_idx = len(classification_lst) - 1
 757             first_idx = last_idx - int(table["evaluated-window"])
 758             if first_idx < 0:
 759                 first_idx = 0
 760
 761             nr_outliers = 0
 762             consecutive_outliers = 0
 763             failure = False
 764             for item in classification_lst[first_idx:]:
 765                 if item == "outlier":
 766                     nr_outliers += 1
 767                     consecutive_outliers += 1
 768                     if consecutive_outliers == 3:
 769                         failure = True
 770                 else:
 771                     consecutive_outliers = 0
 772
 773             if failure:
 774                 classification = "failure"
 775             elif "regression" in classification_lst[first_idx:]:
 776                 classification = "regression"
 777             elif "progression" in classification_lst[first_idx:]:
 778                 classification = "progression"
 779             else:
 780                 classification = "normal"
 781
 782             if classification == "normal":
 783                 index = len(classification_lst) - 1
 784             else:
 785                 tmp_classification = "outlier" if classification == "failure" \
 786                     else classification
 787                 for idx in range(first_idx, len(classification_lst)):
 788                     if classification_lst[idx] == tmp_classification:
 789                         index = idx
 790                         break
 791                 for idx in range(index+1, len(classification_lst)):
 792                     if classification_lst[idx] == tmp_classification:
 793                         if (abs(rel_change_lst[idx]) >
 794                                 abs(rel_change_lst[index])):
 795                             index = idx
 796
 797             trend = round(float(median_lst[-1]) / 1000000, 2) \
 798                 if not isnan(median_lst[-1]) else '-'
 799             sample = round(float(sample_lst[index]) / 1000000, 2) \
 800                 if not isnan(sample_lst[index]) else '-'
 801             rel_change = rel_change_lst[index] \
 802                 if rel_change_lst[index] is not None else '-'
 803             if not isnan(max_median):
 804                 if not isnan(sample_lst[index]):
 805                     long_trend_threshold = max_median * \
 806                                            (table["long-trend-threshold"] / 100)
 807                     if sample_lst[index] < long_trend_threshold:
 808                         long_trend_classification = "failure"
 809                     else:
 810                         long_trend_classification = '-'
 811                 else:
 812                     long_trend_classification = "failure"
 813             else:
 814                 long_trend_classification = '-'
 815             tbl_lst.append([name,
 816                             trend,
 817                             long_trend_classification,
 818                             classification,
 819                             '-' if classification == "normal" else sample,
 820                             '-' if classification == "normal" else rel_change,
 821                             nr_outliers])
 822
 823     # Sort the table according to the classification
 824     tbl_sorted = list()
 825     for long_trend_class in ("failure", '-'):
 826         tbl_long = [item for item in tbl_lst if item[2] == long_trend_class]
 827         for classification in \
 828                 ("failure", "regression", "progression", "normal"):
 829             tbl_tmp = [item for item in tbl_long if item[3] == classification]
 830             tbl_tmp.sort(key=lambda rel: rel[0])
 831             tbl_sorted.extend(tbl_tmp)
 832
 833     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
 834
 835     logging.info("      Writing file: '{0}'".format(file_name))
 836     with open(file_name, "w") as file_handler:
 837         file_handler.write(header_str)
 838         for test in tbl_sorted:
 839             file_handler.write(",".join([str(item) for item in test]) + '\n')
 840
 841     txt_file_name = "{0}.txt".format(table["output-file"])
 842     txt_table = None
 843     logging.info("      Writing file: '{0}'".format(txt_file_name))
 844     with open(file_name, 'rb') as csv_file:
 845         csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 846         for row in csv_content:
 847             if txt_table is None:
 848                 txt_table = prettytable.PrettyTable(row)
 849             else:
 850                 txt_table.add_row(row)
 851         txt_table.align["Test case"] = "l"
 852     with open(txt_file_name, "w") as txt_file:
 853         txt_file.write(str(txt_table))
 854
 855
 856 def table_performance_trending_dashboard_html(table, input_data):
 857     """Generate the table(s) with algorithm:
 858     table_performance_trending_dashboard_html specified in the specification
 859     file.
 860
 861     :param table: Table to generate.
 862     :param input_data: Data to process.
 863     :type table: pandas.Series
 864     :type input_data: InputData
 865     """
 866
 867     logging.info("  Generating the table {0} ...".
 868                  format(table.get("title", "")))
 869
 870     try:
 871         with open(table["input-file"], 'rb') as csv_file:
 872             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 873             csv_lst = [item for item in csv_content]
 874     except KeyError:
 875         logging.warning("The input file is not defined.")
 876         return
 877     except csv.Error as err:
 878         logging.warning("Not possible to process the file '{0}'.\n{1}".
 879                         format(table["input-file"], err))
 880         return
 881
 882     # Table:
 883     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
 884
 885     # Table header:
 886     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
 887     for idx, item in enumerate(csv_lst[0]):
 888         alignment = "left" if idx == 0 else "center"
 889         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
 890         th.text = item
 891
 892     # Rows:
 893     for r_idx, row in enumerate(csv_lst[1:]):
 894         background = "#D4E4F7" if r_idx % 2 else "white"
 895         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
 896
 897         # Columns:
 898         for c_idx, item in enumerate(row):
 899             alignment = "left" if c_idx == 0 else "center"
 900             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
 901             # Name:
 902             url = "../trending/"
 903             file_name = ""
 904             anchor = "#"
 905             feature = ""
 906             if c_idx == 0:
 907                 if "memif" in item:
 908                     file_name = "container_memif.html"
 909
 910                 elif "vhost" in item:
 911                     if "l2xcbase" in item or "l2bdbasemaclrn" in item:
 912                         file_name = "vm_vhost_l2.html"
 913                     elif "ip4base" in item:
 914                         file_name = "vm_vhost_ip4.html"
 915
 916                 elif "ipsec" in item:
 917                     file_name = "ipsec.html"
 918
 919                 elif "ethip4lispip" in item or "ethip4vxlan" in item:
 920                     file_name = "ip4_tunnels.html"
 921
 922                 elif "ip4base" in item or "ip4scale" in item:
 923                     file_name = "ip4.html"
 924                     if "iacl" in item or "snat" in item or "cop" in item:
 925                         feature = "-features"
 926
 927                 elif "ip6base" in item or "ip6scale" in item:
 928                     file_name = "ip6.html"
 929
 930                 elif "l2xcbase" in item or "l2xcscale" in item \
 931                         or "l2bdbasemaclrn" in item or "l2bdscale" in item \
 932                         or "l2dbbasemaclrn" in item or "l2dbscale" in item:
 933                     file_name = "l2.html"
 934                     if "iacl" in item:
 935                         feature = "-features"
 936
 937                 if "x520" in item:
 938                     anchor += "x520-"
 939                 elif "x710" in item:
 940                     anchor += "x710-"
 941                 elif "xl710" in item:
 942                     anchor += "xl710-"
 943
 944                 if "64b" in item:
 945                     anchor += "64b-"
 946                 elif "78b" in item:
 947                     anchor += "78b"
 948                 elif "imix" in item:
 949                     anchor += "imix-"
 950                 elif "9000b" in item:
 951                     anchor += "9000b-"
 952                 elif "1518" in item:
 953                     anchor += "1518b-"
 954
 955                 if "1t1c" in item:
 956                     anchor += "1t1c"
 957                 elif "2t2c" in item:
 958                     anchor += "2t2c"
 959                 elif "4t4c" in item:
 960                     anchor += "4t4c"
 961
 962                 url = url + file_name + anchor + feature
 963
 964                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
 965                 ref.text = item
 966
 967             if c_idx == 3:
 968                 if item == "regression":
 969                     td.set("bgcolor", "#eca1a6")
 970                 elif item == "failure":
 971                     td.set("bgcolor", "#d6cbd3")
 972                 elif item == "progression":
 973                     td.set("bgcolor", "#bdcebe")
 974             if c_idx > 0:
 975                 td.text = item
 976
 977     try:
 978         with open(table["output-file"], 'w') as html_file:
 979             logging.info("      Writing file: '{0}'".
 980                          format(table["output-file"]))
 981             html_file.write(".. raw:: html\n\n\t")
 982             html_file.write(ET.tostring(dashboard))
 983             html_file.write("\n\t<p><br><br></p>\n")
 984     except KeyError:
 985         logging.warning("The output file is not defined.")
 986         return