resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import prettytable
  21 import pandas as pd
  22
  23 from string import replace
  24 from math import isnan
  25 from xml.etree import ElementTree as ET
  26
  27 from errors import PresentationError
  28 from utils import mean, stdev, relative_change, remove_outliers, find_outliers
  29
  30
  31 def generate_tables(spec, data):
  32     """Generate all tables specified in the specification file.
  33
  34     :param spec: Specification read from the specification file.
  35     :param data: Data to process.
  36     :type spec: Specification
  37     :type data: InputData
  38     """
  39
  40     logging.info("Generating the tables ...")
  41     for table in spec.tables:
  42         try:
  43             eval(table["algorithm"])(table, data)
  44         except NameError:
  45             logging.error("The algorithm '{0}' is not defined.".
  46                           format(table["algorithm"]))
  47     logging.info("Done.")
  48
  49
  50 def table_details(table, input_data):
  51     """Generate the table(s) with algorithm: table_detailed_test_results
  52     specified in the specification file.
  53
  54     :param table: Table to generate.
  55     :param input_data: Data to process.
  56     :type table: pandas.Series
  57     :type input_data: InputData
  58     """
  59
  60     logging.info("  Generating the table {0} ...".
  61                  format(table.get("title", "")))
  62
  63     # Transform the data
  64     data = input_data.filter_data(table)
  65
  66     # Prepare the header of the tables
  67     header = list()
  68     for column in table["columns"]:
  69         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  70
  71     # Generate the data for the table according to the model in the table
  72     # specification
  73     job = table["data"].keys()[0]
  74     build = str(table["data"][job][0])
  75     try:
  76         suites = input_data.suites(job, build)
  77     except KeyError:
  78         logging.error("    No data available. The table will not be generated.")
  79         return
  80
  81     for suite_longname, suite in suites.iteritems():
  82         # Generate data
  83         suite_name = suite["name"]
  84         table_lst = list()
  85         for test in data[job][build].keys():
  86             if data[job][build][test]["parent"] in suite_name:
  87                 row_lst = list()
  88                 for column in table["columns"]:
  89                     try:
  90                         col_data = str(data[job][build][test][column["data"].
  91                                        split(" ")[1]]).replace('"', '""')
  92                         if column["data"].split(" ")[1] in ("vat-history",
  93                                                             "show-run"):
  94                             col_data = replace(col_data, " |br| ", "",
  95                                                maxreplace=1)
  96                             col_data = " |prein| {0} |preout| ".\
  97                                 format(col_data[:-5])
  98                         row_lst.append('"{0}"'.format(col_data))
  99                     except KeyError:
 100                         row_lst.append("No data")
 101                 table_lst.append(row_lst)
 102
 103         # Write the data to file
 104         if table_lst:
 105             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 106                                             table["output-file-ext"])
 107             logging.info("      Writing file: '{}'".format(file_name))
 108             with open(file_name, "w") as file_handler:
 109                 file_handler.write(",".join(header) + "\n")
 110                 for item in table_lst:
 111                     file_handler.write(",".join(item) + "\n")
 112
 113     logging.info("  Done.")
 114
 115
 116 def table_merged_details(table, input_data):
 117     """Generate the table(s) with algorithm: table_merged_details
 118     specified in the specification file.
 119
 120     :param table: Table to generate.
 121     :param input_data: Data to process.
 122     :type table: pandas.Series
 123     :type input_data: InputData
 124     """
 125
 126     logging.info("  Generating the table {0} ...".
 127                  format(table.get("title", "")))
 128
 129     # Transform the data
 130     data = input_data.filter_data(table)
 131     data = input_data.merge_data(data)
 132     data.sort_index(inplace=True)
 133
 134     suites = input_data.filter_data(table, data_set="suites")
 135     suites = input_data.merge_data(suites)
 136
 137     # Prepare the header of the tables
 138     header = list()
 139     for column in table["columns"]:
 140         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 141
 142     for _, suite in suites.iteritems():
 143         # Generate data
 144         suite_name = suite["name"]
 145         table_lst = list()
 146         for test in data.keys():
 147             if data[test]["parent"] in suite_name:
 148                 row_lst = list()
 149                 for column in table["columns"]:
 150                     try:
 151                         col_data = str(data[test][column["data"].
 152                                        split(" ")[1]]).replace('"', '""')
 153                         if column["data"].split(" ")[1] in ("vat-history",
 154                                                             "show-run"):
 155                             col_data = replace(col_data, " |br| ", "",
 156                                                maxreplace=1)
 157                             col_data = " |prein| {0} |preout| ".\
 158                                 format(col_data[:-5])
 159                         row_lst.append('"{0}"'.format(col_data))
 160                     except KeyError:
 161                         row_lst.append("No data")
 162                 table_lst.append(row_lst)
 163
 164         # Write the data to file
 165         if table_lst:
 166             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 167                                             table["output-file-ext"])
 168             logging.info("      Writing file: '{}'".format(file_name))
 169             with open(file_name, "w") as file_handler:
 170                 file_handler.write(",".join(header) + "\n")
 171                 for item in table_lst:
 172                     file_handler.write(",".join(item) + "\n")
 173
 174     logging.info("  Done.")
 175
 176
 177 def table_performance_improvements(table, input_data):
 178     """Generate the table(s) with algorithm: table_performance_improvements
 179     specified in the specification file.
 180
 181     :param table: Table to generate.
 182     :param input_data: Data to process.
 183     :type table: pandas.Series
 184     :type input_data: InputData
 185     """
 186
 187     def _write_line_to_file(file_handler, data):
 188         """Write a line to the .csv file.
 189
 190         :param file_handler: File handler for the csv file. It must be open for
 191          writing text.
 192         :param data: Item to be written to the file.
 193         :type file_handler: BinaryIO
 194         :type data: list
 195         """
 196
 197         line_lst = list()
 198         for item in data:
 199             if isinstance(item["data"], str):
 200                 # Remove -?drdisc from the end
 201                 if item["data"].endswith("drdisc"):
 202                     item["data"] = item["data"][:-8]
 203                 line_lst.append(item["data"])
 204             elif isinstance(item["data"], float):
 205                 line_lst.append("{:.1f}".format(item["data"]))
 206             elif item["data"] is None:
 207                 line_lst.append("")
 208         file_handler.write(",".join(line_lst) + "\n")
 209
 210     logging.info("  Generating the table {0} ...".
 211                  format(table.get("title", "")))
 212
 213     # Read the template
 214     file_name = table.get("template", None)
 215     if file_name:
 216         try:
 217             tmpl = _read_csv_template(file_name)
 218         except PresentationError:
 219             logging.error("  The template '{0}' does not exist. Skipping the "
 220                           "table.".format(file_name))
 221             return None
 222     else:
 223         logging.error("The template is not defined. Skipping the table.")
 224         return None
 225
 226     # Transform the data
 227     data = input_data.filter_data(table)
 228
 229     # Prepare the header of the tables
 230     header = list()
 231     for column in table["columns"]:
 232         header.append(column["title"])
 233
 234     # Generate the data for the table according to the model in the table
 235     # specification
 236     tbl_lst = list()
 237     for tmpl_item in tmpl:
 238         tbl_item = list()
 239         for column in table["columns"]:
 240             cmd = column["data"].split(" ")[0]
 241             args = column["data"].split(" ")[1:]
 242             if cmd == "template":
 243                 try:
 244                     val = float(tmpl_item[int(args[0])])
 245                 except ValueError:
 246                     val = tmpl_item[int(args[0])]
 247                 tbl_item.append({"data": val})
 248             elif cmd == "data":
 249                 jobs = args[0:-1]
 250                 operation = args[-1]
 251                 data_lst = list()
 252                 for job in jobs:
 253                     for build in data[job]:
 254                         try:
 255                             data_lst.append(float(build[tmpl_item[0]]
 256                                                   ["throughput"]["value"]))
 257                         except (KeyError, TypeError):
 258                             # No data, ignore
 259                             continue
 260                 if data_lst:
 261                     tbl_item.append({"data": (eval(operation)(data_lst)) /
 262                                              1000000})
 263                 else:
 264                     tbl_item.append({"data": None})
 265             elif cmd == "operation":
 266                 operation = args[0]
 267                 try:
 268                     nr1 = float(tbl_item[int(args[1])]["data"])
 269                     nr2 = float(tbl_item[int(args[2])]["data"])
 270                     if nr1 and nr2:
 271                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
 272                     else:
 273                         tbl_item.append({"data": None})
 274                 except (IndexError, ValueError, TypeError):
 275                     logging.error("No data for {0}".format(tbl_item[0]["data"]))
 276                     tbl_item.append({"data": None})
 277                     continue
 278             else:
 279                 logging.error("Not supported command {0}. Skipping the table.".
 280                               format(cmd))
 281                 return None
 282         tbl_lst.append(tbl_item)
 283
 284     # Sort the table according to the relative change
 285     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
 286
 287     # Create the tables and write them to the files
 288     file_names = [
 289         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
 290         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
 291         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
 292         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
 293     ]
 294
 295     for file_name in file_names:
 296         logging.info("    Writing the file '{0}'".format(file_name))
 297         with open(file_name, "w") as file_handler:
 298             file_handler.write(",".join(header) + "\n")
 299             for item in tbl_lst:
 300                 if isinstance(item[-1]["data"], float):
 301                     rel_change = round(item[-1]["data"], 1)
 302                 else:
 303                     rel_change = item[-1]["data"]
 304                 if "ndr_top" in file_name \
 305                         and "ndr" in item[0]["data"] \
 306                         and rel_change >= 10.0:
 307                     _write_line_to_file(file_handler, item)
 308                 elif "pdr_top" in file_name \
 309                         and "pdr" in item[0]["data"] \
 310                         and rel_change >= 10.0:
 311                     _write_line_to_file(file_handler, item)
 312                 elif "ndr_low" in file_name \
 313                         and "ndr" in item[0]["data"] \
 314                         and rel_change < 10.0:
 315                     _write_line_to_file(file_handler, item)
 316                 elif "pdr_low" in file_name \
 317                         and "pdr" in item[0]["data"] \
 318                         and rel_change < 10.0:
 319                     _write_line_to_file(file_handler, item)
 320
 321     logging.info("  Done.")
 322
 323
 324 def _read_csv_template(file_name):
 325     """Read the template from a .csv file.
 326
 327     :param file_name: Name / full path / relative path of the file to read.
 328     :type file_name: str
 329     :returns: Data from the template as list (lines) of lists (items on line).
 330     :rtype: list
 331     :raises: PresentationError if it is not possible to read the file.
 332     """
 333
 334     try:
 335         with open(file_name, 'r') as csv_file:
 336             tmpl_data = list()
 337             for line in csv_file:
 338                 tmpl_data.append(line[:-1].split(","))
 339         return tmpl_data
 340     except IOError as err:
 341         raise PresentationError(str(err), level="ERROR")
 342
 343
 344 def table_performance_comparison(table, input_data):
 345     """Generate the table(s) with algorithm: table_performance_comparison
 346     specified in the specification file.
 347
 348     :param table: Table to generate.
 349     :param input_data: Data to process.
 350     :type table: pandas.Series
 351     :type input_data: InputData
 352     """
 353
 354     logging.info("  Generating the table {0} ...".
 355                  format(table.get("title", "")))
 356
 357     # Transform the data
 358     data = input_data.filter_data(table, continue_on_error=True)
 359
 360     # Prepare the header of the tables
 361     try:
 362         header = ["Test case",
 363                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 364                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 365                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 366                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 367                   "Change [%]"]
 368         header_str = ",".join(header) + "\n"
 369     except (AttributeError, KeyError) as err:
 370         logging.error("The model is invalid, missing parameter: {0}".
 371                       format(err))
 372         return
 373
 374     # Prepare data to the table:
 375     tbl_dict = dict()
 376     for job, builds in table["reference"]["data"].items():
 377         for build in builds:
 378             for tst_name, tst_data in data[job][str(build)].iteritems():
 379                 if tbl_dict.get(tst_name, None) is None:
 380                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 381                                             "-".join(tst_data["name"].
 382                                                      split("-")[1:]))
 383                     tbl_dict[tst_name] = {"name": name,
 384                                           "ref-data": list(),
 385                                           "cmp-data": list()}
 386                 try:
 387                     tbl_dict[tst_name]["ref-data"].\
 388                         append(tst_data["throughput"]["value"])
 389                 except TypeError:
 390                     pass  # No data in output.xml for this test
 391
 392     for job, builds in table["compare"]["data"].items():
 393         for build in builds:
 394             for tst_name, tst_data in data[job][str(build)].iteritems():
 395                 try:
 396                     tbl_dict[tst_name]["cmp-data"].\
 397                         append(tst_data["throughput"]["value"])
 398                 except KeyError:
 399                     pass
 400                 except TypeError:
 401                     tbl_dict.pop(tst_name, None)
 402
 403     tbl_lst = list()
 404     for tst_name in tbl_dict.keys():
 405         item = [tbl_dict[tst_name]["name"], ]
 406         if tbl_dict[tst_name]["ref-data"]:
 407             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
 408                                      table["outlier-const"])
 409             item.append(round(mean(data_t) / 1000000, 2))
 410             item.append(round(stdev(data_t) / 1000000, 2))
 411         else:
 412             item.extend([None, None])
 413         if tbl_dict[tst_name]["cmp-data"]:
 414             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
 415                                      table["outlier-const"])
 416             item.append(round(mean(data_t) / 1000000, 2))
 417             item.append(round(stdev(data_t) / 1000000, 2))
 418         else:
 419             item.extend([None, None])
 420         if item[1] is not None and item[3] is not None:
 421             item.append(int(relative_change(float(item[1]), float(item[3]))))
 422         if len(item) == 6:
 423             tbl_lst.append(item)
 424
 425     # Sort the table according to the relative change
 426     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 427
 428     # Generate tables:
 429     # All tests in csv:
 430     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
 431                                                table["output-file-ext"]),
 432                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
 433                                                table["output-file-ext"]),
 434                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
 435                                                table["output-file-ext"]),
 436                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 437                                                table["output-file-ext"]),
 438                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
 439                                                table["output-file-ext"]),
 440                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
 441                                                table["output-file-ext"])
 442                  ]
 443     for file_name in tbl_names:
 444         logging.info("      Writing file: '{0}'".format(file_name))
 445         with open(file_name, "w") as file_handler:
 446             file_handler.write(header_str)
 447             for test in tbl_lst:
 448                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
 449                         file_name.split("-")[-2] in test[0]):  # cores
 450                     test[0] = "-".join(test[0].split("-")[:-1])
 451                     file_handler.write(",".join([str(item) for item in test]) +
 452                                        "\n")
 453
 454     # All tests in txt:
 455     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
 456                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
 457                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
 458                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
 459                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
 460                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
 461                      ]
 462
 463     for i, txt_name in enumerate(tbl_names_txt):
 464         txt_table = None
 465         logging.info("      Writing file: '{0}'".format(txt_name))
 466         with open(tbl_names[i], 'rb') as csv_file:
 467             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 468             for row in csv_content:
 469                 if txt_table is None:
 470                     txt_table = prettytable.PrettyTable(row)
 471                 else:
 472                     txt_table.add_row(row)
 473             txt_table.align["Test case"] = "l"
 474         with open(txt_name, "w") as txt_file:
 475             txt_file.write(str(txt_table))
 476
 477     # Selected tests in csv:
 478     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
 479                                                table["output-file-ext"])
 480     with open(input_file, "r") as in_file:
 481         lines = list()
 482         for line in in_file:
 483             lines.append(line)
 484
 485     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
 486                                                table["output-file-ext"])
 487     logging.info("      Writing file: '{0}'".format(output_file))
 488     with open(output_file, "w") as out_file:
 489         out_file.write(header_str)
 490         for i, line in enumerate(lines[1:]):
 491             if i == table["nr-of-tests-shown"]:
 492                 break
 493             out_file.write(line)
 494
 495     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
 496                                                   table["output-file-ext"])
 497     logging.info("      Writing file: '{0}'".format(output_file))
 498     with open(output_file, "w") as out_file:
 499         out_file.write(header_str)
 500         for i, line in enumerate(lines[-1:0:-1]):
 501             if i == table["nr-of-tests-shown"]:
 502                 break
 503             out_file.write(line)
 504
 505     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 506                                                table["output-file-ext"])
 507     with open(input_file, "r") as in_file:
 508         lines = list()
 509         for line in in_file:
 510             lines.append(line)
 511
 512     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
 513                                                table["output-file-ext"])
 514     logging.info("      Writing file: '{0}'".format(output_file))
 515     with open(output_file, "w") as out_file:
 516         out_file.write(header_str)
 517         for i, line in enumerate(lines[1:]):
 518             if i == table["nr-of-tests-shown"]:
 519                 break
 520             out_file.write(line)
 521
 522     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
 523                                                   table["output-file-ext"])
 524     logging.info("      Writing file: '{0}'".format(output_file))
 525     with open(output_file, "w") as out_file:
 526         out_file.write(header_str)
 527         for i, line in enumerate(lines[-1:0:-1]):
 528             if i == table["nr-of-tests-shown"]:
 529                 break
 530             out_file.write(line)
 531
 532
 533 def table_performance_comparison_mrr(table, input_data):
 534     """Generate the table(s) with algorithm: table_performance_comparison_mrr
 535     specified in the specification file.
 536
 537     :param table: Table to generate.
 538     :param input_data: Data to process.
 539     :type table: pandas.Series
 540     :type input_data: InputData
 541     """
 542
 543     logging.info("  Generating the table {0} ...".
 544                  format(table.get("title", "")))
 545
 546     # Transform the data
 547     data = input_data.filter_data(table, continue_on_error=True)
 548
 549     # Prepare the header of the tables
 550     try:
 551         header = ["Test case",
 552                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 553                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 554                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 555                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 556                   "Change [%]"]
 557         header_str = ",".join(header) + "\n"
 558     except (AttributeError, KeyError) as err:
 559         logging.error("The model is invalid, missing parameter: {0}".
 560                       format(err))
 561         return
 562
 563     # Prepare data to the table:
 564     tbl_dict = dict()
 565     for job, builds in table["reference"]["data"].items():
 566         for build in builds:
 567             for tst_name, tst_data in data[job][str(build)].iteritems():
 568                 if tbl_dict.get(tst_name, None) is None:
 569                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 570                                             "-".join(tst_data["name"].
 571                                                      split("-")[1:]))
 572                     tbl_dict[tst_name] = {"name": name,
 573                                           "ref-data": list(),
 574                                           "cmp-data": list()}
 575                 try:
 576                     tbl_dict[tst_name]["ref-data"].\
 577                         append(tst_data["result"]["throughput"])
 578                 except TypeError:
 579                     pass  # No data in output.xml for this test
 580
 581     for job, builds in table["compare"]["data"].items():
 582         for build in builds:
 583             for tst_name, tst_data in data[job][str(build)].iteritems():
 584                 try:
 585                     tbl_dict[tst_name]["cmp-data"].\
 586                         append(tst_data["result"]["throughput"])
 587                 except KeyError:
 588                     pass
 589                 except TypeError:
 590                     tbl_dict.pop(tst_name, None)
 591
 592     tbl_lst = list()
 593     for tst_name in tbl_dict.keys():
 594         item = [tbl_dict[tst_name]["name"], ]
 595         if tbl_dict[tst_name]["ref-data"]:
 596             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
 597                                      table["outlier-const"])
 598             item.append(round(mean(data_t) / 1000000, 2))
 599             item.append(round(stdev(data_t) / 1000000, 2))
 600         else:
 601             item.extend([None, None])
 602         if tbl_dict[tst_name]["cmp-data"]:
 603             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
 604                                      table["outlier-const"])
 605             item.append(round(mean(data_t) / 1000000, 2))
 606             item.append(round(stdev(data_t) / 1000000, 2))
 607         else:
 608             item.extend([None, None])
 609         if item[1] is not None and item[3] is not None and item[1] != 0:
 610             item.append(int(relative_change(float(item[1]), float(item[3]))))
 611         if len(item) == 6:
 612             tbl_lst.append(item)
 613
 614     # Sort the table according to the relative change
 615     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 616
 617     # Generate tables:
 618     # All tests in csv:
 619     tbl_names = ["{0}-1t1c-full{1}".format(table["output-file"],
 620                                            table["output-file-ext"]),
 621                  "{0}-2t2c-full{1}".format(table["output-file"],
 622                                            table["output-file-ext"]),
 623                  "{0}-4t4c-full{1}".format(table["output-file"],
 624                                            table["output-file-ext"])
 625                  ]
 626     for file_name in tbl_names:
 627         logging.info("      Writing file: '{0}'".format(file_name))
 628         with open(file_name, "w") as file_handler:
 629             file_handler.write(header_str)
 630             for test in tbl_lst:
 631                 if file_name.split("-")[-2] in test[0]:  # cores
 632                     test[0] = "-".join(test[0].split("-")[:-1])
 633                     file_handler.write(",".join([str(item) for item in test]) +
 634                                        "\n")
 635
 636     # All tests in txt:
 637     tbl_names_txt = ["{0}-1t1c-full.txt".format(table["output-file"]),
 638                      "{0}-2t2c-full.txt".format(table["output-file"]),
 639                      "{0}-4t4c-full.txt".format(table["output-file"])
 640                      ]
 641
 642     for i, txt_name in enumerate(tbl_names_txt):
 643         txt_table = None
 644         logging.info("      Writing file: '{0}'".format(txt_name))
 645         with open(tbl_names[i], 'rb') as csv_file:
 646             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 647             for row in csv_content:
 648                 if txt_table is None:
 649                     txt_table = prettytable.PrettyTable(row)
 650                 else:
 651                     txt_table.add_row(row)
 652             txt_table.align["Test case"] = "l"
 653         with open(txt_name, "w") as txt_file:
 654             txt_file.write(str(txt_table))
 655
 656
 657 def table_performance_trending_dashboard(table, input_data):
 658     """Generate the table(s) with algorithm: table_performance_comparison
 659     specified in the specification file.
 660
 661     :param table: Table to generate.
 662     :param input_data: Data to process.
 663     :type table: pandas.Series
 664     :type input_data: InputData
 665     """
 666
 667     logging.info("  Generating the table {0} ...".
 668                  format(table.get("title", "")))
 669
 670     # Transform the data
 671     data = input_data.filter_data(table, continue_on_error=True)
 672
 673     # Prepare the header of the tables
 674     header = ["Test Case",
 675               "Throughput Trend [Mpps]",
 676               "Trend Compliance",
 677               "Top Anomaly [Mpps]",
 678               "Change [%]",
 679               "Outliers [Number]"
 680               ]
 681     header_str = ",".join(header) + "\n"
 682
 683     # Prepare data to the table:
 684     tbl_dict = dict()
 685     for job, builds in table["data"].items():
 686         for build in builds:
 687             for tst_name, tst_data in data[job][str(build)].iteritems():
 688                 if tbl_dict.get(tst_name, None) is None:
 689                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 690                                             "-".join(tst_data["name"].
 691                                                      split("-")[1:]))
 692                     tbl_dict[tst_name] = {"name": name,
 693                                           "data": dict()}
 694                 try:
 695                     tbl_dict[tst_name]["data"][str(build)] =  \
 696                         tst_data["result"]["throughput"]
 697                 except (TypeError, KeyError):
 698                     pass  # No data in output.xml for this test
 699
 700     tbl_lst = list()
 701     for tst_name in tbl_dict.keys():
 702         if len(tbl_dict[tst_name]["data"]) > 2:
 703
 704             pd_data = pd.Series(tbl_dict[tst_name]["data"])
 705             win_size = pd_data.size \
 706                 if pd_data.size < table["window"] else table["window"]
 707             # Test name:
 708             name = tbl_dict[tst_name]["name"]
 709
 710             median = pd_data.rolling(window=win_size, min_periods=2).median()
 711             trimmed_data, _ = find_outliers(pd_data, outlier_const=1.5)
 712             stdev_t = pd_data.rolling(window=win_size, min_periods=2).std()
 713
 714             rel_change_lst = [None, ]
 715             classification_lst = [None, ]
 716             median_lst = [None, ]
 717             sample_lst = [None, ]
 718             first = True
 719             for build_nr, value in pd_data.iteritems():
 720                 if first:
 721                     first = False
 722                     continue
 723                 # Relative changes list:
 724                 if not isnan(value) \
 725                         and not isnan(median[build_nr]) \
 726                         and median[build_nr] != 0:
 727                     rel_change_lst.append(round(
 728                         relative_change(float(median[build_nr]), float(value)),
 729                         2))
 730                 else:
 731                     rel_change_lst.append(None)
 732
 733                 # Classification list:
 734                 if isnan(trimmed_data[build_nr]) \
 735                         or isnan(median[build_nr]) \
 736                         or isnan(stdev_t[build_nr]) \
 737                         or isnan(value):
 738                     classification_lst.append("outlier")
 739                 elif value < (median[build_nr] - 3 * stdev_t[build_nr]):
 740                     classification_lst.append("regression")
 741                 elif value > (median[build_nr] + 3 * stdev_t[build_nr]):
 742                     classification_lst.append("progression")
 743                 else:
 744                     classification_lst.append("normal")
 745                 sample_lst.append(value)
 746                 median_lst.append(median[build_nr])
 747
 748             last_idx = len(classification_lst) - 1
 749             first_idx = last_idx - int(table["evaluated-window"])
 750             if first_idx < 0:
 751                 first_idx = 0
 752
 753             nr_outliers = 0
 754             consecutive_outliers = 0
 755             failure = False
 756             for item in classification_lst[first_idx:]:
 757                 if item == "outlier":
 758                     nr_outliers += 1
 759                     consecutive_outliers += 1
 760                     if consecutive_outliers == 3:
 761                         failure = True
 762                 else:
 763                     consecutive_outliers = 0
 764
 765             if failure:
 766                 classification = "failure"
 767             elif "regression" in classification_lst[first_idx:]:
 768                 classification = "regression"
 769             elif "progression" in classification_lst[first_idx:]:
 770                 classification = "progression"
 771             else:
 772                 classification = "normal"
 773
 774             if classification == "normal":
 775                 index = len(classification_lst) - 1
 776             else:
 777                 tmp_classification = "outlier" if classification == "failure" \
 778                     else classification
 779                 for idx in range(first_idx, len(classification_lst)):
 780                     if classification_lst[idx] == tmp_classification:
 781                         index = idx
 782                         break
 783                 for idx in range(index+1, len(classification_lst)):
 784                     if classification_lst[idx] == tmp_classification:
 785                         if rel_change_lst[idx] > rel_change_lst[index]:
 786                             index = idx
 787
 788             # if "regression" in classification_lst[first_idx:]:
 789             #     classification = "regression"
 790             # elif "outlier" in classification_lst[first_idx:]:
 791             #     classification = "outlier"
 792             # elif "progression" in classification_lst[first_idx:]:
 793             #     classification = "progression"
 794             # elif "normal" in classification_lst[first_idx:]:
 795             #     classification = "normal"
 796             # else:
 797             #     classification = None
 798             #
 799             # nr_outliers = 0
 800             # consecutive_outliers = 0
 801             # failure = False
 802             # for item in classification_lst[first_idx:]:
 803             #     if item == "outlier":
 804             #         nr_outliers += 1
 805             #         consecutive_outliers += 1
 806             #         if consecutive_outliers == 3:
 807             #             failure = True
 808             #     else:
 809             #         consecutive_outliers = 0
 810             #
 811             # idx = len(classification_lst) - 1
 812             # while idx:
 813             #     if classification_lst[idx] == classification:
 814             #         break
 815             #     idx -= 1
 816             #
 817             # if failure:
 818             #     classification = "failure"
 819             # elif classification == "outlier":
 820             #     classification = "normal"
 821
 822             trend = round(float(median_lst[-1]) / 1000000, 2) \
 823                 if not isnan(median_lst[-1]) else ''
 824             sample = round(float(sample_lst[index]) / 1000000, 2) \
 825                 if not isnan(sample_lst[index]) else ''
 826             rel_change = rel_change_lst[index] \
 827                 if rel_change_lst[index] is not None else ''
 828             tbl_lst.append([name,
 829                             trend,
 830                             classification,
 831                             '-' if classification == "normal" else sample,
 832                             '-' if classification == "normal" else rel_change,
 833                             nr_outliers])
 834
 835     # Sort the table according to the classification
 836     tbl_sorted = list()
 837     for classification in ("failure", "regression", "progression", "normal"):
 838         tbl_tmp = [item for item in tbl_lst if item[2] == classification]
 839         tbl_tmp.sort(key=lambda rel: rel[0])
 840         tbl_sorted.extend(tbl_tmp)
 841
 842     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
 843
 844     logging.info("      Writing file: '{0}'".format(file_name))
 845     with open(file_name, "w") as file_handler:
 846         file_handler.write(header_str)
 847         for test in tbl_sorted:
 848             file_handler.write(",".join([str(item) for item in test]) + '\n')
 849
 850     txt_file_name = "{0}.txt".format(table["output-file"])
 851     txt_table = None
 852     logging.info("      Writing file: '{0}'".format(txt_file_name))
 853     with open(file_name, 'rb') as csv_file:
 854         csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 855         for row in csv_content:
 856             if txt_table is None:
 857                 txt_table = prettytable.PrettyTable(row)
 858             else:
 859                 txt_table.add_row(row)
 860         txt_table.align["Test case"] = "l"
 861     with open(txt_file_name, "w") as txt_file:
 862         txt_file.write(str(txt_table))
 863
 864
 865 def table_performance_trending_dashboard_html(table, input_data):
 866     """Generate the table(s) with algorithm:
 867     table_performance_trending_dashboard_html specified in the specification
 868     file.
 869
 870     :param table: Table to generate.
 871     :param input_data: Data to process.
 872     :type table: pandas.Series
 873     :type input_data: InputData
 874     """
 875
 876     logging.info("  Generating the table {0} ...".
 877                  format(table.get("title", "")))
 878
 879     try:
 880         with open(table["input-file"], 'rb') as csv_file:
 881             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 882             csv_lst = [item for item in csv_content]
 883     except KeyError:
 884         logging.warning("The input file is not defined.")
 885         return
 886     except csv.Error as err:
 887         logging.warning("Not possible to process the file '{0}'.\n{1}".
 888                         format(table["input-file"], err))
 889         return
 890
 891     # Table:
 892     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
 893
 894     # Table header:
 895     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
 896     for idx, item in enumerate(csv_lst[0]):
 897         alignment = "left" if idx == 0 else "center"
 898         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
 899         th.text = item
 900
 901     # Rows:
 902     for r_idx, row in enumerate(csv_lst[1:]):
 903         background = "#D4E4F7" if r_idx % 2 else "white"
 904         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
 905
 906         # Columns:
 907         for c_idx, item in enumerate(row):
 908             alignment = "left" if c_idx == 0 else "center"
 909             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
 910             # Name:
 911             url = "../trending/"
 912             file_name = ""
 913             anchor = "#"
 914             feature = ""
 915             if c_idx == 0:
 916                 if "memif" in item:
 917                     file_name = "container_memif.html"
 918
 919                 elif "vhost" in item:
 920                     if "l2xcbase" in item or "l2bdbasemaclrn" in item:
 921                         file_name = "vm_vhost_l2.html"
 922                     elif "ip4base" in item:
 923                         file_name = "vm_vhost_ip4.html"
 924
 925                 elif "ipsec" in item:
 926                     file_name = "ipsec.html"
 927
 928                 elif "ethip4lispip" in item or "ethip4vxlan" in item:
 929                     file_name = "ip4_tunnels.html"
 930
 931                 elif "ip4base" in item or "ip4scale" in item:
 932                     file_name = "ip4.html"
 933                     if "iacl" in item or "snat" in item or "cop" in item:
 934                         feature = "-features"
 935
 936                 elif "ip6base" in item or "ip6scale" in item:
 937                     file_name = "ip6.html"
 938
 939                 elif "l2xcbase" in item or "l2xcscale" in item \
 940                         or "l2bdbasemaclrn" in item or "l2bdscale" in item \
 941                         or "l2dbbasemaclrn" in item or "l2dbscale" in item:
 942                     file_name = "l2.html"
 943                     if "iacl" in item:
 944                         feature = "-features"
 945
 946                 if "x520" in item:
 947                     anchor += "x520-"
 948                 elif "x710" in item:
 949                     anchor += "x710-"
 950                 elif "xl710" in item:
 951                     anchor += "xl710-"
 952
 953                 if "64b" in item:
 954                     anchor += "64b-"
 955                 elif "78b" in item:
 956                     anchor += "78b"
 957                 elif "imix" in item:
 958                     anchor += "imix-"
 959                 elif "9000b" in item:
 960                     anchor += "9000b-"
 961                 elif "1518" in item:
 962                     anchor += "1518b-"
 963
 964                 if "1t1c" in item:
 965                     anchor += "1t1c"
 966                 elif "2t2c" in item:
 967                     anchor += "2t2c"
 968                 elif "4t4c" in item:
 969                     anchor += "4t4c"
 970
 971                 url = url + file_name + anchor + feature
 972
 973                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
 974                 ref.text = item
 975
 976             if c_idx == 2:
 977                 if item == "regression":
 978                     td.set("bgcolor", "#eca1a6")
 979                 elif item == "failure":
 980                     td.set("bgcolor", "#d6cbd3")
 981                 elif item == "progression":
 982                     td.set("bgcolor", "#bdcebe")
 983             if c_idx > 0:
 984                 td.text = item
 985
 986     try:
 987         with open(table["output-file"], 'w') as html_file:
 988             logging.info("      Writing file: '{0}'".
 989                          format(table["output-file"]))
 990             html_file.write(".. raw:: html\n\n\t")
 991             html_file.write(ET.tostring(dashboard))
 992             html_file.write("\n\t<p><br><br></p>\n")
 993     except KeyError:
 994         logging.warning("The output file is not defined.")
 995         return