resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import prettytable
  21
  22 from string import replace
  23 from pprint import pformat
  24
  25 from errors import PresentationError
  26 from utils import mean, stdev, relative_change
  27
  28
  29 def generate_tables(spec, data):
  30     """Generate all tables specified in the specification file.
  31
  32     :param spec: Specification read from the specification file.
  33     :param data: Data to process.
  34     :type spec: Specification
  35     :type data: InputData
  36     """
  37
  38     logging.info("Generating the tables ...")
  39     for table in spec.tables:
  40         try:
  41             eval(table["algorithm"])(table, data)
  42         except NameError:
  43             logging.error("The algorithm '{0}' is not defined.".
  44                           format(table["algorithm"]))
  45     logging.info("Done.")
  46
  47
  48 def table_details(table, input_data):
  49     """Generate the table(s) with algorithm: table_detailed_test_results
  50     specified in the specification file.
  51
  52     :param table: Table to generate.
  53     :param input_data: Data to process.
  54     :type table: pandas.Series
  55     :type input_data: InputData
  56     """
  57
  58     logging.info("  Generating the table {0} ...".
  59                  format(table.get("title", "")))
  60
  61     # Transform the data
  62     data = input_data.filter_data(table)
  63
  64     # Prepare the header of the tables
  65     header = list()
  66     for column in table["columns"]:
  67         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  68
  69     # Generate the data for the table according to the model in the table
  70     # specification
  71     job = table["data"].keys()[0]
  72     build = str(table["data"][job][0])
  73     try:
  74         suites = input_data.suites(job, build)
  75     except KeyError:
  76         logging.error("    No data available. The table will not be generated.")
  77         return
  78
  79     for suite_longname, suite in suites.iteritems():
  80         # Generate data
  81         suite_name = suite["name"]
  82         table_lst = list()
  83         for test in data[job][build].keys():
  84             if data[job][build][test]["parent"] in suite_name:
  85                 row_lst = list()
  86                 for column in table["columns"]:
  87                     try:
  88                         col_data = str(data[job][build][test][column["data"].
  89                                        split(" ")[1]]).replace('"', '""')
  90                         if column["data"].split(" ")[1] in ("vat-history",
  91                                                             "show-run"):
  92                             col_data = replace(col_data, " |br| ", "",
  93                                                maxreplace=1)
  94                             col_data = " |prein| {0} |preout| ".\
  95                                 format(col_data[:-5])
  96                         row_lst.append('"{0}"'.format(col_data))
  97                     except KeyError:
  98                         row_lst.append("No data")
  99                 table_lst.append(row_lst)
 100
 101         # Write the data to file
 102         if table_lst:
 103             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 104                                             table["output-file-ext"])
 105             logging.info("      Writing file: '{}'".format(file_name))
 106             with open(file_name, "w") as file_handler:
 107                 file_handler.write(",".join(header) + "\n")
 108                 for item in table_lst:
 109                     file_handler.write(",".join(item) + "\n")
 110
 111     logging.info("  Done.")
 112
 113
 114 def table_merged_details(table, input_data):
 115     """Generate the table(s) with algorithm: table_merged_details
 116     specified in the specification file.
 117
 118     :param table: Table to generate.
 119     :param input_data: Data to process.
 120     :type table: pandas.Series
 121     :type input_data: InputData
 122     """
 123
 124     logging.info("  Generating the table {0} ...".
 125                  format(table.get("title", "")))
 126
 127     # Transform the data
 128     data = input_data.filter_data(table)
 129     data = input_data.merge_data(data)
 130     data.sort_index(inplace=True)
 131
 132     suites = input_data.filter_data(table, data_set="suites")
 133     suites = input_data.merge_data(suites)
 134
 135     # Prepare the header of the tables
 136     header = list()
 137     for column in table["columns"]:
 138         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 139
 140     for _, suite in suites.iteritems():
 141         # Generate data
 142         suite_name = suite["name"]
 143         table_lst = list()
 144         for test in data.keys():
 145             if data[test]["parent"] in suite_name:
 146                 row_lst = list()
 147                 for column in table["columns"]:
 148                     try:
 149                         col_data = str(data[test][column["data"].
 150                                        split(" ")[1]]).replace('"', '""')
 151                         if column["data"].split(" ")[1] in ("vat-history",
 152                                                             "show-run"):
 153                             col_data = replace(col_data, " |br| ", "",
 154                                                maxreplace=1)
 155                             col_data = " |prein| {0} |preout| ".\
 156                                 format(col_data[:-5])
 157                         row_lst.append('"{0}"'.format(col_data))
 158                     except KeyError:
 159                         row_lst.append("No data")
 160                 table_lst.append(row_lst)
 161
 162         # Write the data to file
 163         if table_lst:
 164             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 165                                             table["output-file-ext"])
 166             logging.info("      Writing file: '{}'".format(file_name))
 167             with open(file_name, "w") as file_handler:
 168                 file_handler.write(",".join(header) + "\n")
 169                 for item in table_lst:
 170                     file_handler.write(",".join(item) + "\n")
 171
 172     logging.info("  Done.")
 173
 174
 175 def table_performance_improvements(table, input_data):
 176     """Generate the table(s) with algorithm: table_performance_improvements
 177     specified in the specification file.
 178
 179     :param table: Table to generate.
 180     :param input_data: Data to process.
 181     :type table: pandas.Series
 182     :type input_data: InputData
 183     """
 184
 185     def _write_line_to_file(file_handler, data):
 186         """Write a line to the .csv file.
 187
 188         :param file_handler: File handler for the csv file. It must be open for
 189          writing text.
 190         :param data: Item to be written to the file.
 191         :type file_handler: BinaryIO
 192         :type data: list
 193         """
 194
 195         line_lst = list()
 196         for item in data:
 197             if isinstance(item["data"], str):
 198                 line_lst.append(item["data"])
 199             elif isinstance(item["data"], float):
 200                 line_lst.append("{:.1f}".format(item["data"]))
 201             elif item["data"] is None:
 202                 line_lst.append("")
 203         file_handler.write(",".join(line_lst) + "\n")
 204
 205     logging.info("  Generating the table {0} ...".
 206                  format(table.get("title", "")))
 207
 208     # Read the template
 209     file_name = table.get("template", None)
 210     if file_name:
 211         try:
 212             tmpl = _read_csv_template(file_name)
 213         except PresentationError:
 214             logging.error("  The template '{0}' does not exist. Skipping the "
 215                           "table.".format(file_name))
 216             return None
 217     else:
 218         logging.error("The template is not defined. Skipping the table.")
 219         return None
 220
 221     # Transform the data
 222     data = input_data.filter_data(table)
 223
 224     # Prepare the header of the tables
 225     header = list()
 226     for column in table["columns"]:
 227         header.append(column["title"])
 228
 229     # Generate the data for the table according to the model in the table
 230     # specification
 231     tbl_lst = list()
 232     for tmpl_item in tmpl:
 233         tbl_item = list()
 234         for column in table["columns"]:
 235             cmd = column["data"].split(" ")[0]
 236             args = column["data"].split(" ")[1:]
 237             if cmd == "template":
 238                 try:
 239                     val = float(tmpl_item[int(args[0])])
 240                 except ValueError:
 241                     val = tmpl_item[int(args[0])]
 242                 tbl_item.append({"data": val})
 243             elif cmd == "data":
 244                 jobs = args[0:-1]
 245                 operation = args[-1]
 246                 data_lst = list()
 247                 for job in jobs:
 248                     for build in data[job]:
 249                         try:
 250                             data_lst.append(float(build[tmpl_item[0]]
 251                                                   ["throughput"]["value"]))
 252                         except (KeyError, TypeError):
 253                             # No data, ignore
 254                             continue
 255                 if data_lst:
 256                     tbl_item.append({"data": (eval(operation)(data_lst)) /
 257                                              1000000})
 258                 else:
 259                     tbl_item.append({"data": None})
 260             elif cmd == "operation":
 261                 operation = args[0]
 262                 try:
 263                     nr1 = float(tbl_item[int(args[1])]["data"])
 264                     nr2 = float(tbl_item[int(args[2])]["data"])
 265                     if nr1 and nr2:
 266                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
 267                     else:
 268                         tbl_item.append({"data": None})
 269                 except (IndexError, ValueError, TypeError):
 270                     logging.error("No data for {0}".format(tbl_item[1]["data"]))
 271                     tbl_item.append({"data": None})
 272                     continue
 273             else:
 274                 logging.error("Not supported command {0}. Skipping the table.".
 275                               format(cmd))
 276                 return None
 277         tbl_lst.append(tbl_item)
 278
 279     # Sort the table according to the relative change
 280     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
 281
 282     # Create the tables and write them to the files
 283     file_names = [
 284         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
 285         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
 286         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
 287         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
 288     ]
 289
 290     for file_name in file_names:
 291         logging.info("    Writing the file '{0}'".format(file_name))
 292         with open(file_name, "w") as file_handler:
 293             file_handler.write(",".join(header) + "\n")
 294             for item in tbl_lst:
 295                 if isinstance(item[-1]["data"], float):
 296                     rel_change = round(item[-1]["data"], 1)
 297                 else:
 298                     rel_change = item[-1]["data"]
 299                 if "ndr_top" in file_name \
 300                         and "ndr" in item[1]["data"] \
 301                         and rel_change >= 10.0:
 302                     _write_line_to_file(file_handler, item)
 303                 elif "pdr_top" in file_name \
 304                         and "pdr" in item[1]["data"] \
 305                         and rel_change >= 10.0:
 306                     _write_line_to_file(file_handler, item)
 307                 elif "ndr_low" in file_name \
 308                         and "ndr" in item[1]["data"] \
 309                         and rel_change < 10.0:
 310                     _write_line_to_file(file_handler, item)
 311                 elif "pdr_low" in file_name \
 312                         and "pdr" in item[1]["data"] \
 313                         and rel_change < 10.0:
 314                     _write_line_to_file(file_handler, item)
 315
 316     logging.info("  Done.")
 317
 318
 319 def _read_csv_template(file_name):
 320     """Read the template from a .csv file.
 321
 322     :param file_name: Name / full path / relative path of the file to read.
 323     :type file_name: str
 324     :returns: Data from the template as list (lines) of lists (items on line).
 325     :rtype: list
 326     :raises: PresentationError if it is not possible to read the file.
 327     """
 328
 329     try:
 330         with open(file_name, 'r') as csv_file:
 331             tmpl_data = list()
 332             for line in csv_file:
 333                 tmpl_data.append(line[:-1].split(","))
 334         return tmpl_data
 335     except IOError as err:
 336         raise PresentationError(str(err), level="ERROR")
 337
 338
 339 def table_performance_comparison(table, input_data):
 340     """Generate the table(s) with algorithm: table_performance_comparison
 341     specified in the specification file.
 342
 343     :param table: Table to generate.
 344     :param input_data: Data to process.
 345     :type table: pandas.Series
 346     :type input_data: InputData
 347     """
 348
 349     logging.info("  Generating the table {0} ...".
 350                  format(table.get("title", "")))
 351
 352     # Transform the data
 353     data = input_data.filter_data(table)
 354     logging.info(data)
 355
 356     # Prepare the header of the tables
 357     try:
 358         header = ["Test case",
 359                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 360                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 361                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 362                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 363                   "Change [%]"]
 364         header_str = ",".join(header) + "\n"
 365     except (AttributeError, KeyError) as err:
 366         logging.error("The model is invalid, missing parameter: {0}".
 367                       format(err))
 368         return
 369
 370     # Prepare data to the table:
 371     tbl_dict = dict()
 372     for job, builds in table["reference"]["data"].items():
 373         for build in builds:
 374             for tst_name, tst_data in data[job][str(build)].iteritems():
 375                 if tbl_dict.get(tst_name, None) is None:
 376                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 377                                             "-".join(tst_data["name"].
 378                                                      split("-")[1:]))
 379                     tbl_dict[tst_name] = {"name": name,
 380                                           "ref-data": list(),
 381                                           "cmp-data": list()}
 382                 try:
 383                     tbl_dict[tst_name]["ref-data"].\
 384                         append(tst_data["throughput"]["value"])
 385                 except TypeError:
 386                     pass  # No data in output.xml for this test
 387     logging.info(pformat(tbl_dict))
 388
 389     for job, builds in table["compare"]["data"].items():
 390         for build in builds:
 391             for tst_name, tst_data in data[job][str(build)].iteritems():
 392                 try:
 393                     tbl_dict[tst_name]["cmp-data"].\
 394                         append(tst_data["throughput"]["value"])
 395                 except KeyError:
 396                     pass
 397                 except TypeError:
 398                     tbl_dict.pop(tst_name, None)
 399
 400     logging.info(pformat(tbl_dict))
 401
 402     tbl_lst = list()
 403     for tst_name in tbl_dict.keys():
 404         item = [tbl_dict[tst_name]["name"], ]
 405         if tbl_dict[tst_name]["ref-data"]:
 406             item.append(round(mean(tbl_dict[tst_name]["ref-data"]) / 1000000,
 407                               2))
 408             item.append(round(stdev(tbl_dict[tst_name]["ref-data"]) / 1000000,
 409                               2))
 410         else:
 411             item.extend([None, None])
 412         if tbl_dict[tst_name]["cmp-data"]:
 413             item.append(round(mean(tbl_dict[tst_name]["cmp-data"]) / 1000000,
 414                               2))
 415             item.append(round(stdev(tbl_dict[tst_name]["cmp-data"]) / 1000000,
 416                               2))
 417         else:
 418             item.extend([None, None])
 419         if item[1] is not None and item[3] is not None:
 420             item.append(int(relative_change(float(item[1]), float(item[3]))))
 421         if len(item) == 6:
 422             tbl_lst.append(item)
 423
 424     # Sort the table according to the relative change
 425     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 426     logging.info(pformat(tbl_lst))
 427
 428     # Generate tables:
 429     # All tests in csv:
 430     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
 431                                                table["output-file-ext"]),
 432                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
 433                                                table["output-file-ext"]),
 434                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
 435                                                table["output-file-ext"]),
 436                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 437                                                table["output-file-ext"]),
 438                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
 439                                                table["output-file-ext"]),
 440                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
 441                                                table["output-file-ext"])
 442                  ]
 443     for file_name in tbl_names:
 444         logging.info("      Writing file: '{}'".format(file_name))
 445         with open(file_name, "w") as file_handler:
 446             file_handler.write(header_str)
 447             for test in tbl_lst:
 448                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
 449                         file_name.split("-")[-2] in test[0]):  # cores
 450                     test[0] = "-".join(test[0].split("-")[:-1])
 451                     file_handler.write(",".join([str(item) for item in test]) +
 452                                        "\n")
 453
 454     # All tests in txt:
 455     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
 456                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
 457                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
 458                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
 459                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
 460                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
 461                      ]
 462
 463     for i, txt_name in enumerate(tbl_names_txt):
 464         txt_table = None
 465         logging.info("      Writing file: '{}'".format(txt_name))
 466         with open(tbl_names[i], 'rb') as csv_file:
 467             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 468             for row in csv_content:
 469                 if txt_table is None:
 470                     txt_table = prettytable.PrettyTable(row)
 471                 else:
 472                     txt_table.add_row(row)
 473         with open(txt_name, "w") as txt_file:
 474             txt_file.write(str(txt_table))
 475
 476     # Selected tests in csv:
 477     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
 478                                                table["output-file-ext"])
 479     with open(input_file, "r") as in_file:
 480         lines = list()
 481         for line in in_file:
 482             lines.append(line)
 483
 484     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
 485                                                table["output-file-ext"])
 486     logging.info("      Writing file: '{}'".format(output_file))
 487     with open(output_file, "w") as out_file:
 488         out_file.write(header_str)
 489         for i, line in enumerate(lines[1:]):
 490             if i == table["nr-of-tests-shown"]:
 491                 break
 492             out_file.write(line)
 493
 494     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
 495                                                   table["output-file-ext"])
 496     logging.info("      Writing file: '{}'".format(output_file))
 497     with open(output_file, "w") as out_file:
 498         out_file.write(header_str)
 499         for i, line in enumerate(lines[-1:0:-1]):
 500             if i == table["nr-of-tests-shown"]:
 501                 break
 502             out_file.write(line)
 503
 504     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 505                                                table["output-file-ext"])
 506     with open(input_file, "r") as in_file:
 507         lines = list()
 508         for line in in_file:
 509             lines.append(line)
 510
 511     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
 512                                                table["output-file-ext"])
 513     logging.info("      Writing file: '{}'".format(output_file))
 514     with open(output_file, "w") as out_file:
 515         out_file.write(header_str)
 516         for i, line in enumerate(lines[1:]):
 517             if i == table["nr-of-tests-shown"]:
 518                 break
 519             out_file.write(line)
 520
 521     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
 522                                                   table["output-file-ext"])
 523     logging.info("      Writing file: '{}'".format(output_file))
 524     with open(output_file, "w") as out_file:
 525         out_file.write(header_str)
 526         for i, line in enumerate(lines[-1:0:-1]):
 527             if i == table["nr-of-tests-shown"]:
 528                 break
 529             out_file.write(line)