resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import prettytable
  21
  22 from string import replace
  23
  24 from errors import PresentationError
  25 from utils import mean, stdev, relative_change
  26
  27
  28 def generate_tables(spec, data):
  29     """Generate all tables specified in the specification file.
  30
  31     :param spec: Specification read from the specification file.
  32     :param data: Data to process.
  33     :type spec: Specification
  34     :type data: InputData
  35     """
  36
  37     logging.info("Generating the tables ...")
  38     for table in spec.tables:
  39         try:
  40             eval(table["algorithm"])(table, data)
  41         except NameError:
  42             logging.error("The algorithm '{0}' is not defined.".
  43                           format(table["algorithm"]))
  44     logging.info("Done.")
  45
  46
  47 def table_details(table, input_data):
  48     """Generate the table(s) with algorithm: table_detailed_test_results
  49     specified in the specification file.
  50
  51     :param table: Table to generate.
  52     :param input_data: Data to process.
  53     :type table: pandas.Series
  54     :type input_data: InputData
  55     """
  56
  57     logging.info("  Generating the table {0} ...".
  58                  format(table.get("title", "")))
  59
  60     # Transform the data
  61     data = input_data.filter_data(table)
  62
  63     # Prepare the header of the tables
  64     header = list()
  65     for column in table["columns"]:
  66         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  67
  68     # Generate the data for the table according to the model in the table
  69     # specification
  70     job = table["data"].keys()[0]
  71     build = str(table["data"][job][0])
  72     try:
  73         suites = input_data.suites(job, build)
  74     except KeyError:
  75         logging.error("    No data available. The table will not be generated.")
  76         return
  77
  78     for suite_longname, suite in suites.iteritems():
  79         # Generate data
  80         suite_name = suite["name"]
  81         table_lst = list()
  82         for test in data[job][build].keys():
  83             if data[job][build][test]["parent"] in suite_name:
  84                 row_lst = list()
  85                 for column in table["columns"]:
  86                     try:
  87                         col_data = str(data[job][build][test][column["data"].
  88                                        split(" ")[1]]).replace('"', '""')
  89                         if column["data"].split(" ")[1] in ("vat-history",
  90                                                             "show-run"):
  91                             col_data = replace(col_data, " |br| ", "",
  92                                                maxreplace=1)
  93                             col_data = " |prein| {0} |preout| ".\
  94                                 format(col_data[:-5])
  95                         row_lst.append('"{0}"'.format(col_data))
  96                     except KeyError:
  97                         row_lst.append("No data")
  98                 table_lst.append(row_lst)
  99
 100         # Write the data to file
 101         if table_lst:
 102             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 103                                             table["output-file-ext"])
 104             logging.info("      Writing file: '{}'".format(file_name))
 105             with open(file_name, "w") as file_handler:
 106                 file_handler.write(",".join(header) + "\n")
 107                 for item in table_lst:
 108                     file_handler.write(",".join(item) + "\n")
 109
 110     logging.info("  Done.")
 111
 112
 113 def table_merged_details(table, input_data):
 114     """Generate the table(s) with algorithm: table_merged_details
 115     specified in the specification file.
 116
 117     :param table: Table to generate.
 118     :param input_data: Data to process.
 119     :type table: pandas.Series
 120     :type input_data: InputData
 121     """
 122
 123     logging.info("  Generating the table {0} ...".
 124                  format(table.get("title", "")))
 125
 126     # Transform the data
 127     data = input_data.filter_data(table)
 128     data = input_data.merge_data(data)
 129     data.sort_index(inplace=True)
 130
 131     suites = input_data.filter_data(table, data_set="suites")
 132     suites = input_data.merge_data(suites)
 133
 134     # Prepare the header of the tables
 135     header = list()
 136     for column in table["columns"]:
 137         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 138
 139     for _, suite in suites.iteritems():
 140         # Generate data
 141         suite_name = suite["name"]
 142         table_lst = list()
 143         for test in data.keys():
 144             if data[test]["parent"] in suite_name:
 145                 row_lst = list()
 146                 for column in table["columns"]:
 147                     try:
 148                         col_data = str(data[test][column["data"].
 149                                        split(" ")[1]]).replace('"', '""')
 150                         if column["data"].split(" ")[1] in ("vat-history",
 151                                                             "show-run"):
 152                             col_data = replace(col_data, " |br| ", "",
 153                                                maxreplace=1)
 154                             col_data = " |prein| {0} |preout| ".\
 155                                 format(col_data[:-5])
 156                         row_lst.append('"{0}"'.format(col_data))
 157                     except KeyError:
 158                         row_lst.append("No data")
 159                 table_lst.append(row_lst)
 160
 161         # Write the data to file
 162         if table_lst:
 163             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 164                                             table["output-file-ext"])
 165             logging.info("      Writing file: '{}'".format(file_name))
 166             with open(file_name, "w") as file_handler:
 167                 file_handler.write(",".join(header) + "\n")
 168                 for item in table_lst:
 169                     file_handler.write(",".join(item) + "\n")
 170
 171     logging.info("  Done.")
 172
 173
 174 def table_performance_improvements(table, input_data):
 175     """Generate the table(s) with algorithm: table_performance_improvements
 176     specified in the specification file.
 177
 178     :param table: Table to generate.
 179     :param input_data: Data to process.
 180     :type table: pandas.Series
 181     :type input_data: InputData
 182     """
 183
 184     def _write_line_to_file(file_handler, data):
 185         """Write a line to the .csv file.
 186
 187         :param file_handler: File handler for the csv file. It must be open for
 188          writing text.
 189         :param data: Item to be written to the file.
 190         :type file_handler: BinaryIO
 191         :type data: list
 192         """
 193
 194         line_lst = list()
 195         for item in data:
 196             if isinstance(item["data"], str):
 197                 line_lst.append(item["data"])
 198             elif isinstance(item["data"], float):
 199                 line_lst.append("{:.1f}".format(item["data"]))
 200             elif item["data"] is None:
 201                 line_lst.append("")
 202         file_handler.write(",".join(line_lst) + "\n")
 203
 204     logging.info("  Generating the table {0} ...".
 205                  format(table.get("title", "")))
 206
 207     # Read the template
 208     file_name = table.get("template", None)
 209     if file_name:
 210         try:
 211             tmpl = _read_csv_template(file_name)
 212         except PresentationError:
 213             logging.error("  The template '{0}' does not exist. Skipping the "
 214                           "table.".format(file_name))
 215             return None
 216     else:
 217         logging.error("The template is not defined. Skipping the table.")
 218         return None
 219
 220     # Transform the data
 221     data = input_data.filter_data(table)
 222
 223     # Prepare the header of the tables
 224     header = list()
 225     for column in table["columns"]:
 226         header.append(column["title"])
 227
 228     # Generate the data for the table according to the model in the table
 229     # specification
 230     tbl_lst = list()
 231     for tmpl_item in tmpl:
 232         tbl_item = list()
 233         for column in table["columns"]:
 234             cmd = column["data"].split(" ")[0]
 235             args = column["data"].split(" ")[1:]
 236             if cmd == "template":
 237                 try:
 238                     val = float(tmpl_item[int(args[0])])
 239                 except ValueError:
 240                     val = tmpl_item[int(args[0])]
 241                 tbl_item.append({"data": val})
 242             elif cmd == "data":
 243                 jobs = args[0:-1]
 244                 operation = args[-1]
 245                 data_lst = list()
 246                 for job in jobs:
 247                     for build in data[job]:
 248                         try:
 249                             data_lst.append(float(build[tmpl_item[0]]
 250                                                   ["throughput"]["value"]))
 251                         except (KeyError, TypeError):
 252                             # No data, ignore
 253                             continue
 254                 if data_lst:
 255                     tbl_item.append({"data": (eval(operation)(data_lst)) /
 256                                              1000000})
 257                 else:
 258                     tbl_item.append({"data": None})
 259             elif cmd == "operation":
 260                 operation = args[0]
 261                 try:
 262                     nr1 = float(tbl_item[int(args[1])]["data"])
 263                     nr2 = float(tbl_item[int(args[2])]["data"])
 264                     if nr1 and nr2:
 265                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
 266                     else:
 267                         tbl_item.append({"data": None})
 268                 except (IndexError, ValueError, TypeError):
 269                     logging.error("No data for {0}".format(tbl_item[1]["data"]))
 270                     tbl_item.append({"data": None})
 271                     continue
 272             else:
 273                 logging.error("Not supported command {0}. Skipping the table.".
 274                               format(cmd))
 275                 return None
 276         tbl_lst.append(tbl_item)
 277
 278     # Sort the table according to the relative change
 279     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
 280
 281     # Create the tables and write them to the files
 282     file_names = [
 283         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
 284         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
 285         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
 286         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
 287     ]
 288
 289     for file_name in file_names:
 290         logging.info("    Writing the file '{0}'".format(file_name))
 291         with open(file_name, "w") as file_handler:
 292             file_handler.write(",".join(header) + "\n")
 293             for item in tbl_lst:
 294                 if isinstance(item[-1]["data"], float):
 295                     rel_change = round(item[-1]["data"], 1)
 296                 else:
 297                     rel_change = item[-1]["data"]
 298                 if "ndr_top" in file_name \
 299                         and "ndr" in item[1]["data"] \
 300                         and rel_change >= 10.0:
 301                     _write_line_to_file(file_handler, item)
 302                 elif "pdr_top" in file_name \
 303                         and "pdr" in item[1]["data"] \
 304                         and rel_change >= 10.0:
 305                     _write_line_to_file(file_handler, item)
 306                 elif "ndr_low" in file_name \
 307                         and "ndr" in item[1]["data"] \
 308                         and rel_change < 10.0:
 309                     _write_line_to_file(file_handler, item)
 310                 elif "pdr_low" in file_name \
 311                         and "pdr" in item[1]["data"] \
 312                         and rel_change < 10.0:
 313                     _write_line_to_file(file_handler, item)
 314
 315     logging.info("  Done.")
 316
 317
 318 def _read_csv_template(file_name):
 319     """Read the template from a .csv file.
 320
 321     :param file_name: Name / full path / relative path of the file to read.
 322     :type file_name: str
 323     :returns: Data from the template as list (lines) of lists (items on line).
 324     :rtype: list
 325     :raises: PresentationError if it is not possible to read the file.
 326     """
 327
 328     try:
 329         with open(file_name, 'r') as csv_file:
 330             tmpl_data = list()
 331             for line in csv_file:
 332                 tmpl_data.append(line[:-1].split(","))
 333         return tmpl_data
 334     except IOError as err:
 335         raise PresentationError(str(err), level="ERROR")
 336
 337
 338 def table_performance_comparison(table, input_data):
 339     """Generate the table(s) with algorithm: table_performance_comparison
 340     specified in the specification file.
 341
 342     :param table: Table to generate.
 343     :param input_data: Data to process.
 344     :type table: pandas.Series
 345     :type input_data: InputData
 346     """
 347
 348     # Transform the data
 349     data = input_data.filter_data(table)
 350
 351     # Prepare the header of the tables
 352     try:
 353         header = ["Test case",
 354                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 355                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 356                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 357                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 358                   "Change [%]"]
 359         header_str = ",".join(header) + "\n"
 360     except (AttributeError, KeyError) as err:
 361         logging.error("The model is invalid, missing parameter: {0}".
 362                       format(err))
 363         return
 364
 365     # Prepare data to the table:
 366     tbl_dict = dict()
 367     for job, builds in table["reference"]["data"].items():
 368         for build in builds:
 369             for tst_name, tst_data in data[job][str(build)].iteritems():
 370                 if tbl_dict.get(tst_name, None) is None:
 371                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 372                                             "-".join(tst_data["name"].
 373                                                      split("-")[1:]))
 374                     tbl_dict[tst_name] = {"name": name,
 375                                           "ref-data": list(),
 376                                           "cmp-data": list()}
 377                 tbl_dict[tst_name]["ref-data"].\
 378                     append(tst_data["throughput"]["value"])
 379
 380     for job, builds in table["compare"]["data"].items():
 381         for build in builds:
 382             for tst_name, tst_data in data[job][str(build)].iteritems():
 383                 tbl_dict[tst_name]["cmp-data"].\
 384                     append(tst_data["throughput"]["value"])
 385
 386     tbl_lst = list()
 387     for tst_name in tbl_dict.keys():
 388         item = [tbl_dict[tst_name]["name"], ]
 389         if tbl_dict[tst_name]["ref-data"]:
 390             item.append(round(mean(tbl_dict[tst_name]["ref-data"]) / 1000000,
 391                               2))
 392             item.append(round(stdev(tbl_dict[tst_name]["ref-data"]) / 1000000,
 393                               2))
 394         else:
 395             item.extend([None, None])
 396         if tbl_dict[tst_name]["cmp-data"]:
 397             item.append(round(mean(tbl_dict[tst_name]["cmp-data"]) / 1000000,
 398                               2))
 399             item.append(round(stdev(tbl_dict[tst_name]["cmp-data"]) / 1000000,
 400                               2))
 401         else:
 402             item.extend([None, None])
 403         if item[1] is not None and item[3] is not None:
 404             item.append(int(relative_change(float(item[1]), float(item[3]))))
 405         if len(item) == 6:
 406             tbl_lst.append(item)
 407
 408     # Sort the table according to the relative change
 409     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 410
 411     # Generate tables:
 412     # All tests in csv:
 413     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
 414                                                table["output-file-ext"]),
 415                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
 416                                                table["output-file-ext"]),
 417                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
 418                                                table["output-file-ext"]),
 419                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 420                                                table["output-file-ext"]),
 421                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
 422                                                table["output-file-ext"]),
 423                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
 424                                                table["output-file-ext"])
 425                  ]
 426     for file_name in tbl_names:
 427         with open(file_name, "w") as file_handler:
 428             file_handler.write(header_str)
 429             for test in tbl_lst:
 430                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
 431                         file_name.split("-")[-2] in test[0]):  # cores
 432                     test[0] = "-".join(test[0].split("-")[:-1])
 433                     file_handler.write(",".join([str(item) for item in test]) +
 434                                        "\n")
 435
 436     # All tests in txt:
 437     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
 438                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
 439                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
 440                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
 441                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
 442                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
 443                      ]
 444
 445     for i, txt_name in enumerate(tbl_names_txt):
 446         txt_table = None
 447         with open(tbl_names[i], 'rb') as csv_file:
 448             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 449             for row in csv_content:
 450                 if txt_table is None:
 451                     txt_table = prettytable.PrettyTable(row)
 452                 else:
 453                     txt_table.add_row(row)
 454         with open(txt_name, "w") as txt_file:
 455             txt_file.write(str(txt_table))
 456
 457     # Selected tests in csv:
 458     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
 459                                                table["output-file-ext"])
 460     with open(input_file, "r") as in_file:
 461         lines = list()
 462         for line in in_file:
 463             lines.append(line)
 464
 465     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
 466                                                table["output-file-ext"])
 467     with open(output_file, "w") as out_file:
 468         out_file.write(header_str)
 469         for i, line in enumerate(lines[1:]):
 470             if i == table["nr-of-tests-shown"]:
 471                 break
 472             out_file.write(line)
 473
 474     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
 475                                                   table["output-file-ext"])
 476     with open(output_file, "w") as out_file:
 477         out_file.write(header_str)
 478         for i, line in enumerate(lines[-1:0:-1]):
 479             if i == table["nr-of-tests-shown"]:
 480                 break
 481             out_file.write(line)
 482
 483     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 484                                                table["output-file-ext"])
 485     with open(input_file, "r") as in_file:
 486         lines = list()
 487         for line in in_file:
 488             lines.append(line)
 489
 490     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
 491                                                table["output-file-ext"])
 492     with open(output_file, "w") as out_file:
 493         out_file.write(header_str)
 494         for i, line in enumerate(lines[1:]):
 495             if i == table["nr-of-tests-shown"]:
 496                 break
 497             out_file.write(line)
 498
 499     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
 500                                                   table["output-file-ext"])
 501     with open(output_file, "w") as out_file:
 502         out_file.write(header_str)
 503         for i, line in enumerate(lines[-1:0:-1]):
 504             if i == table["nr-of-tests-shown"]:
 505                 break
 506             out_file.write(line)