resources/tools/presentation/generator_tables.py

   1 # Copyright (c) 2017 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Algorithms to generate tables.
  15 """
  16
  17
  18 import logging
  19 import csv
  20 import prettytable
  21
  22 from string import replace
  23
  24 from errors import PresentationError
  25 from utils import mean, stdev, relative_change
  26
  27
  28 def generate_tables(spec, data):
  29     """Generate all tables specified in the specification file.
  30
  31     :param spec: Specification read from the specification file.
  32     :param data: Data to process.
  33     :type spec: Specification
  34     :type data: InputData
  35     """
  36
  37     logging.info("Generating the tables ...")
  38     for table in spec.tables:
  39         try:
  40             eval(table["algorithm"])(table, data)
  41         except NameError:
  42             logging.error("The algorithm '{0}' is not defined.".
  43                           format(table["algorithm"]))
  44     logging.info("Done.")
  45
  46
  47 def table_details(table, input_data):
  48     """Generate the table(s) with algorithm: table_detailed_test_results
  49     specified in the specification file.
  50
  51     :param table: Table to generate.
  52     :param input_data: Data to process.
  53     :type table: pandas.Series
  54     :type input_data: InputData
  55     """
  56
  57     logging.info("  Generating the table {0} ...".
  58                  format(table.get("title", "")))
  59
  60     # Transform the data
  61     data = input_data.filter_data(table)
  62
  63     # Prepare the header of the tables
  64     header = list()
  65     for column in table["columns"]:
  66         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
  67
  68     # Generate the data for the table according to the model in the table
  69     # specification
  70     job = table["data"].keys()[0]
  71     build = str(table["data"][job][0])
  72     try:
  73         suites = input_data.suites(job, build)
  74     except KeyError:
  75         logging.error("    No data available. The table will not be generated.")
  76         return
  77
  78     for suite_longname, suite in suites.iteritems():
  79         # Generate data
  80         suite_name = suite["name"]
  81         table_lst = list()
  82         for test in data[job][build].keys():
  83             if data[job][build][test]["parent"] in suite_name:
  84                 row_lst = list()
  85                 for column in table["columns"]:
  86                     try:
  87                         col_data = str(data[job][build][test][column["data"].
  88                                        split(" ")[1]]).replace('"', '""')
  89                         if column["data"].split(" ")[1] in ("vat-history",
  90                                                             "show-run"):
  91                             col_data = replace(col_data, " |br| ", "",
  92                                                maxreplace=1)
  93                             col_data = " |prein| {0} |preout| ".\
  94                                 format(col_data[:-5])
  95                         row_lst.append('"{0}"'.format(col_data))
  96                     except KeyError:
  97                         row_lst.append("No data")
  98                 table_lst.append(row_lst)
  99
 100         # Write the data to file
 101         if table_lst:
 102             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 103                                             table["output-file-ext"])
 104             logging.info("      Writing file: '{}'".format(file_name))
 105             with open(file_name, "w") as file_handler:
 106                 file_handler.write(",".join(header) + "\n")
 107                 for item in table_lst:
 108                     file_handler.write(",".join(item) + "\n")
 109
 110     logging.info("  Done.")
 111
 112
 113 def table_merged_details(table, input_data):
 114     """Generate the table(s) with algorithm: table_merged_details
 115     specified in the specification file.
 116
 117     :param table: Table to generate.
 118     :param input_data: Data to process.
 119     :type table: pandas.Series
 120     :type input_data: InputData
 121     """
 122
 123     logging.info("  Generating the table {0} ...".
 124                  format(table.get("title", "")))
 125
 126     # Transform the data
 127     data = input_data.filter_data(table)
 128     data = input_data.merge_data(data)
 129     data.sort_index(inplace=True)
 130
 131     suites = input_data.filter_data(table, data_set="suites")
 132     suites = input_data.merge_data(suites)
 133
 134     # Prepare the header of the tables
 135     header = list()
 136     for column in table["columns"]:
 137         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
 138
 139     for _, suite in suites.iteritems():
 140         # Generate data
 141         suite_name = suite["name"]
 142         table_lst = list()
 143         for test in data.keys():
 144             if data[test]["parent"] in suite_name:
 145                 row_lst = list()
 146                 for column in table["columns"]:
 147                     try:
 148                         col_data = str(data[test][column["data"].
 149                                        split(" ")[1]]).replace('"', '""')
 150                         if column["data"].split(" ")[1] in ("vat-history",
 151                                                             "show-run"):
 152                             col_data = replace(col_data, " |br| ", "",
 153                                                maxreplace=1)
 154                             col_data = " |prein| {0} |preout| ".\
 155                                 format(col_data[:-5])
 156                         row_lst.append('"{0}"'.format(col_data))
 157                     except KeyError:
 158                         row_lst.append("No data")
 159                 table_lst.append(row_lst)
 160
 161         # Write the data to file
 162         if table_lst:
 163             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
 164                                             table["output-file-ext"])
 165             logging.info("      Writing file: '{}'".format(file_name))
 166             with open(file_name, "w") as file_handler:
 167                 file_handler.write(",".join(header) + "\n")
 168                 for item in table_lst:
 169                     file_handler.write(",".join(item) + "\n")
 170
 171     logging.info("  Done.")
 172
 173
 174 def table_performance_improvements(table, input_data):
 175     """Generate the table(s) with algorithm: table_performance_improvements
 176     specified in the specification file.
 177
 178     :param table: Table to generate.
 179     :param input_data: Data to process.
 180     :type table: pandas.Series
 181     :type input_data: InputData
 182     """
 183
 184     def _write_line_to_file(file_handler, data):
 185         """Write a line to the .csv file.
 186
 187         :param file_handler: File handler for the csv file. It must be open for
 188          writing text.
 189         :param data: Item to be written to the file.
 190         :type file_handler: BinaryIO
 191         :type data: list
 192         """
 193
 194         line_lst = list()
 195         for item in data:
 196             if isinstance(item["data"], str):
 197                 line_lst.append(item["data"])
 198                 # Remove -?drdisc from the end
 199                 if item["data"].endswith("drdisc"):
 200                     item["data"] = item["data"][:-8]
 201             elif isinstance(item["data"], float):
 202                 line_lst.append("{:.1f}".format(item["data"]))
 203             elif item["data"] is None:
 204                 line_lst.append("")
 205         file_handler.write(",".join(line_lst) + "\n")
 206
 207     logging.info("  Generating the table {0} ...".
 208                  format(table.get("title", "")))
 209
 210     # Read the template
 211     file_name = table.get("template", None)
 212     if file_name:
 213         try:
 214             tmpl = _read_csv_template(file_name)
 215         except PresentationError:
 216             logging.error("  The template '{0}' does not exist. Skipping the "
 217                           "table.".format(file_name))
 218             return None
 219     else:
 220         logging.error("The template is not defined. Skipping the table.")
 221         return None
 222
 223     # Transform the data
 224     data = input_data.filter_data(table)
 225
 226     # Prepare the header of the tables
 227     header = list()
 228     for column in table["columns"]:
 229         header.append(column["title"])
 230
 231     # Generate the data for the table according to the model in the table
 232     # specification
 233     tbl_lst = list()
 234     for tmpl_item in tmpl:
 235         tbl_item = list()
 236         for column in table["columns"]:
 237             cmd = column["data"].split(" ")[0]
 238             args = column["data"].split(" ")[1:]
 239             if cmd == "template":
 240                 try:
 241                     val = float(tmpl_item[int(args[0])])
 242                 except ValueError:
 243                     val = tmpl_item[int(args[0])]
 244                 tbl_item.append({"data": val})
 245             elif cmd == "data":
 246                 jobs = args[0:-1]
 247                 operation = args[-1]
 248                 data_lst = list()
 249                 for job in jobs:
 250                     for build in data[job]:
 251                         try:
 252                             data_lst.append(float(build[tmpl_item[0]]
 253                                                   ["throughput"]["value"]))
 254                         except (KeyError, TypeError):
 255                             # No data, ignore
 256                             continue
 257                 if data_lst:
 258                     tbl_item.append({"data": (eval(operation)(data_lst)) /
 259                                              1000000})
 260                 else:
 261                     tbl_item.append({"data": None})
 262             elif cmd == "operation":
 263                 operation = args[0]
 264                 try:
 265                     nr1 = float(tbl_item[int(args[1])]["data"])
 266                     nr2 = float(tbl_item[int(args[2])]["data"])
 267                     if nr1 and nr2:
 268                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
 269                     else:
 270                         tbl_item.append({"data": None})
 271                 except (IndexError, ValueError, TypeError):
 272                     logging.error("No data for {0}".format(tbl_item[1]["data"]))
 273                     tbl_item.append({"data": None})
 274                     continue
 275             else:
 276                 logging.error("Not supported command {0}. Skipping the table.".
 277                               format(cmd))
 278                 return None
 279         tbl_lst.append(tbl_item)
 280
 281     # Sort the table according to the relative change
 282     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
 283
 284     # Create the tables and write them to the files
 285     file_names = [
 286         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
 287         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
 288         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
 289         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
 290     ]
 291
 292     for file_name in file_names:
 293         logging.info("    Writing the file '{0}'".format(file_name))
 294         with open(file_name, "w") as file_handler:
 295             file_handler.write(",".join(header) + "\n")
 296             for item in tbl_lst:
 297                 if isinstance(item[-1]["data"], float):
 298                     rel_change = round(item[-1]["data"], 1)
 299                 else:
 300                     rel_change = item[-1]["data"]
 301                 if "ndr_top" in file_name \
 302                         and "ndr" in item[1]["data"] \
 303                         and rel_change >= 10.0:
 304                     _write_line_to_file(file_handler, item)
 305                 elif "pdr_top" in file_name \
 306                         and "pdr" in item[1]["data"] \
 307                         and rel_change >= 10.0:
 308                     _write_line_to_file(file_handler, item)
 309                 elif "ndr_low" in file_name \
 310                         and "ndr" in item[1]["data"] \
 311                         and rel_change < 10.0:
 312                     _write_line_to_file(file_handler, item)
 313                 elif "pdr_low" in file_name \
 314                         and "pdr" in item[1]["data"] \
 315                         and rel_change < 10.0:
 316                     _write_line_to_file(file_handler, item)
 317
 318     logging.info("  Done.")
 319
 320
 321 def _read_csv_template(file_name):
 322     """Read the template from a .csv file.
 323
 324     :param file_name: Name / full path / relative path of the file to read.
 325     :type file_name: str
 326     :returns: Data from the template as list (lines) of lists (items on line).
 327     :rtype: list
 328     :raises: PresentationError if it is not possible to read the file.
 329     """
 330
 331     try:
 332         with open(file_name, 'r') as csv_file:
 333             tmpl_data = list()
 334             for line in csv_file:
 335                 tmpl_data.append(line[:-1].split(","))
 336         return tmpl_data
 337     except IOError as err:
 338         raise PresentationError(str(err), level="ERROR")
 339
 340
 341 def table_performance_comparison(table, input_data):
 342     """Generate the table(s) with algorithm: table_performance_comparison
 343     specified in the specification file.
 344
 345     :param table: Table to generate.
 346     :param input_data: Data to process.
 347     :type table: pandas.Series
 348     :type input_data: InputData
 349     """
 350
 351     logging.info("  Generating the table {0} ...".
 352                  format(table.get("title", "")))
 353
 354     # Transform the data
 355     data = input_data.filter_data(table)
 356     logging.info(data)
 357
 358     # Prepare the header of the tables
 359     try:
 360         header = ["Test case",
 361                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
 362                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
 363                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
 364                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
 365                   "Change [%]"]
 366         header_str = ",".join(header) + "\n"
 367     except (AttributeError, KeyError) as err:
 368         logging.error("The model is invalid, missing parameter: {0}".
 369                       format(err))
 370         return
 371
 372     # Prepare data to the table:
 373     tbl_dict = dict()
 374     for job, builds in table["reference"]["data"].items():
 375         for build in builds:
 376             for tst_name, tst_data in data[job][str(build)].iteritems():
 377                 if tbl_dict.get(tst_name, None) is None:
 378                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
 379                                             "-".join(tst_data["name"].
 380                                                      split("-")[1:]))
 381                     tbl_dict[tst_name] = {"name": name,
 382                                           "ref-data": list(),
 383                                           "cmp-data": list()}
 384                 try:
 385                     tbl_dict[tst_name]["ref-data"].\
 386                         append(tst_data["throughput"]["value"])
 387                 except TypeError:
 388                     pass  # No data in output.xml for this test
 389
 390     for job, builds in table["compare"]["data"].items():
 391         for build in builds:
 392             for tst_name, tst_data in data[job][str(build)].iteritems():
 393                 try:
 394                     tbl_dict[tst_name]["cmp-data"].\
 395                         append(tst_data["throughput"]["value"])
 396                 except KeyError:
 397                     pass
 398                 except TypeError:
 399                     tbl_dict.pop(tst_name, None)
 400
 401     tbl_lst = list()
 402     for tst_name in tbl_dict.keys():
 403         item = [tbl_dict[tst_name]["name"], ]
 404         if tbl_dict[tst_name]["ref-data"]:
 405             item.append(round(mean(tbl_dict[tst_name]["ref-data"]) / 1000000,
 406                               2))
 407             item.append(round(stdev(tbl_dict[tst_name]["ref-data"]) / 1000000,
 408                               2))
 409         else:
 410             item.extend([None, None])
 411         if tbl_dict[tst_name]["cmp-data"]:
 412             item.append(round(mean(tbl_dict[tst_name]["cmp-data"]) / 1000000,
 413                               2))
 414             item.append(round(stdev(tbl_dict[tst_name]["cmp-data"]) / 1000000,
 415                               2))
 416         else:
 417             item.extend([None, None])
 418         if item[1] is not None and item[3] is not None:
 419             item.append(int(relative_change(float(item[1]), float(item[3]))))
 420         if len(item) == 6:
 421             tbl_lst.append(item)
 422
 423     # Sort the table according to the relative change
 424     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
 425
 426     # Generate tables:
 427     # All tests in csv:
 428     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
 429                                                table["output-file-ext"]),
 430                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
 431                                                table["output-file-ext"]),
 432                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
 433                                                table["output-file-ext"]),
 434                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 435                                                table["output-file-ext"]),
 436                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
 437                                                table["output-file-ext"]),
 438                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
 439                                                table["output-file-ext"])
 440                  ]
 441     for file_name in tbl_names:
 442         logging.info("      Writing file: '{}'".format(file_name))
 443         with open(file_name, "w") as file_handler:
 444             file_handler.write(header_str)
 445             for test in tbl_lst:
 446                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
 447                         file_name.split("-")[-2] in test[0]):  # cores
 448                     test[0] = "-".join(test[0].split("-")[:-1])
 449                     file_handler.write(",".join([str(item) for item in test]) +
 450                                        "\n")
 451
 452     # All tests in txt:
 453     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
 454                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
 455                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
 456                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
 457                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
 458                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
 459                      ]
 460
 461     for i, txt_name in enumerate(tbl_names_txt):
 462         txt_table = None
 463         logging.info("      Writing file: '{}'".format(txt_name))
 464         with open(tbl_names[i], 'rb') as csv_file:
 465             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 466             for row in csv_content:
 467                 if txt_table is None:
 468                     txt_table = prettytable.PrettyTable(row)
 469                 else:
 470                     txt_table.add_row(row)
 471         with open(txt_name, "w") as txt_file:
 472             txt_file.write(str(txt_table))
 473
 474     # Selected tests in csv:
 475     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
 476                                                table["output-file-ext"])
 477     with open(input_file, "r") as in_file:
 478         lines = list()
 479         for line in in_file:
 480             lines.append(line)
 481
 482     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
 483                                                table["output-file-ext"])
 484     logging.info("      Writing file: '{}'".format(output_file))
 485     with open(output_file, "w") as out_file:
 486         out_file.write(header_str)
 487         for i, line in enumerate(lines[1:]):
 488             if i == table["nr-of-tests-shown"]:
 489                 break
 490             out_file.write(line)
 491
 492     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
 493                                                   table["output-file-ext"])
 494     logging.info("      Writing file: '{}'".format(output_file))
 495     with open(output_file, "w") as out_file:
 496         out_file.write(header_str)
 497         for i, line in enumerate(lines[-1:0:-1]):
 498             if i == table["nr-of-tests-shown"]:
 499                 break
 500             out_file.write(line)
 501
 502     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
 503                                                table["output-file-ext"])
 504     with open(input_file, "r") as in_file:
 505         lines = list()
 506         for line in in_file:
 507             lines.append(line)
 508
 509     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
 510                                                table["output-file-ext"])
 511     logging.info("      Writing file: '{}'".format(output_file))
 512     with open(output_file, "w") as out_file:
 513         out_file.write(header_str)
 514         for i, line in enumerate(lines[1:]):
 515             if i == table["nr-of-tests-shown"]:
 516                 break
 517             out_file.write(line)
 518
 519     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
 520                                                   table["output-file-ext"])
 521     logging.info("      Writing file: '{}'".format(output_file))
 522     with open(output_file, "w") as out_file:
 523         out_file.write(header_str)
 524         for i, line in enumerate(lines[-1:0:-1]):
 525             if i == table["nr-of-tests-shown"]:
 526                 break
 527             out_file.write(line)