CSIT-902: Compare performance results before/after meltdown/spectre
[csit.git] / resources / tools / presentation / generator_tables.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Algorithms to generate tables.
15 """
16
17
18 import logging
19 import csv
20 import prettytable
21
22 from string import replace
23
24 from errors import PresentationError
25 from utils import mean, stdev, relative_change
26
27
28 def generate_tables(spec, data):
29     """Generate all tables specified in the specification file.
30
31     :param spec: Specification read from the specification file.
32     :param data: Data to process.
33     :type spec: Specification
34     :type data: InputData
35     """
36
37     logging.info("Generating the tables ...")
38     for table in spec.tables:
39         try:
40             eval(table["algorithm"])(table, data)
41         except NameError:
42             logging.error("The algorithm '{0}' is not defined.".
43                           format(table["algorithm"]))
44     logging.info("Done.")
45
46
47 def table_details(table, input_data):
48     """Generate the table(s) with algorithm: table_detailed_test_results
49     specified in the specification file.
50
51     :param table: Table to generate.
52     :param input_data: Data to process.
53     :type table: pandas.Series
54     :type input_data: InputData
55     """
56
57     logging.info("  Generating the table {0} ...".
58                  format(table.get("title", "")))
59
60     # Transform the data
61     data = input_data.filter_data(table)
62
63     # Prepare the header of the tables
64     header = list()
65     for column in table["columns"]:
66         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
67
68     # Generate the data for the table according to the model in the table
69     # specification
70     job = table["data"].keys()[0]
71     build = str(table["data"][job][0])
72     try:
73         suites = input_data.suites(job, build)
74     except KeyError:
75         logging.error("    No data available. The table will not be generated.")
76         return
77
78     for suite_longname, suite in suites.iteritems():
79         # Generate data
80         suite_name = suite["name"]
81         table_lst = list()
82         for test in data[job][build].keys():
83             if data[job][build][test]["parent"] in suite_name:
84                 row_lst = list()
85                 for column in table["columns"]:
86                     try:
87                         col_data = str(data[job][build][test][column["data"].
88                                        split(" ")[1]]).replace('"', '""')
89                         if column["data"].split(" ")[1] in ("vat-history",
90                                                             "show-run"):
91                             col_data = replace(col_data, " |br| ", "",
92                                                maxreplace=1)
93                             col_data = " |prein| {0} |preout| ".\
94                                 format(col_data[:-5])
95                         row_lst.append('"{0}"'.format(col_data))
96                     except KeyError:
97                         row_lst.append("No data")
98                 table_lst.append(row_lst)
99
100         # Write the data to file
101         if table_lst:
102             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
103                                             table["output-file-ext"])
104             logging.info("      Writing file: '{}'".format(file_name))
105             with open(file_name, "w") as file_handler:
106                 file_handler.write(",".join(header) + "\n")
107                 for item in table_lst:
108                     file_handler.write(",".join(item) + "\n")
109
110     logging.info("  Done.")
111
112
113 def table_merged_details(table, input_data):
114     """Generate the table(s) with algorithm: table_merged_details
115     specified in the specification file.
116
117     :param table: Table to generate.
118     :param input_data: Data to process.
119     :type table: pandas.Series
120     :type input_data: InputData
121     """
122
123     logging.info("  Generating the table {0} ...".
124                  format(table.get("title", "")))
125
126     # Transform the data
127     data = input_data.filter_data(table)
128     data = input_data.merge_data(data)
129     data.sort_index(inplace=True)
130
131     suites = input_data.filter_data(table, data_set="suites")
132     suites = input_data.merge_data(suites)
133
134     # Prepare the header of the tables
135     header = list()
136     for column in table["columns"]:
137         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
138
139     for _, suite in suites.iteritems():
140         # Generate data
141         suite_name = suite["name"]
142         table_lst = list()
143         for test in data.keys():
144             if data[test]["parent"] in suite_name:
145                 row_lst = list()
146                 for column in table["columns"]:
147                     try:
148                         col_data = str(data[test][column["data"].
149                                        split(" ")[1]]).replace('"', '""')
150                         if column["data"].split(" ")[1] in ("vat-history",
151                                                             "show-run"):
152                             col_data = replace(col_data, " |br| ", "",
153                                                maxreplace=1)
154                             col_data = " |prein| {0} |preout| ".\
155                                 format(col_data[:-5])
156                         row_lst.append('"{0}"'.format(col_data))
157                     except KeyError:
158                         row_lst.append("No data")
159                 table_lst.append(row_lst)
160
161         # Write the data to file
162         if table_lst:
163             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
164                                             table["output-file-ext"])
165             logging.info("      Writing file: '{}'".format(file_name))
166             with open(file_name, "w") as file_handler:
167                 file_handler.write(",".join(header) + "\n")
168                 for item in table_lst:
169                     file_handler.write(",".join(item) + "\n")
170
171     logging.info("  Done.")
172
173
174 def table_performance_improvements(table, input_data):
175     """Generate the table(s) with algorithm: table_performance_improvements
176     specified in the specification file.
177
178     :param table: Table to generate.
179     :param input_data: Data to process.
180     :type table: pandas.Series
181     :type input_data: InputData
182     """
183
184     def _write_line_to_file(file_handler, data):
185         """Write a line to the .csv file.
186
187         :param file_handler: File handler for the csv file. It must be open for
188          writing text.
189         :param data: Item to be written to the file.
190         :type file_handler: BinaryIO
191         :type data: list
192         """
193
194         line_lst = list()
195         for item in data:
196             if isinstance(item["data"], str):
197                 line_lst.append(item["data"])
198             elif isinstance(item["data"], float):
199                 line_lst.append("{:.1f}".format(item["data"]))
200             elif item["data"] is None:
201                 line_lst.append("")
202         file_handler.write(",".join(line_lst) + "\n")
203
204     logging.info("  Generating the table {0} ...".
205                  format(table.get("title", "")))
206
207     # Read the template
208     file_name = table.get("template", None)
209     if file_name:
210         try:
211             tmpl = _read_csv_template(file_name)
212         except PresentationError:
213             logging.error("  The template '{0}' does not exist. Skipping the "
214                           "table.".format(file_name))
215             return None
216     else:
217         logging.error("The template is not defined. Skipping the table.")
218         return None
219
220     # Transform the data
221     data = input_data.filter_data(table)
222
223     # Prepare the header of the tables
224     header = list()
225     for column in table["columns"]:
226         header.append(column["title"])
227
228     # Generate the data for the table according to the model in the table
229     # specification
230     tbl_lst = list()
231     for tmpl_item in tmpl:
232         tbl_item = list()
233         for column in table["columns"]:
234             cmd = column["data"].split(" ")[0]
235             args = column["data"].split(" ")[1:]
236             if cmd == "template":
237                 try:
238                     val = float(tmpl_item[int(args[0])])
239                 except ValueError:
240                     val = tmpl_item[int(args[0])]
241                 tbl_item.append({"data": val})
242             elif cmd == "data":
243                 jobs = args[0:-1]
244                 operation = args[-1]
245                 data_lst = list()
246                 for job in jobs:
247                     for build in data[job]:
248                         try:
249                             data_lst.append(float(build[tmpl_item[0]]
250                                                   ["throughput"]["value"]))
251                         except (KeyError, TypeError):
252                             # No data, ignore
253                             continue
254                 if data_lst:
255                     tbl_item.append({"data": (eval(operation)(data_lst)) /
256                                              1000000})
257                 else:
258                     tbl_item.append({"data": None})
259             elif cmd == "operation":
260                 operation = args[0]
261                 try:
262                     nr1 = float(tbl_item[int(args[1])]["data"])
263                     nr2 = float(tbl_item[int(args[2])]["data"])
264                     if nr1 and nr2:
265                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
266                     else:
267                         tbl_item.append({"data": None})
268                 except (IndexError, ValueError, TypeError):
269                     logging.error("No data for {0}".format(tbl_item[1]["data"]))
270                     tbl_item.append({"data": None})
271                     continue
272             else:
273                 logging.error("Not supported command {0}. Skipping the table.".
274                               format(cmd))
275                 return None
276         tbl_lst.append(tbl_item)
277
278     # Sort the table according to the relative change
279     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
280
281     # Create the tables and write them to the files
282     file_names = [
283         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
284         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
285         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
286         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
287     ]
288
289     for file_name in file_names:
290         logging.info("    Writing the file '{0}'".format(file_name))
291         with open(file_name, "w") as file_handler:
292             file_handler.write(",".join(header) + "\n")
293             for item in tbl_lst:
294                 if isinstance(item[-1]["data"], float):
295                     rel_change = round(item[-1]["data"], 1)
296                 else:
297                     rel_change = item[-1]["data"]
298                 if "ndr_top" in file_name \
299                         and "ndr" in item[1]["data"] \
300                         and rel_change >= 10.0:
301                     _write_line_to_file(file_handler, item)
302                 elif "pdr_top" in file_name \
303                         and "pdr" in item[1]["data"] \
304                         and rel_change >= 10.0:
305                     _write_line_to_file(file_handler, item)
306                 elif "ndr_low" in file_name \
307                         and "ndr" in item[1]["data"] \
308                         and rel_change < 10.0:
309                     _write_line_to_file(file_handler, item)
310                 elif "pdr_low" in file_name \
311                         and "pdr" in item[1]["data"] \
312                         and rel_change < 10.0:
313                     _write_line_to_file(file_handler, item)
314
315     logging.info("  Done.")
316
317
318 def _read_csv_template(file_name):
319     """Read the template from a .csv file.
320
321     :param file_name: Name / full path / relative path of the file to read.
322     :type file_name: str
323     :returns: Data from the template as list (lines) of lists (items on line).
324     :rtype: list
325     :raises: PresentationError if it is not possible to read the file.
326     """
327
328     try:
329         with open(file_name, 'r') as csv_file:
330             tmpl_data = list()
331             for line in csv_file:
332                 tmpl_data.append(line[:-1].split(","))
333         return tmpl_data
334     except IOError as err:
335         raise PresentationError(str(err), level="ERROR")
336
337
338 def table_performance_comparision(table, input_data):
339     """Generate the table(s) with algorithm: table_performance_comparision
340     specified in the specification file.
341
342     :param table: Table to generate.
343     :param input_data: Data to process.
344     :type table: pandas.Series
345     :type input_data: InputData
346     """
347
348     # Transform the data
349     data = input_data.filter_data(table)
350
351     # Prepare the header of the tables
352     try:
353         header = ["Test case",
354                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
355                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
356                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
357                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
358                   "Change [%]"]
359         header_str = ",".join(header) + "\n"
360     except (AttributeError, KeyError) as err:
361         logging.error("The model is invalid, missing parameter: {0}".
362                       format(err))
363         return
364
365     # Prepare data to the table:
366     tbl_dict = dict()
367     for job, builds in table["reference"]["data"].items():
368         for build in builds:
369             for tst_name, tst_data in data[job][str(build)].iteritems():
370                 if tbl_dict.get(tst_name, None) is None:
371                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
372                                             "-".join(tst_data["name"].
373                                                      split("-")[1:]))
374                     tbl_dict[tst_name] = {"name": name,
375                                           "ref-data": list(),
376                                           "cmp-data": list()}
377                 tbl_dict[tst_name]["ref-data"].\
378                     append(tst_data["throughput"]["value"])
379
380     for job, builds in table["compare"]["data"].items():
381         for build in builds:
382             for tst_name, tst_data in data[job][str(build)].iteritems():
383                 tbl_dict[tst_name]["cmp-data"].\
384                     append(tst_data["throughput"]["value"])
385
386     tbl_lst = list()
387     for tst_name in tbl_dict.keys():
388         item = [tbl_dict[tst_name]["name"], ]
389         if tbl_dict[tst_name]["ref-data"]:
390             item.append(round(mean(tbl_dict[tst_name]["ref-data"]) / 1000000,
391                               2))
392             item.append(round(stdev(tbl_dict[tst_name]["ref-data"]) / 1000000,
393                               2))
394         else:
395             item.extend([None, None])
396         if tbl_dict[tst_name]["cmp-data"]:
397             item.append(round(mean(tbl_dict[tst_name]["cmp-data"]) / 1000000,
398                               2))
399             item.append(round(stdev(tbl_dict[tst_name]["cmp-data"]) / 1000000,
400                               2))
401         else:
402             item.extend([None, None])
403         if item[1] is not None and item[3] is not None:
404             item.append(int(relative_change(float(item[1]), float(item[3]))))
405         if len(item) == 6:
406             tbl_lst.append(item)
407
408     # Sort the table according to the relative change
409     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
410
411     # Generate tables:
412     # All tests in csv:
413     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
414                                                table["output-file-ext"]),
415                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
416                                                table["output-file-ext"]),
417                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
418                                                table["output-file-ext"]),
419                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
420                                                table["output-file-ext"]),
421                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
422                                                table["output-file-ext"]),
423                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
424                                                table["output-file-ext"])
425                  ]
426     for file_name in tbl_names:
427         with open(file_name, "w") as file_handler:
428             file_handler.write(header_str)
429             for test in tbl_lst:
430                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
431                         file_name.split("-")[-2] in test[0]):  # cores
432                     test[0] = "-".join(test[0].split("-")[:-1])
433                     file_handler.write(",".join([str(item) for item in test]) +
434                                        "\n")
435
436     # All tests in txt:
437     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
438                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
439                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
440                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
441                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
442                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
443                      ]
444
445     for i, txt_name in enumerate(tbl_names_txt):
446         txt_table = None
447         with open(tbl_names[i], 'rb') as csv_file:
448             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
449             for row in csv_content:
450                 if txt_table is None:
451                     txt_table = prettytable.PrettyTable(row)
452                 else:
453                     txt_table.add_row(row)
454         with open(txt_name, "w") as txt_file:
455             txt_file.write(str(txt_table))
456
457     # Selected tests in csv:
458     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
459                                                table["output-file-ext"])
460     with open(input_file, "r") as in_file:
461         lines = list()
462         for line in in_file:
463             lines.append(line)
464
465     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
466                                                table["output-file-ext"])
467     with open(output_file, "w") as out_file:
468         out_file.write(header_str)
469         for i, line in enumerate(lines[1:]):
470             if i == table["nr-of-tests-shown"]:
471                 break
472             out_file.write(line)
473
474     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
475                                                   table["output-file-ext"])
476     with open(output_file, "w") as out_file:
477         out_file.write(header_str)
478         for i, line in enumerate(lines[-1:0:-1]):
479             if i == table["nr-of-tests-shown"]:
480                 break
481             out_file.write(line)
482
483     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
484                                                table["output-file-ext"])
485     with open(input_file, "r") as in_file:
486         lines = list()
487         for line in in_file:
488             lines.append(line)
489
490     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
491                                                table["output-file-ext"])
492     with open(output_file, "w") as out_file:
493         out_file.write(header_str)
494         for i, line in enumerate(lines[1:]):
495             if i == table["nr-of-tests-shown"]:
496                 break
497             out_file.write(line)
498
499     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
500                                                   table["output-file-ext"])
501     with open(output_file, "w") as out_file:
502         out_file.write(header_str)
503         for i, line in enumerate(lines[-1:0:-1]):
504             if i == table["nr-of-tests-shown"]:
505                 break
506             out_file.write(line)

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.