Report: Fix packet_throughput_graphs/ip4
[csit.git] / resources / tools / presentation / generator_tables.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Algorithms to generate tables.
15 """
16
17
18 import logging
19 import csv
20 import prettytable
21
22 from string import replace
23
24 from errors import PresentationError
25 from utils import mean, stdev, relative_change, remove_outliers
26
27
28 def generate_tables(spec, data):
29     """Generate all tables specified in the specification file.
30
31     :param spec: Specification read from the specification file.
32     :param data: Data to process.
33     :type spec: Specification
34     :type data: InputData
35     """
36
37     logging.info("Generating the tables ...")
38     for table in spec.tables:
39         try:
40             eval(table["algorithm"])(table, data)
41         except NameError:
42             logging.error("The algorithm '{0}' is not defined.".
43                           format(table["algorithm"]))
44     logging.info("Done.")
45
46
47 def table_details(table, input_data):
48     """Generate the table(s) with algorithm: table_detailed_test_results
49     specified in the specification file.
50
51     :param table: Table to generate.
52     :param input_data: Data to process.
53     :type table: pandas.Series
54     :type input_data: InputData
55     """
56
57     logging.info("  Generating the table {0} ...".
58                  format(table.get("title", "")))
59
60     # Transform the data
61     data = input_data.filter_data(table)
62
63     # Prepare the header of the tables
64     header = list()
65     for column in table["columns"]:
66         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
67
68     # Generate the data for the table according to the model in the table
69     # specification
70     job = table["data"].keys()[0]
71     build = str(table["data"][job][0])
72     try:
73         suites = input_data.suites(job, build)
74     except KeyError:
75         logging.error("    No data available. The table will not be generated.")
76         return
77
78     for suite_longname, suite in suites.iteritems():
79         # Generate data
80         suite_name = suite["name"]
81         table_lst = list()
82         for test in data[job][build].keys():
83             if data[job][build][test]["parent"] in suite_name:
84                 row_lst = list()
85                 for column in table["columns"]:
86                     try:
87                         col_data = str(data[job][build][test][column["data"].
88                                        split(" ")[1]]).replace('"', '""')
89                         if column["data"].split(" ")[1] in ("vat-history",
90                                                             "show-run"):
91                             col_data = replace(col_data, " |br| ", "",
92                                                maxreplace=1)
93                             col_data = " |prein| {0} |preout| ".\
94                                 format(col_data[:-5])
95                         row_lst.append('"{0}"'.format(col_data))
96                     except KeyError:
97                         row_lst.append("No data")
98                 table_lst.append(row_lst)
99
100         # Write the data to file
101         if table_lst:
102             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
103                                             table["output-file-ext"])
104             logging.info("      Writing file: '{}'".format(file_name))
105             with open(file_name, "w") as file_handler:
106                 file_handler.write(",".join(header) + "\n")
107                 for item in table_lst:
108                     file_handler.write(",".join(item) + "\n")
109
110     logging.info("  Done.")
111
112
113 def table_merged_details(table, input_data):
114     """Generate the table(s) with algorithm: table_merged_details
115     specified in the specification file.
116
117     :param table: Table to generate.
118     :param input_data: Data to process.
119     :type table: pandas.Series
120     :type input_data: InputData
121     """
122
123     logging.info("  Generating the table {0} ...".
124                  format(table.get("title", "")))
125
126     # Transform the data
127     data = input_data.filter_data(table)
128     data = input_data.merge_data(data)
129     data.sort_index(inplace=True)
130
131     suites = input_data.filter_data(table, data_set="suites")
132     suites = input_data.merge_data(suites)
133
134     # Prepare the header of the tables
135     header = list()
136     for column in table["columns"]:
137         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
138
139     for _, suite in suites.iteritems():
140         # Generate data
141         suite_name = suite["name"]
142         table_lst = list()
143         for test in data.keys():
144             if data[test]["parent"] in suite_name:
145                 row_lst = list()
146                 for column in table["columns"]:
147                     try:
148                         col_data = str(data[test][column["data"].
149                                        split(" ")[1]]).replace('"', '""')
150                         if column["data"].split(" ")[1] in ("vat-history",
151                                                             "show-run"):
152                             col_data = replace(col_data, " |br| ", "",
153                                                maxreplace=1)
154                             col_data = " |prein| {0} |preout| ".\
155                                 format(col_data[:-5])
156                         row_lst.append('"{0}"'.format(col_data))
157                     except KeyError:
158                         row_lst.append("No data")
159                 table_lst.append(row_lst)
160
161         # Write the data to file
162         if table_lst:
163             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
164                                             table["output-file-ext"])
165             logging.info("      Writing file: '{}'".format(file_name))
166             with open(file_name, "w") as file_handler:
167                 file_handler.write(",".join(header) + "\n")
168                 for item in table_lst:
169                     file_handler.write(",".join(item) + "\n")
170
171     logging.info("  Done.")
172
173
174 def table_performance_improvements(table, input_data):
175     """Generate the table(s) with algorithm: table_performance_improvements
176     specified in the specification file.
177
178     :param table: Table to generate.
179     :param input_data: Data to process.
180     :type table: pandas.Series
181     :type input_data: InputData
182     """
183
184     def _write_line_to_file(file_handler, data):
185         """Write a line to the .csv file.
186
187         :param file_handler: File handler for the csv file. It must be open for
188          writing text.
189         :param data: Item to be written to the file.
190         :type file_handler: BinaryIO
191         :type data: list
192         """
193
194         line_lst = list()
195         for item in data:
196             if isinstance(item["data"], str):
197                 # Remove -?drdisc from the end
198                 if item["data"].endswith("drdisc"):
199                     item["data"] = item["data"][:-8]
200                 line_lst.append(item["data"])
201             elif isinstance(item["data"], float):
202                 line_lst.append("{:.1f}".format(item["data"]))
203             elif item["data"] is None:
204                 line_lst.append("")
205         file_handler.write(",".join(line_lst) + "\n")
206
207     logging.info("  Generating the table {0} ...".
208                  format(table.get("title", "")))
209
210     # Read the template
211     file_name = table.get("template", None)
212     if file_name:
213         try:
214             tmpl = _read_csv_template(file_name)
215         except PresentationError:
216             logging.error("  The template '{0}' does not exist. Skipping the "
217                           "table.".format(file_name))
218             return None
219     else:
220         logging.error("The template is not defined. Skipping the table.")
221         return None
222
223     # Transform the data
224     data = input_data.filter_data(table)
225
226     # Prepare the header of the tables
227     header = list()
228     for column in table["columns"]:
229         header.append(column["title"])
230
231     # Generate the data for the table according to the model in the table
232     # specification
233     tbl_lst = list()
234     for tmpl_item in tmpl:
235         tbl_item = list()
236         for column in table["columns"]:
237             cmd = column["data"].split(" ")[0]
238             args = column["data"].split(" ")[1:]
239             if cmd == "template":
240                 try:
241                     val = float(tmpl_item[int(args[0])])
242                 except ValueError:
243                     val = tmpl_item[int(args[0])]
244                 tbl_item.append({"data": val})
245             elif cmd == "data":
246                 jobs = args[0:-1]
247                 operation = args[-1]
248                 data_lst = list()
249                 for job in jobs:
250                     for build in data[job]:
251                         try:
252                             data_lst.append(float(build[tmpl_item[0]]
253                                                   ["throughput"]["value"]))
254                         except (KeyError, TypeError):
255                             # No data, ignore
256                             continue
257                 if data_lst:
258                     tbl_item.append({"data": (eval(operation)(data_lst)) /
259                                              1000000})
260                 else:
261                     tbl_item.append({"data": None})
262             elif cmd == "operation":
263                 operation = args[0]
264                 try:
265                     nr1 = float(tbl_item[int(args[1])]["data"])
266                     nr2 = float(tbl_item[int(args[2])]["data"])
267                     if nr1 and nr2:
268                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
269                     else:
270                         tbl_item.append({"data": None})
271                 except (IndexError, ValueError, TypeError):
272                     logging.error("No data for {0}".format(tbl_item[0]["data"]))
273                     tbl_item.append({"data": None})
274                     continue
275             else:
276                 logging.error("Not supported command {0}. Skipping the table.".
277                               format(cmd))
278                 return None
279         tbl_lst.append(tbl_item)
280
281     # Sort the table according to the relative change
282     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
283
284     # Create the tables and write them to the files
285     file_names = [
286         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
287         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
288         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
289         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
290     ]
291
292     for file_name in file_names:
293         logging.info("    Writing the file '{0}'".format(file_name))
294         with open(file_name, "w") as file_handler:
295             file_handler.write(",".join(header) + "\n")
296             for item in tbl_lst:
297                 if isinstance(item[-1]["data"], float):
298                     rel_change = round(item[-1]["data"], 1)
299                 else:
300                     rel_change = item[-1]["data"]
301                 if "ndr_top" in file_name \
302                         and "ndr" in item[0]["data"] \
303                         and rel_change >= 10.0:
304                     _write_line_to_file(file_handler, item)
305                 elif "pdr_top" in file_name \
306                         and "pdr" in item[0]["data"] \
307                         and rel_change >= 10.0:
308                     _write_line_to_file(file_handler, item)
309                 elif "ndr_low" in file_name \
310                         and "ndr" in item[0]["data"] \
311                         and rel_change < 10.0:
312                     _write_line_to_file(file_handler, item)
313                 elif "pdr_low" in file_name \
314                         and "pdr" in item[0]["data"] \
315                         and rel_change < 10.0:
316                     _write_line_to_file(file_handler, item)
317
318     logging.info("  Done.")
319
320
321 def _read_csv_template(file_name):
322     """Read the template from a .csv file.
323
324     :param file_name: Name / full path / relative path of the file to read.
325     :type file_name: str
326     :returns: Data from the template as list (lines) of lists (items on line).
327     :rtype: list
328     :raises: PresentationError if it is not possible to read the file.
329     """
330
331     try:
332         with open(file_name, 'r') as csv_file:
333             tmpl_data = list()
334             for line in csv_file:
335                 tmpl_data.append(line[:-1].split(","))
336         return tmpl_data
337     except IOError as err:
338         raise PresentationError(str(err), level="ERROR")
339
340
341 def table_performance_comparison(table, input_data):
342     """Generate the table(s) with algorithm: table_performance_comparison
343     specified in the specification file.
344
345     :param table: Table to generate.
346     :param input_data: Data to process.
347     :type table: pandas.Series
348     :type input_data: InputData
349     """
350
351     logging.info("  Generating the table {0} ...".
352                  format(table.get("title", "")))
353
354     # Transform the data
355     data = input_data.filter_data(table)
356
357     # Prepare the header of the tables
358     try:
359         header = ["Test case",
360                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
361                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
362                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
363                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
364                   "Change [%]"]
365         header_str = ",".join(header) + "\n"
366     except (AttributeError, KeyError) as err:
367         logging.error("The model is invalid, missing parameter: {0}".
368                       format(err))
369         return
370
371     # Prepare data to the table:
372     tbl_dict = dict()
373     for job, builds in table["reference"]["data"].items():
374         for build in builds:
375             for tst_name, tst_data in data[job][str(build)].iteritems():
376                 if tbl_dict.get(tst_name, None) is None:
377                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
378                                             "-".join(tst_data["name"].
379                                                      split("-")[1:]))
380                     tbl_dict[tst_name] = {"name": name,
381                                           "ref-data": list(),
382                                           "cmp-data": list()}
383                 try:
384                     tbl_dict[tst_name]["ref-data"].\
385                         append(tst_data["throughput"]["value"])
386                 except TypeError:
387                     pass  # No data in output.xml for this test
388
389     for job, builds in table["compare"]["data"].items():
390         for build in builds:
391             for tst_name, tst_data in data[job][str(build)].iteritems():
392                 try:
393                     tbl_dict[tst_name]["cmp-data"].\
394                         append(tst_data["throughput"]["value"])
395                 except KeyError:
396                     pass
397                 except TypeError:
398                     tbl_dict.pop(tst_name, None)
399
400     tbl_lst = list()
401     for tst_name in tbl_dict.keys():
402         item = [tbl_dict[tst_name]["name"], ]
403         if tbl_dict[tst_name]["ref-data"]:
404             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
405                                      table["outlier-const"])
406             item.append(round(mean(data_t) / 1000000, 2))
407             item.append(round(stdev(data_t) / 1000000, 2))
408         else:
409             item.extend([None, None])
410         if tbl_dict[tst_name]["cmp-data"]:
411             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
412                                      table["outlier-const"])
413             item.append(round(mean(data_t) / 1000000, 2))
414             item.append(round(stdev(data_t) / 1000000, 2))
415         else:
416             item.extend([None, None])
417         if item[1] is not None and item[3] is not None:
418             item.append(int(relative_change(float(item[1]), float(item[3]))))
419         if len(item) == 6:
420             tbl_lst.append(item)
421
422     # Sort the table according to the relative change
423     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
424
425     # Generate tables:
426     # All tests in csv:
427     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
428                                                table["output-file-ext"]),
429                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
430                                                table["output-file-ext"]),
431                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
432                                                table["output-file-ext"]),
433                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
434                                                table["output-file-ext"]),
435                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
436                                                table["output-file-ext"]),
437                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
438                                                table["output-file-ext"])
439                  ]
440     for file_name in tbl_names:
441         logging.info("      Writing file: '{}'".format(file_name))
442         with open(file_name, "w") as file_handler:
443             file_handler.write(header_str)
444             for test in tbl_lst:
445                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
446                         file_name.split("-")[-2] in test[0]):  # cores
447                     test[0] = "-".join(test[0].split("-")[:-1])
448                     file_handler.write(",".join([str(item) for item in test]) +
449                                        "\n")
450
451     # All tests in txt:
452     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
453                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
454                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
455                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
456                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
457                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
458                      ]
459
460     for i, txt_name in enumerate(tbl_names_txt):
461         txt_table = None
462         logging.info("      Writing file: '{}'".format(txt_name))
463         with open(tbl_names[i], 'rb') as csv_file:
464             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
465             for row in csv_content:
466                 if txt_table is None:
467                     txt_table = prettytable.PrettyTable(row)
468                 else:
469                     txt_table.add_row(row)
470             txt_table.align["Test case"] = "l"
471         with open(txt_name, "w") as txt_file:
472             txt_file.write(str(txt_table))
473
474     # Selected tests in csv:
475     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
476                                                table["output-file-ext"])
477     with open(input_file, "r") as in_file:
478         lines = list()
479         for line in in_file:
480             lines.append(line)
481
482     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
483                                                table["output-file-ext"])
484     logging.info("      Writing file: '{}'".format(output_file))
485     with open(output_file, "w") as out_file:
486         out_file.write(header_str)
487         for i, line in enumerate(lines[1:]):
488             if i == table["nr-of-tests-shown"]:
489                 break
490             out_file.write(line)
491
492     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
493                                                   table["output-file-ext"])
494     logging.info("      Writing file: '{}'".format(output_file))
495     with open(output_file, "w") as out_file:
496         out_file.write(header_str)
497         for i, line in enumerate(lines[-1:0:-1]):
498             if i == table["nr-of-tests-shown"]:
499                 break
500             out_file.write(line)
501
502     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
503                                                table["output-file-ext"])
504     with open(input_file, "r") as in_file:
505         lines = list()
506         for line in in_file:
507             lines.append(line)
508
509     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
510                                                table["output-file-ext"])
511     logging.info("      Writing file: '{}'".format(output_file))
512     with open(output_file, "w") as out_file:
513         out_file.write(header_str)
514         for i, line in enumerate(lines[1:]):
515             if i == table["nr-of-tests-shown"]:
516                 break
517             out_file.write(line)
518
519     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
520                                                   table["output-file-ext"])
521     logging.info("      Writing file: '{}'".format(output_file))
522     with open(output_file, "w") as out_file:
523         out_file.write(header_str)
524         for i, line in enumerate(lines[-1:0:-1]):
525             if i == table["nr-of-tests-shown"]:
526                 break
527             out_file.write(line)

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.