CSIT-1041: Trending dashboard
[csit.git] / resources / tools / presentation / generator_tables.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Algorithms to generate tables.
15 """
16
17
18 import logging
19 import csv
20 import prettytable
21 import numpy as np
22 import pandas as pd
23
24 from string import replace
25 from math import isnan
26
27 from errors import PresentationError
28 from utils import mean, stdev, relative_change, remove_outliers, find_outliers
29
30
31 def generate_tables(spec, data):
32     """Generate all tables specified in the specification file.
33
34     :param spec: Specification read from the specification file.
35     :param data: Data to process.
36     :type spec: Specification
37     :type data: InputData
38     """
39
40     logging.info("Generating the tables ...")
41     for table in spec.tables:
42         try:
43             eval(table["algorithm"])(table, data)
44         except NameError:
45             logging.error("The algorithm '{0}' is not defined.".
46                           format(table["algorithm"]))
47     logging.info("Done.")
48
49
50 def table_details(table, input_data):
51     """Generate the table(s) with algorithm: table_detailed_test_results
52     specified in the specification file.
53
54     :param table: Table to generate.
55     :param input_data: Data to process.
56     :type table: pandas.Series
57     :type input_data: InputData
58     """
59
60     logging.info("  Generating the table {0} ...".
61                  format(table.get("title", "")))
62
63     # Transform the data
64     data = input_data.filter_data(table)
65
66     # Prepare the header of the tables
67     header = list()
68     for column in table["columns"]:
69         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
70
71     # Generate the data for the table according to the model in the table
72     # specification
73     job = table["data"].keys()[0]
74     build = str(table["data"][job][0])
75     try:
76         suites = input_data.suites(job, build)
77     except KeyError:
78         logging.error("    No data available. The table will not be generated.")
79         return
80
81     for suite_longname, suite in suites.iteritems():
82         # Generate data
83         suite_name = suite["name"]
84         table_lst = list()
85         for test in data[job][build].keys():
86             if data[job][build][test]["parent"] in suite_name:
87                 row_lst = list()
88                 for column in table["columns"]:
89                     try:
90                         col_data = str(data[job][build][test][column["data"].
91                                        split(" ")[1]]).replace('"', '""')
92                         if column["data"].split(" ")[1] in ("vat-history",
93                                                             "show-run"):
94                             col_data = replace(col_data, " |br| ", "",
95                                                maxreplace=1)
96                             col_data = " |prein| {0} |preout| ".\
97                                 format(col_data[:-5])
98                         row_lst.append('"{0}"'.format(col_data))
99                     except KeyError:
100                         row_lst.append("No data")
101                 table_lst.append(row_lst)
102
103         # Write the data to file
104         if table_lst:
105             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
106                                             table["output-file-ext"])
107             logging.info("      Writing file: '{}'".format(file_name))
108             with open(file_name, "w") as file_handler:
109                 file_handler.write(",".join(header) + "\n")
110                 for item in table_lst:
111                     file_handler.write(",".join(item) + "\n")
112
113     logging.info("  Done.")
114
115
116 def table_merged_details(table, input_data):
117     """Generate the table(s) with algorithm: table_merged_details
118     specified in the specification file.
119
120     :param table: Table to generate.
121     :param input_data: Data to process.
122     :type table: pandas.Series
123     :type input_data: InputData
124     """
125
126     logging.info("  Generating the table {0} ...".
127                  format(table.get("title", "")))
128
129     # Transform the data
130     data = input_data.filter_data(table)
131     data = input_data.merge_data(data)
132     data.sort_index(inplace=True)
133
134     suites = input_data.filter_data(table, data_set="suites")
135     suites = input_data.merge_data(suites)
136
137     # Prepare the header of the tables
138     header = list()
139     for column in table["columns"]:
140         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
141
142     for _, suite in suites.iteritems():
143         # Generate data
144         suite_name = suite["name"]
145         table_lst = list()
146         for test in data.keys():
147             if data[test]["parent"] in suite_name:
148                 row_lst = list()
149                 for column in table["columns"]:
150                     try:
151                         col_data = str(data[test][column["data"].
152                                        split(" ")[1]]).replace('"', '""')
153                         if column["data"].split(" ")[1] in ("vat-history",
154                                                             "show-run"):
155                             col_data = replace(col_data, " |br| ", "",
156                                                maxreplace=1)
157                             col_data = " |prein| {0} |preout| ".\
158                                 format(col_data[:-5])
159                         row_lst.append('"{0}"'.format(col_data))
160                     except KeyError:
161                         row_lst.append("No data")
162                 table_lst.append(row_lst)
163
164         # Write the data to file
165         if table_lst:
166             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
167                                             table["output-file-ext"])
168             logging.info("      Writing file: '{}'".format(file_name))
169             with open(file_name, "w") as file_handler:
170                 file_handler.write(",".join(header) + "\n")
171                 for item in table_lst:
172                     file_handler.write(",".join(item) + "\n")
173
174     logging.info("  Done.")
175
176
177 def table_performance_improvements(table, input_data):
178     """Generate the table(s) with algorithm: table_performance_improvements
179     specified in the specification file.
180
181     :param table: Table to generate.
182     :param input_data: Data to process.
183     :type table: pandas.Series
184     :type input_data: InputData
185     """
186
187     def _write_line_to_file(file_handler, data):
188         """Write a line to the .csv file.
189
190         :param file_handler: File handler for the csv file. It must be open for
191          writing text.
192         :param data: Item to be written to the file.
193         :type file_handler: BinaryIO
194         :type data: list
195         """
196
197         line_lst = list()
198         for item in data:
199             if isinstance(item["data"], str):
200                 # Remove -?drdisc from the end
201                 if item["data"].endswith("drdisc"):
202                     item["data"] = item["data"][:-8]
203                 line_lst.append(item["data"])
204             elif isinstance(item["data"], float):
205                 line_lst.append("{:.1f}".format(item["data"]))
206             elif item["data"] is None:
207                 line_lst.append("")
208         file_handler.write(",".join(line_lst) + "\n")
209
210     logging.info("  Generating the table {0} ...".
211                  format(table.get("title", "")))
212
213     # Read the template
214     file_name = table.get("template", None)
215     if file_name:
216         try:
217             tmpl = _read_csv_template(file_name)
218         except PresentationError:
219             logging.error("  The template '{0}' does not exist. Skipping the "
220                           "table.".format(file_name))
221             return None
222     else:
223         logging.error("The template is not defined. Skipping the table.")
224         return None
225
226     # Transform the data
227     data = input_data.filter_data(table)
228
229     # Prepare the header of the tables
230     header = list()
231     for column in table["columns"]:
232         header.append(column["title"])
233
234     # Generate the data for the table according to the model in the table
235     # specification
236     tbl_lst = list()
237     for tmpl_item in tmpl:
238         tbl_item = list()
239         for column in table["columns"]:
240             cmd = column["data"].split(" ")[0]
241             args = column["data"].split(" ")[1:]
242             if cmd == "template":
243                 try:
244                     val = float(tmpl_item[int(args[0])])
245                 except ValueError:
246                     val = tmpl_item[int(args[0])]
247                 tbl_item.append({"data": val})
248             elif cmd == "data":
249                 jobs = args[0:-1]
250                 operation = args[-1]
251                 data_lst = list()
252                 for job in jobs:
253                     for build in data[job]:
254                         try:
255                             data_lst.append(float(build[tmpl_item[0]]
256                                                   ["throughput"]["value"]))
257                         except (KeyError, TypeError):
258                             # No data, ignore
259                             continue
260                 if data_lst:
261                     tbl_item.append({"data": (eval(operation)(data_lst)) /
262                                              1000000})
263                 else:
264                     tbl_item.append({"data": None})
265             elif cmd == "operation":
266                 operation = args[0]
267                 try:
268                     nr1 = float(tbl_item[int(args[1])]["data"])
269                     nr2 = float(tbl_item[int(args[2])]["data"])
270                     if nr1 and nr2:
271                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
272                     else:
273                         tbl_item.append({"data": None})
274                 except (IndexError, ValueError, TypeError):
275                     logging.error("No data for {0}".format(tbl_item[0]["data"]))
276                     tbl_item.append({"data": None})
277                     continue
278             else:
279                 logging.error("Not supported command {0}. Skipping the table.".
280                               format(cmd))
281                 return None
282         tbl_lst.append(tbl_item)
283
284     # Sort the table according to the relative change
285     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
286
287     # Create the tables and write them to the files
288     file_names = [
289         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
290         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
291         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
292         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
293     ]
294
295     for file_name in file_names:
296         logging.info("    Writing the file '{0}'".format(file_name))
297         with open(file_name, "w") as file_handler:
298             file_handler.write(",".join(header) + "\n")
299             for item in tbl_lst:
300                 if isinstance(item[-1]["data"], float):
301                     rel_change = round(item[-1]["data"], 1)
302                 else:
303                     rel_change = item[-1]["data"]
304                 if "ndr_top" in file_name \
305                         and "ndr" in item[0]["data"] \
306                         and rel_change >= 10.0:
307                     _write_line_to_file(file_handler, item)
308                 elif "pdr_top" in file_name \
309                         and "pdr" in item[0]["data"] \
310                         and rel_change >= 10.0:
311                     _write_line_to_file(file_handler, item)
312                 elif "ndr_low" in file_name \
313                         and "ndr" in item[0]["data"] \
314                         and rel_change < 10.0:
315                     _write_line_to_file(file_handler, item)
316                 elif "pdr_low" in file_name \
317                         and "pdr" in item[0]["data"] \
318                         and rel_change < 10.0:
319                     _write_line_to_file(file_handler, item)
320
321     logging.info("  Done.")
322
323
324 def _read_csv_template(file_name):
325     """Read the template from a .csv file.
326
327     :param file_name: Name / full path / relative path of the file to read.
328     :type file_name: str
329     :returns: Data from the template as list (lines) of lists (items on line).
330     :rtype: list
331     :raises: PresentationError if it is not possible to read the file.
332     """
333
334     try:
335         with open(file_name, 'r') as csv_file:
336             tmpl_data = list()
337             for line in csv_file:
338                 tmpl_data.append(line[:-1].split(","))
339         return tmpl_data
340     except IOError as err:
341         raise PresentationError(str(err), level="ERROR")
342
343
344 def table_performance_comparison(table, input_data):
345     """Generate the table(s) with algorithm: table_performance_comparison
346     specified in the specification file.
347
348     :param table: Table to generate.
349     :param input_data: Data to process.
350     :type table: pandas.Series
351     :type input_data: InputData
352     """
353
354     logging.info("  Generating the table {0} ...".
355                  format(table.get("title", "")))
356
357     # Transform the data
358     data = input_data.filter_data(table)
359
360     # Prepare the header of the tables
361     try:
362         header = ["Test case",
363                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
364                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
365                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
366                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
367                   "Change [%]"]
368         header_str = ",".join(header) + "\n"
369     except (AttributeError, KeyError) as err:
370         logging.error("The model is invalid, missing parameter: {0}".
371                       format(err))
372         return
373
374     # Prepare data to the table:
375     tbl_dict = dict()
376     for job, builds in table["reference"]["data"].items():
377         for build in builds:
378             for tst_name, tst_data in data[job][str(build)].iteritems():
379                 if tbl_dict.get(tst_name, None) is None:
380                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
381                                             "-".join(tst_data["name"].
382                                                      split("-")[1:]))
383                     tbl_dict[tst_name] = {"name": name,
384                                           "ref-data": list(),
385                                           "cmp-data": list()}
386                 try:
387                     tbl_dict[tst_name]["ref-data"].\
388                         append(tst_data["throughput"]["value"])
389                 except TypeError:
390                     pass  # No data in output.xml for this test
391
392     for job, builds in table["compare"]["data"].items():
393         for build in builds:
394             for tst_name, tst_data in data[job][str(build)].iteritems():
395                 try:
396                     tbl_dict[tst_name]["cmp-data"].\
397                         append(tst_data["throughput"]["value"])
398                 except KeyError:
399                     pass
400                 except TypeError:
401                     tbl_dict.pop(tst_name, None)
402
403     tbl_lst = list()
404     for tst_name in tbl_dict.keys():
405         item = [tbl_dict[tst_name]["name"], ]
406         if tbl_dict[tst_name]["ref-data"]:
407             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
408                                      table["outlier-const"])
409             item.append(round(mean(data_t) / 1000000, 2))
410             item.append(round(stdev(data_t) / 1000000, 2))
411         else:
412             item.extend([None, None])
413         if tbl_dict[tst_name]["cmp-data"]:
414             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
415                                      table["outlier-const"])
416             item.append(round(mean(data_t) / 1000000, 2))
417             item.append(round(stdev(data_t) / 1000000, 2))
418         else:
419             item.extend([None, None])
420         if item[1] is not None and item[3] is not None:
421             item.append(int(relative_change(float(item[1]), float(item[3]))))
422         if len(item) == 6:
423             tbl_lst.append(item)
424
425     # Sort the table according to the relative change
426     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
427
428     # Generate tables:
429     # All tests in csv:
430     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
431                                                table["output-file-ext"]),
432                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
433                                                table["output-file-ext"]),
434                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
435                                                table["output-file-ext"]),
436                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
437                                                table["output-file-ext"]),
438                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
439                                                table["output-file-ext"]),
440                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
441                                                table["output-file-ext"])
442                  ]
443     for file_name in tbl_names:
444         logging.info("      Writing file: '{}'".format(file_name))
445         with open(file_name, "w") as file_handler:
446             file_handler.write(header_str)
447             for test in tbl_lst:
448                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
449                         file_name.split("-")[-2] in test[0]):  # cores
450                     test[0] = "-".join(test[0].split("-")[:-1])
451                     file_handler.write(",".join([str(item) for item in test]) +
452                                        "\n")
453
454     # All tests in txt:
455     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
456                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
457                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
458                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
459                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
460                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
461                      ]
462
463     for i, txt_name in enumerate(tbl_names_txt):
464         txt_table = None
465         logging.info("      Writing file: '{}'".format(txt_name))
466         with open(tbl_names[i], 'rb') as csv_file:
467             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
468             for row in csv_content:
469                 if txt_table is None:
470                     txt_table = prettytable.PrettyTable(row)
471                 else:
472                     txt_table.add_row(row)
473             txt_table.align["Test case"] = "l"
474         with open(txt_name, "w") as txt_file:
475             txt_file.write(str(txt_table))
476
477     # Selected tests in csv:
478     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
479                                                table["output-file-ext"])
480     with open(input_file, "r") as in_file:
481         lines = list()
482         for line in in_file:
483             lines.append(line)
484
485     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
486                                                table["output-file-ext"])
487     logging.info("      Writing file: '{}'".format(output_file))
488     with open(output_file, "w") as out_file:
489         out_file.write(header_str)
490         for i, line in enumerate(lines[1:]):
491             if i == table["nr-of-tests-shown"]:
492                 break
493             out_file.write(line)
494
495     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
496                                                   table["output-file-ext"])
497     logging.info("      Writing file: '{}'".format(output_file))
498     with open(output_file, "w") as out_file:
499         out_file.write(header_str)
500         for i, line in enumerate(lines[-1:0:-1]):
501             if i == table["nr-of-tests-shown"]:
502                 break
503             out_file.write(line)
504
505     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
506                                                table["output-file-ext"])
507     with open(input_file, "r") as in_file:
508         lines = list()
509         for line in in_file:
510             lines.append(line)
511
512     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
513                                                table["output-file-ext"])
514     logging.info("      Writing file: '{}'".format(output_file))
515     with open(output_file, "w") as out_file:
516         out_file.write(header_str)
517         for i, line in enumerate(lines[1:]):
518             if i == table["nr-of-tests-shown"]:
519                 break
520             out_file.write(line)
521
522     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
523                                                   table["output-file-ext"])
524     logging.info("      Writing file: '{}'".format(output_file))
525     with open(output_file, "w") as out_file:
526         out_file.write(header_str)
527         for i, line in enumerate(lines[-1:0:-1]):
528             if i == table["nr-of-tests-shown"]:
529                 break
530             out_file.write(line)
531
532
533 def table_performance_trending_dashboard(table, input_data):
534     """Generate the table(s) with algorithm: table_performance_comparison
535     specified in the specification file.
536
537     :param table: Table to generate.
538     :param input_data: Data to process.
539     :type table: pandas.Series
540     :type input_data: InputData
541     """
542
543     logging.info("  Generating the table {0} ...".
544                  format(table.get("title", "")))
545
546     # Transform the data
547     data = input_data.filter_data(table)
548
549     # Prepare the header of the tables
550     header = ["Test case",
551               "Thput trend [Mpps]",
552               "Change [Mpps]",
553               "Change [%]",
554               "Anomaly"]
555     header_str = ",".join(header) + "\n"
556
557     # Prepare data to the table:
558     tbl_dict = dict()
559     for job, builds in table["data"].items():
560         for build in builds:
561             for tst_name, tst_data in data[job][str(build)].iteritems():
562                 if tbl_dict.get(tst_name, None) is None:
563                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
564                                             "-".join(tst_data["name"].
565                                                      split("-")[1:]))
566                     tbl_dict[tst_name] = {"name": name,
567                                           "data": list()}
568                 try:
569                     tbl_dict[tst_name]["data"]. \
570                         append(tst_data["throughput"]["value"])
571                 except TypeError:
572                     pass  # No data in output.xml for this test
573
574     tbl_lst = list()
575     for tst_name in tbl_dict.keys():
576         if len(tbl_dict[tst_name]["data"]) > 2:
577             pd_data = pd.Series(tbl_dict[tst_name]["data"])
578             win_size = pd_data.size \
579                 if pd_data.size < table["window"] else table["window"]
580             # Test name:
581             name = tbl_dict[tst_name]["name"]
582             # Throughput trend:
583             trend = list(pd_data.rolling(window=win_size).median())[-2]
584             # Anomaly:
585             t_data, _ = find_outliers(pd_data)
586             last = list(t_data)[-1]
587             t_stdev = list(t_data.rolling(window=win_size, min_periods=2).
588                          std())[-2]
589             if isnan(last):
590                 anomaly = "outlier"
591             elif last < (trend - 3 * t_stdev):
592                 anomaly = "regression"
593             elif last > (trend + 3 * t_stdev):
594                 anomaly = "progression"
595             else:
596                 anomaly = "normal"
597             # Change:
598             change = round(float(last - trend) / 1000000, 2)
599             # Relative change:
600             rel_change = int(relative_change(float(trend), float(last)))
601
602             tbl_lst.append([name,
603                             round(float(last) / 1000000, 2),
604                             change,
605                             rel_change,
606                             anomaly])
607
608     # Sort the table according to the relative change
609     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
610
611     file_name = "{}.{}".format(table["output-file"], table["output-file-ext"])
612
613     logging.info("      Writing file: '{}'".format(file_name))
614     with open(file_name, "w") as file_handler:
615         file_handler.write(header_str)
616         for test in tbl_lst:
617             file_handler.write(",".join([str(item) for item in test]) + '\n')
618
619     txt_file_name = "{}.txt".format(table["output-file"])
620     txt_table = None
621     logging.info("      Writing file: '{}'".format(txt_file_name))
622     with open(file_name, 'rb') as csv_file:
623         csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
624         for row in csv_content:
625             if txt_table is None:
626                 txt_table = prettytable.PrettyTable(row)
627             else:
628                 txt_table.add_row(row)
629         txt_table.align["Test case"] = "l"
630     with open(txt_file_name, "w") as txt_file:
631         txt_file.write(str(txt_table))

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.