50a662335687f2fd01fdea3b05ad9c13ed855482
[csit.git] / resources / tools / presentation / generator_tables.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Algorithms to generate tables.
15 """
16
17
18 import logging
19 import csv
20 import prettytable
21 import pandas as pd
22
23 from string import replace
24 from math import isnan
25 from xml.etree import ElementTree as ET
26
27 from errors import PresentationError
28 from utils import mean, stdev, relative_change, remove_outliers, split_outliers
29
30
31 def generate_tables(spec, data):
32     """Generate all tables specified in the specification file.
33
34     :param spec: Specification read from the specification file.
35     :param data: Data to process.
36     :type spec: Specification
37     :type data: InputData
38     """
39
40     logging.info("Generating the tables ...")
41     for table in spec.tables:
42         try:
43             eval(table["algorithm"])(table, data)
44         except NameError:
45             logging.error("The algorithm '{0}' is not defined.".
46                           format(table["algorithm"]))
47     logging.info("Done.")
48
49
50 def table_details(table, input_data):
51     """Generate the table(s) with algorithm: table_detailed_test_results
52     specified in the specification file.
53
54     :param table: Table to generate.
55     :param input_data: Data to process.
56     :type table: pandas.Series
57     :type input_data: InputData
58     """
59
60     logging.info("  Generating the table {0} ...".
61                  format(table.get("title", "")))
62
63     # Transform the data
64     data = input_data.filter_data(table)
65
66     # Prepare the header of the tables
67     header = list()
68     for column in table["columns"]:
69         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
70
71     # Generate the data for the table according to the model in the table
72     # specification
73     job = table["data"].keys()[0]
74     build = str(table["data"][job][0])
75     try:
76         suites = input_data.suites(job, build)
77     except KeyError:
78         logging.error("    No data available. The table will not be generated.")
79         return
80
81     for suite_longname, suite in suites.iteritems():
82         # Generate data
83         suite_name = suite["name"]
84         table_lst = list()
85         for test in data[job][build].keys():
86             if data[job][build][test]["parent"] in suite_name:
87                 row_lst = list()
88                 for column in table["columns"]:
89                     try:
90                         col_data = str(data[job][build][test][column["data"].
91                                        split(" ")[1]]).replace('"', '""')
92                         if column["data"].split(" ")[1] in ("vat-history",
93                                                             "show-run"):
94                             col_data = replace(col_data, " |br| ", "",
95                                                maxreplace=1)
96                             col_data = " |prein| {0} |preout| ".\
97                                 format(col_data[:-5])
98                         row_lst.append('"{0}"'.format(col_data))
99                     except KeyError:
100                         row_lst.append("No data")
101                 table_lst.append(row_lst)
102
103         # Write the data to file
104         if table_lst:
105             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
106                                             table["output-file-ext"])
107             logging.info("      Writing file: '{}'".format(file_name))
108             with open(file_name, "w") as file_handler:
109                 file_handler.write(",".join(header) + "\n")
110                 for item in table_lst:
111                     file_handler.write(",".join(item) + "\n")
112
113     logging.info("  Done.")
114
115
116 def table_merged_details(table, input_data):
117     """Generate the table(s) with algorithm: table_merged_details
118     specified in the specification file.
119
120     :param table: Table to generate.
121     :param input_data: Data to process.
122     :type table: pandas.Series
123     :type input_data: InputData
124     """
125
126     logging.info("  Generating the table {0} ...".
127                  format(table.get("title", "")))
128
129     # Transform the data
130     data = input_data.filter_data(table)
131     data = input_data.merge_data(data)
132     data.sort_index(inplace=True)
133
134     suites = input_data.filter_data(table, data_set="suites")
135     suites = input_data.merge_data(suites)
136
137     # Prepare the header of the tables
138     header = list()
139     for column in table["columns"]:
140         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
141
142     for _, suite in suites.iteritems():
143         # Generate data
144         suite_name = suite["name"]
145         table_lst = list()
146         for test in data.keys():
147             if data[test]["parent"] in suite_name:
148                 row_lst = list()
149                 for column in table["columns"]:
150                     try:
151                         col_data = str(data[test][column["data"].
152                                        split(" ")[1]]).replace('"', '""')
153                         if column["data"].split(" ")[1] in ("vat-history",
154                                                             "show-run"):
155                             col_data = replace(col_data, " |br| ", "",
156                                                maxreplace=1)
157                             col_data = " |prein| {0} |preout| ".\
158                                 format(col_data[:-5])
159                         row_lst.append('"{0}"'.format(col_data))
160                     except KeyError:
161                         row_lst.append("No data")
162                 table_lst.append(row_lst)
163
164         # Write the data to file
165         if table_lst:
166             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
167                                             table["output-file-ext"])
168             logging.info("      Writing file: '{}'".format(file_name))
169             with open(file_name, "w") as file_handler:
170                 file_handler.write(",".join(header) + "\n")
171                 for item in table_lst:
172                     file_handler.write(",".join(item) + "\n")
173
174     logging.info("  Done.")
175
176
177 def table_performance_improvements(table, input_data):
178     """Generate the table(s) with algorithm: table_performance_improvements
179     specified in the specification file.
180
181     :param table: Table to generate.
182     :param input_data: Data to process.
183     :type table: pandas.Series
184     :type input_data: InputData
185     """
186
187     def _write_line_to_file(file_handler, data):
188         """Write a line to the .csv file.
189
190         :param file_handler: File handler for the csv file. It must be open for
191          writing text.
192         :param data: Item to be written to the file.
193         :type file_handler: BinaryIO
194         :type data: list
195         """
196
197         line_lst = list()
198         for item in data:
199             if isinstance(item["data"], str):
200                 # Remove -?drdisc from the end
201                 if item["data"].endswith("drdisc"):
202                     item["data"] = item["data"][:-8]
203                 line_lst.append(item["data"])
204             elif isinstance(item["data"], float):
205                 line_lst.append("{:.1f}".format(item["data"]))
206             elif item["data"] is None:
207                 line_lst.append("")
208         file_handler.write(",".join(line_lst) + "\n")
209
210     logging.info("  Generating the table {0} ...".
211                  format(table.get("title", "")))
212
213     # Read the template
214     file_name = table.get("template", None)
215     if file_name:
216         try:
217             tmpl = _read_csv_template(file_name)
218         except PresentationError:
219             logging.error("  The template '{0}' does not exist. Skipping the "
220                           "table.".format(file_name))
221             return None
222     else:
223         logging.error("The template is not defined. Skipping the table.")
224         return None
225
226     # Transform the data
227     data = input_data.filter_data(table)
228
229     # Prepare the header of the tables
230     header = list()
231     for column in table["columns"]:
232         header.append(column["title"])
233
234     # Generate the data for the table according to the model in the table
235     # specification
236     tbl_lst = list()
237     for tmpl_item in tmpl:
238         tbl_item = list()
239         for column in table["columns"]:
240             cmd = column["data"].split(" ")[0]
241             args = column["data"].split(" ")[1:]
242             if cmd == "template":
243                 try:
244                     val = float(tmpl_item[int(args[0])])
245                 except ValueError:
246                     val = tmpl_item[int(args[0])]
247                 tbl_item.append({"data": val})
248             elif cmd == "data":
249                 jobs = args[0:-1]
250                 operation = args[-1]
251                 data_lst = list()
252                 for job in jobs:
253                     for build in data[job]:
254                         try:
255                             data_lst.append(float(build[tmpl_item[0]]
256                                                   ["throughput"]["value"]))
257                         except (KeyError, TypeError):
258                             # No data, ignore
259                             continue
260                 if data_lst:
261                     tbl_item.append({"data": (eval(operation)(data_lst)) /
262                                              1000000})
263                 else:
264                     tbl_item.append({"data": None})
265             elif cmd == "operation":
266                 operation = args[0]
267                 try:
268                     nr1 = float(tbl_item[int(args[1])]["data"])
269                     nr2 = float(tbl_item[int(args[2])]["data"])
270                     if nr1 and nr2:
271                         tbl_item.append({"data": eval(operation)(nr1, nr2)})
272                     else:
273                         tbl_item.append({"data": None})
274                 except (IndexError, ValueError, TypeError):
275                     logging.error("No data for {0}".format(tbl_item[0]["data"]))
276                     tbl_item.append({"data": None})
277                     continue
278             else:
279                 logging.error("Not supported command {0}. Skipping the table.".
280                               format(cmd))
281                 return None
282         tbl_lst.append(tbl_item)
283
284     # Sort the table according to the relative change
285     tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
286
287     # Create the tables and write them to the files
288     file_names = [
289         "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
290         "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
291         "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
292         "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
293     ]
294
295     for file_name in file_names:
296         logging.info("    Writing the file '{0}'".format(file_name))
297         with open(file_name, "w") as file_handler:
298             file_handler.write(",".join(header) + "\n")
299             for item in tbl_lst:
300                 if isinstance(item[-1]["data"], float):
301                     rel_change = round(item[-1]["data"], 1)
302                 else:
303                     rel_change = item[-1]["data"]
304                 if "ndr_top" in file_name \
305                         and "ndr" in item[0]["data"] \
306                         and rel_change >= 10.0:
307                     _write_line_to_file(file_handler, item)
308                 elif "pdr_top" in file_name \
309                         and "pdr" in item[0]["data"] \
310                         and rel_change >= 10.0:
311                     _write_line_to_file(file_handler, item)
312                 elif "ndr_low" in file_name \
313                         and "ndr" in item[0]["data"] \
314                         and rel_change < 10.0:
315                     _write_line_to_file(file_handler, item)
316                 elif "pdr_low" in file_name \
317                         and "pdr" in item[0]["data"] \
318                         and rel_change < 10.0:
319                     _write_line_to_file(file_handler, item)
320
321     logging.info("  Done.")
322
323
324 def _read_csv_template(file_name):
325     """Read the template from a .csv file.
326
327     :param file_name: Name / full path / relative path of the file to read.
328     :type file_name: str
329     :returns: Data from the template as list (lines) of lists (items on line).
330     :rtype: list
331     :raises: PresentationError if it is not possible to read the file.
332     """
333
334     try:
335         with open(file_name, 'r') as csv_file:
336             tmpl_data = list()
337             for line in csv_file:
338                 tmpl_data.append(line[:-1].split(","))
339         return tmpl_data
340     except IOError as err:
341         raise PresentationError(str(err), level="ERROR")
342
343
344 def table_performance_comparison(table, input_data):
345     """Generate the table(s) with algorithm: table_performance_comparison
346     specified in the specification file.
347
348     :param table: Table to generate.
349     :param input_data: Data to process.
350     :type table: pandas.Series
351     :type input_data: InputData
352     """
353
354     logging.info("  Generating the table {0} ...".
355                  format(table.get("title", "")))
356
357     # Transform the data
358     data = input_data.filter_data(table, continue_on_error=True)
359
360     # Prepare the header of the tables
361     try:
362         header = ["Test case",
363                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
364                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
365                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
366                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
367                   "Change [%]"]
368         header_str = ",".join(header) + "\n"
369     except (AttributeError, KeyError) as err:
370         logging.error("The model is invalid, missing parameter: {0}".
371                       format(err))
372         return
373
374     # Prepare data to the table:
375     tbl_dict = dict()
376     for job, builds in table["reference"]["data"].items():
377         for build in builds:
378             for tst_name, tst_data in data[job][str(build)].iteritems():
379                 if tbl_dict.get(tst_name, None) is None:
380                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
381                                             "-".join(tst_data["name"].
382                                                      split("-")[1:]))
383                     tbl_dict[tst_name] = {"name": name,
384                                           "ref-data": list(),
385                                           "cmp-data": list()}
386                 try:
387                     tbl_dict[tst_name]["ref-data"].\
388                         append(tst_data["throughput"]["value"])
389                 except TypeError:
390                     pass  # No data in output.xml for this test
391
392     for job, builds in table["compare"]["data"].items():
393         for build in builds:
394             for tst_name, tst_data in data[job][str(build)].iteritems():
395                 try:
396                     tbl_dict[tst_name]["cmp-data"].\
397                         append(tst_data["throughput"]["value"])
398                 except KeyError:
399                     pass
400                 except TypeError:
401                     tbl_dict.pop(tst_name, None)
402
403     tbl_lst = list()
404     for tst_name in tbl_dict.keys():
405         item = [tbl_dict[tst_name]["name"], ]
406         if tbl_dict[tst_name]["ref-data"]:
407             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
408                                      outlier_const=table["outlier-const"])
409             # TODO: Specify window size.
410             item.append(round(mean(data_t) / 1000000, 2))
411             item.append(round(stdev(data_t) / 1000000, 2))
412         else:
413             item.extend([None, None])
414         if tbl_dict[tst_name]["cmp-data"]:
415             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
416                                      outlier_const=table["outlier-const"])
417             # TODO: Specify window size.
418             item.append(round(mean(data_t) / 1000000, 2))
419             item.append(round(stdev(data_t) / 1000000, 2))
420         else:
421             item.extend([None, None])
422         if item[1] is not None and item[3] is not None:
423             item.append(int(relative_change(float(item[1]), float(item[3]))))
424         if len(item) == 6:
425             tbl_lst.append(item)
426
427     # Sort the table according to the relative change
428     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
429
430     # Generate tables:
431     # All tests in csv:
432     tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
433                                                table["output-file-ext"]),
434                  "{0}-ndr-2t2c-full{1}".format(table["output-file"],
435                                                table["output-file-ext"]),
436                  "{0}-ndr-4t4c-full{1}".format(table["output-file"],
437                                                table["output-file-ext"]),
438                  "{0}-pdr-1t1c-full{1}".format(table["output-file"],
439                                                table["output-file-ext"]),
440                  "{0}-pdr-2t2c-full{1}".format(table["output-file"],
441                                                table["output-file-ext"]),
442                  "{0}-pdr-4t4c-full{1}".format(table["output-file"],
443                                                table["output-file-ext"])
444                  ]
445     for file_name in tbl_names:
446         logging.info("      Writing file: '{0}'".format(file_name))
447         with open(file_name, "w") as file_handler:
448             file_handler.write(header_str)
449             for test in tbl_lst:
450                 if (file_name.split("-")[-3] in test[0] and    # NDR vs PDR
451                         file_name.split("-")[-2] in test[0]):  # cores
452                     test[0] = "-".join(test[0].split("-")[:-1])
453                     file_handler.write(",".join([str(item) for item in test]) +
454                                        "\n")
455
456     # All tests in txt:
457     tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
458                      "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
459                      "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
460                      "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
461                      "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
462                      "{0}-pdr-4t4c-full.txt".format(table["output-file"])
463                      ]
464
465     for i, txt_name in enumerate(tbl_names_txt):
466         txt_table = None
467         logging.info("      Writing file: '{0}'".format(txt_name))
468         with open(tbl_names[i], 'rb') as csv_file:
469             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
470             for row in csv_content:
471                 if txt_table is None:
472                     txt_table = prettytable.PrettyTable(row)
473                 else:
474                     txt_table.add_row(row)
475             txt_table.align["Test case"] = "l"
476         with open(txt_name, "w") as txt_file:
477             txt_file.write(str(txt_table))
478
479     # Selected tests in csv:
480     input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
481                                                table["output-file-ext"])
482     with open(input_file, "r") as in_file:
483         lines = list()
484         for line in in_file:
485             lines.append(line)
486
487     output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
488                                                table["output-file-ext"])
489     logging.info("      Writing file: '{0}'".format(output_file))
490     with open(output_file, "w") as out_file:
491         out_file.write(header_str)
492         for i, line in enumerate(lines[1:]):
493             if i == table["nr-of-tests-shown"]:
494                 break
495             out_file.write(line)
496
497     output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
498                                                   table["output-file-ext"])
499     logging.info("      Writing file: '{0}'".format(output_file))
500     with open(output_file, "w") as out_file:
501         out_file.write(header_str)
502         for i, line in enumerate(lines[-1:0:-1]):
503             if i == table["nr-of-tests-shown"]:
504                 break
505             out_file.write(line)
506
507     input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
508                                                table["output-file-ext"])
509     with open(input_file, "r") as in_file:
510         lines = list()
511         for line in in_file:
512             lines.append(line)
513
514     output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
515                                                table["output-file-ext"])
516     logging.info("      Writing file: '{0}'".format(output_file))
517     with open(output_file, "w") as out_file:
518         out_file.write(header_str)
519         for i, line in enumerate(lines[1:]):
520             if i == table["nr-of-tests-shown"]:
521                 break
522             out_file.write(line)
523
524     output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
525                                                   table["output-file-ext"])
526     logging.info("      Writing file: '{0}'".format(output_file))
527     with open(output_file, "w") as out_file:
528         out_file.write(header_str)
529         for i, line in enumerate(lines[-1:0:-1]):
530             if i == table["nr-of-tests-shown"]:
531                 break
532             out_file.write(line)
533
534
535 def table_performance_comparison_mrr(table, input_data):
536     """Generate the table(s) with algorithm: table_performance_comparison_mrr
537     specified in the specification file.
538
539     :param table: Table to generate.
540     :param input_data: Data to process.
541     :type table: pandas.Series
542     :type input_data: InputData
543     """
544
545     logging.info("  Generating the table {0} ...".
546                  format(table.get("title", "")))
547
548     # Transform the data
549     data = input_data.filter_data(table, continue_on_error=True)
550
551     # Prepare the header of the tables
552     try:
553         header = ["Test case",
554                   "{0} Throughput [Mpps]".format(table["reference"]["title"]),
555                   "{0} stdev [Mpps]".format(table["reference"]["title"]),
556                   "{0} Throughput [Mpps]".format(table["compare"]["title"]),
557                   "{0} stdev [Mpps]".format(table["compare"]["title"]),
558                   "Change [%]"]
559         header_str = ",".join(header) + "\n"
560     except (AttributeError, KeyError) as err:
561         logging.error("The model is invalid, missing parameter: {0}".
562                       format(err))
563         return
564
565     # Prepare data to the table:
566     tbl_dict = dict()
567     for job, builds in table["reference"]["data"].items():
568         for build in builds:
569             for tst_name, tst_data in data[job][str(build)].iteritems():
570                 if tbl_dict.get(tst_name, None) is None:
571                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
572                                             "-".join(tst_data["name"].
573                                                      split("-")[1:]))
574                     tbl_dict[tst_name] = {"name": name,
575                                           "ref-data": list(),
576                                           "cmp-data": list()}
577                 try:
578                     tbl_dict[tst_name]["ref-data"].\
579                         append(tst_data["result"]["throughput"])
580                 except TypeError:
581                     pass  # No data in output.xml for this test
582
583     for job, builds in table["compare"]["data"].items():
584         for build in builds:
585             for tst_name, tst_data in data[job][str(build)].iteritems():
586                 try:
587                     tbl_dict[tst_name]["cmp-data"].\
588                         append(tst_data["result"]["throughput"])
589                 except KeyError:
590                     pass
591                 except TypeError:
592                     tbl_dict.pop(tst_name, None)
593
594     tbl_lst = list()
595     for tst_name in tbl_dict.keys():
596         item = [tbl_dict[tst_name]["name"], ]
597         if tbl_dict[tst_name]["ref-data"]:
598             data_t = remove_outliers(tbl_dict[tst_name]["ref-data"],
599                                      outlier_const=table["outlier-const"])
600             # TODO: Specify window size.
601             item.append(round(mean(data_t) / 1000000, 2))
602             item.append(round(stdev(data_t) / 1000000, 2))
603         else:
604             item.extend([None, None])
605         if tbl_dict[tst_name]["cmp-data"]:
606             data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"],
607                                      outlier_const=table["outlier-const"])
608             # TODO: Specify window size.
609             item.append(round(mean(data_t) / 1000000, 2))
610             item.append(round(stdev(data_t) / 1000000, 2))
611         else:
612             item.extend([None, None])
613         if item[1] is not None and item[3] is not None and item[1] != 0:
614             item.append(int(relative_change(float(item[1]), float(item[3]))))
615         if len(item) == 6:
616             tbl_lst.append(item)
617
618     # Sort the table according to the relative change
619     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
620
621     # Generate tables:
622     # All tests in csv:
623     tbl_names = ["{0}-1t1c-full{1}".format(table["output-file"],
624                                            table["output-file-ext"]),
625                  "{0}-2t2c-full{1}".format(table["output-file"],
626                                            table["output-file-ext"]),
627                  "{0}-4t4c-full{1}".format(table["output-file"],
628                                            table["output-file-ext"])
629                  ]
630     for file_name in tbl_names:
631         logging.info("      Writing file: '{0}'".format(file_name))
632         with open(file_name, "w") as file_handler:
633             file_handler.write(header_str)
634             for test in tbl_lst:
635                 if file_name.split("-")[-2] in test[0]:  # cores
636                     test[0] = "-".join(test[0].split("-")[:-1])
637                     file_handler.write(",".join([str(item) for item in test]) +
638                                        "\n")
639
640     # All tests in txt:
641     tbl_names_txt = ["{0}-1t1c-full.txt".format(table["output-file"]),
642                      "{0}-2t2c-full.txt".format(table["output-file"]),
643                      "{0}-4t4c-full.txt".format(table["output-file"])
644                      ]
645
646     for i, txt_name in enumerate(tbl_names_txt):
647         txt_table = None
648         logging.info("      Writing file: '{0}'".format(txt_name))
649         with open(tbl_names[i], 'rb') as csv_file:
650             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
651             for row in csv_content:
652                 if txt_table is None:
653                     txt_table = prettytable.PrettyTable(row)
654                 else:
655                     txt_table.add_row(row)
656             txt_table.align["Test case"] = "l"
657         with open(txt_name, "w") as txt_file:
658             txt_file.write(str(txt_table))
659
660
661 def table_performance_trending_dashboard(table, input_data):
662     """Generate the table(s) with algorithm: table_performance_comparison
663     specified in the specification file.
664
665     :param table: Table to generate.
666     :param input_data: Data to process.
667     :type table: pandas.Series
668     :type input_data: InputData
669     """
670
671     logging.info("  Generating the table {0} ...".
672                  format(table.get("title", "")))
673
674     # Transform the data
675     data = input_data.filter_data(table, continue_on_error=True)
676
677     # Prepare the header of the tables
678     header = ["Test Case",
679               "Throughput Trend [Mpps]",
680               "Trend Compliance",
681               "Top Anomaly [Mpps]",
682               "Change [%]",
683               "Outliers [Number]"
684               ]
685     header_str = ",".join(header) + "\n"
686
687     # Prepare data to the table:
688     tbl_dict = dict()
689     for job, builds in table["data"].items():
690         for build in builds:
691             for tst_name, tst_data in data[job][str(build)].iteritems():
692                 if tbl_dict.get(tst_name, None) is None:
693                     name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
694                                             "-".join(tst_data["name"].
695                                                      split("-")[1:]))
696                     tbl_dict[tst_name] = {"name": name,
697                                           "data": dict()}
698                 try:
699                     tbl_dict[tst_name]["data"][str(build)] =  \
700                         tst_data["result"]["throughput"]
701                 except (TypeError, KeyError):
702                     pass  # No data in output.xml for this test
703
704     tbl_lst = list()
705     for tst_name in tbl_dict.keys():
706         if len(tbl_dict[tst_name]["data"]) > 2:
707
708             pd_data = pd.Series(tbl_dict[tst_name]["data"])
709             win_size = pd_data.size \
710                 if pd_data.size < table["window"] else table["window"]
711             # Test name:
712             name = tbl_dict[tst_name]["name"]
713
714             median = pd_data.rolling(window=win_size, min_periods=2).median()
715             trimmed_data, _ = split_outliers(pd_data, outlier_const=1.5,
716                                              window=win_size)
717             stdev_t = pd_data.rolling(window=win_size, min_periods=2).std()
718
719             rel_change_lst = [None, ]
720             classification_lst = [None, ]
721             median_lst = [None, ]
722             sample_lst = [None, ]
723             first = True
724             for build_nr, value in pd_data.iteritems():
725                 if first:
726                     first = False
727                     continue
728                 # Relative changes list:
729                 if not isnan(value) \
730                         and not isnan(median[build_nr]) \
731                         and median[build_nr] != 0:
732                     rel_change_lst.append(round(
733                         relative_change(float(median[build_nr]), float(value)),
734                         2))
735                 else:
736                     rel_change_lst.append(None)
737
738                 # Classification list:
739                 if isnan(trimmed_data[build_nr]) \
740                         or isnan(median[build_nr]) \
741                         or isnan(stdev_t[build_nr]) \
742                         or isnan(value):
743                     classification_lst.append("outlier")
744                 elif value < (median[build_nr] - 3 * stdev_t[build_nr]):
745                     classification_lst.append("regression")
746                 elif value > (median[build_nr] + 3 * stdev_t[build_nr]):
747                     classification_lst.append("progression")
748                 else:
749                     classification_lst.append("normal")
750                 sample_lst.append(value)
751                 median_lst.append(median[build_nr])
752
753             last_idx = len(classification_lst) - 1
754             first_idx = last_idx - int(table["evaluated-window"])
755             if first_idx < 0:
756                 first_idx = 0
757
758             nr_outliers = 0
759             consecutive_outliers = 0
760             failure = False
761             for item in classification_lst[first_idx:]:
762                 if item == "outlier":
763                     nr_outliers += 1
764                     consecutive_outliers += 1
765                     if consecutive_outliers == 3:
766                         failure = True
767                 else:
768                     consecutive_outliers = 0
769
770             if failure:
771                 classification = "failure"
772             elif "regression" in classification_lst[first_idx:]:
773                 classification = "regression"
774             elif "progression" in classification_lst[first_idx:]:
775                 classification = "progression"
776             else:
777                 classification = "normal"
778
779             if classification == "normal":
780                 index = len(classification_lst) - 1
781             else:
782                 tmp_classification = "outlier" if classification == "failure" \
783                     else classification
784                 for idx in range(first_idx, len(classification_lst)):
785                     if classification_lst[idx] == tmp_classification:
786                         index = idx
787                         break
788                 for idx in range(index+1, len(classification_lst)):
789                     if classification_lst[idx] == tmp_classification:
790                         if rel_change_lst[idx] > rel_change_lst[index]:
791                             index = idx
792
793             # if "regression" in classification_lst[first_idx:]:
794             #     classification = "regression"
795             # elif "outlier" in classification_lst[first_idx:]:
796             #     classification = "outlier"
797             # elif "progression" in classification_lst[first_idx:]:
798             #     classification = "progression"
799             # elif "normal" in classification_lst[first_idx:]:
800             #     classification = "normal"
801             # else:
802             #     classification = None
803             #
804             # nr_outliers = 0
805             # consecutive_outliers = 0
806             # failure = False
807             # for item in classification_lst[first_idx:]:
808             #     if item == "outlier":
809             #         nr_outliers += 1
810             #         consecutive_outliers += 1
811             #         if consecutive_outliers == 3:
812             #             failure = True
813             #     else:
814             #         consecutive_outliers = 0
815             #
816             # idx = len(classification_lst) - 1
817             # while idx:
818             #     if classification_lst[idx] == classification:
819             #         break
820             #     idx -= 1
821             #
822             # if failure:
823             #     classification = "failure"
824             # elif classification == "outlier":
825             #     classification = "normal"
826
827             trend = round(float(median_lst[-1]) / 1000000, 2) \
828                 if not isnan(median_lst[-1]) else ''
829             sample = round(float(sample_lst[index]) / 1000000, 2) \
830                 if not isnan(sample_lst[index]) else ''
831             rel_change = rel_change_lst[index] \
832                 if rel_change_lst[index] is not None else ''
833             tbl_lst.append([name,
834                             trend,
835                             classification,
836                             '-' if classification == "normal" else sample,
837                             '-' if classification == "normal" else rel_change,
838                             nr_outliers])
839
840     # Sort the table according to the classification
841     tbl_sorted = list()
842     for classification in ("failure", "regression", "progression", "normal"):
843         tbl_tmp = [item for item in tbl_lst if item[2] == classification]
844         tbl_tmp.sort(key=lambda rel: rel[0])
845         tbl_sorted.extend(tbl_tmp)
846
847     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
848
849     logging.info("      Writing file: '{0}'".format(file_name))
850     with open(file_name, "w") as file_handler:
851         file_handler.write(header_str)
852         for test in tbl_sorted:
853             file_handler.write(",".join([str(item) for item in test]) + '\n')
854
855     txt_file_name = "{0}.txt".format(table["output-file"])
856     txt_table = None
857     logging.info("      Writing file: '{0}'".format(txt_file_name))
858     with open(file_name, 'rb') as csv_file:
859         csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
860         for row in csv_content:
861             if txt_table is None:
862                 txt_table = prettytable.PrettyTable(row)
863             else:
864                 txt_table.add_row(row)
865         txt_table.align["Test case"] = "l"
866     with open(txt_file_name, "w") as txt_file:
867         txt_file.write(str(txt_table))
868
869
870 def table_performance_trending_dashboard_html(table, input_data):
871     """Generate the table(s) with algorithm:
872     table_performance_trending_dashboard_html specified in the specification
873     file.
874
875     :param table: Table to generate.
876     :param input_data: Data to process.
877     :type table: pandas.Series
878     :type input_data: InputData
879     """
880
881     logging.info("  Generating the table {0} ...".
882                  format(table.get("title", "")))
883
884     try:
885         with open(table["input-file"], 'rb') as csv_file:
886             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
887             csv_lst = [item for item in csv_content]
888     except KeyError:
889         logging.warning("The input file is not defined.")
890         return
891     except csv.Error as err:
892         logging.warning("Not possible to process the file '{0}'.\n{1}".
893                         format(table["input-file"], err))
894         return
895
896     # Table:
897     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
898
899     # Table header:
900     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
901     for idx, item in enumerate(csv_lst[0]):
902         alignment = "left" if idx == 0 else "center"
903         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
904         th.text = item
905
906     # Rows:
907     for r_idx, row in enumerate(csv_lst[1:]):
908         background = "#D4E4F7" if r_idx % 2 else "white"
909         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
910
911         # Columns:
912         for c_idx, item in enumerate(row):
913             alignment = "left" if c_idx == 0 else "center"
914             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
915             # Name:
916             url = "../trending/"
917             file_name = ""
918             anchor = "#"
919             feature = ""
920             if c_idx == 0:
921                 if "memif" in item:
922                     file_name = "container_memif.html"
923
924                 elif "vhost" in item:
925                     if "l2xcbase" in item or "l2bdbasemaclrn" in item:
926                         file_name = "vm_vhost_l2.html"
927                     elif "ip4base" in item:
928                         file_name = "vm_vhost_ip4.html"
929
930                 elif "ipsec" in item:
931                     file_name = "ipsec.html"
932
933                 elif "ethip4lispip" in item or "ethip4vxlan" in item:
934                     file_name = "ip4_tunnels.html"
935
936                 elif "ip4base" in item or "ip4scale" in item:
937                     file_name = "ip4.html"
938                     if "iacl" in item or "snat" in item or "cop" in item:
939                         feature = "-features"
940
941                 elif "ip6base" in item or "ip6scale" in item:
942                     file_name = "ip6.html"
943
944                 elif "l2xcbase" in item or "l2xcscale" in item \
945                         or "l2bdbasemaclrn" in item or "l2bdscale" in item \
946                         or "l2dbbasemaclrn" in item or "l2dbscale" in item:
947                     file_name = "l2.html"
948                     if "iacl" in item:
949                         feature = "-features"
950
951                 if "x520" in item:
952                     anchor += "x520-"
953                 elif "x710" in item:
954                     anchor += "x710-"
955                 elif "xl710" in item:
956                     anchor += "xl710-"
957
958                 if "64b" in item:
959                     anchor += "64b-"
960                 elif "78b" in item:
961                     anchor += "78b"
962                 elif "imix" in item:
963                     anchor += "imix-"
964                 elif "9000b" in item:
965                     anchor += "9000b-"
966                 elif "1518" in item:
967                     anchor += "1518b-"
968
969                 if "1t1c" in item:
970                     anchor += "1t1c"
971                 elif "2t2c" in item:
972                     anchor += "2t2c"
973                 elif "4t4c" in item:
974                     anchor += "4t4c"
975
976                 url = url + file_name + anchor + feature
977
978                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
979                 ref.text = item
980
981             if c_idx == 2:
982                 if item == "regression":
983                     td.set("bgcolor", "#eca1a6")
984                 elif item == "failure":
985                     td.set("bgcolor", "#d6cbd3")
986                 elif item == "progression":
987                     td.set("bgcolor", "#bdcebe")
988             if c_idx > 0:
989                 td.text = item
990
991     try:
992         with open(table["output-file"], 'w') as html_file:
993             logging.info("      Writing file: '{0}'".
994                          format(table["output-file"]))
995             html_file.write(".. raw:: html\n\n\t")
996             html_file.write(ET.tostring(dashboard))
997             html_file.write("\n\t<p><br><br></p>\n")
998     except KeyError:
999         logging.warning("The output file is not defined.")
1000         return

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.