1 # Copyright (c) 2018 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Algorithms to generate tables.
22 from string import replace
23 from collections import OrderedDict
24 from numpy import nan, isnan
25 from xml.etree import ElementTree as ET
27 from errors import PresentationError
28 from utils import mean, stdev, relative_change, classify_anomalies, \
29 convert_csv_to_pretty_txt
32 def generate_tables(spec, data):
33 """Generate all tables specified in the specification file.
35 :param spec: Specification read from the specification file.
36 :param data: Data to process.
37 :type spec: Specification
41 logging.info("Generating the tables ...")
42 for table in spec.tables:
44 eval(table["algorithm"])(table, data)
45 except NameError as err:
46 logging.error("Probably algorithm '{alg}' is not defined: {err}".
47 format(alg=table["algorithm"], err=repr(err)))
51 def table_details(table, input_data):
52 """Generate the table(s) with algorithm: table_detailed_test_results
53 specified in the specification file.
55 :param table: Table to generate.
56 :param input_data: Data to process.
57 :type table: pandas.Series
58 :type input_data: InputData
61 logging.info(" Generating the table {0} ...".
62 format(table.get("title", "")))
65 logging.info(" Creating the data set for the {0} '{1}'.".
66 format(table.get("type", ""), table.get("title", "")))
67 data = input_data.filter_data(table)
69 # Prepare the header of the tables
71 for column in table["columns"]:
72 header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
74 # Generate the data for the table according to the model in the table
76 job = table["data"].keys()[0]
77 build = str(table["data"][job][0])
79 suites = input_data.suites(job, build)
81 logging.error(" No data available. The table will not be generated.")
84 for suite_longname, suite in suites.iteritems():
86 suite_name = suite["name"]
88 for test in data[job][build].keys():
89 if data[job][build][test]["parent"] in suite_name:
91 for column in table["columns"]:
93 col_data = str(data[job][build][test][column["data"].
94 split(" ")[1]]).replace('"', '""')
95 if column["data"].split(" ")[1] in ("vat-history",
97 col_data = replace(col_data, " |br| ", "",
99 col_data = " |prein| {0} |preout| ".\
100 format(col_data[:-5])
101 row_lst.append('"{0}"'.format(col_data))
103 row_lst.append("No data")
104 table_lst.append(row_lst)
106 # Write the data to file
108 file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
109 table["output-file-ext"])
110 logging.info(" Writing file: '{}'".format(file_name))
111 with open(file_name, "w") as file_handler:
112 file_handler.write(",".join(header) + "\n")
113 for item in table_lst:
114 file_handler.write(",".join(item) + "\n")
116 logging.info(" Done.")
119 def table_merged_details(table, input_data):
120 """Generate the table(s) with algorithm: table_merged_details
121 specified in the specification file.
123 :param table: Table to generate.
124 :param input_data: Data to process.
125 :type table: pandas.Series
126 :type input_data: InputData
129 logging.info(" Generating the table {0} ...".
130 format(table.get("title", "")))
133 logging.info(" Creating the data set for the {0} '{1}'.".
134 format(table.get("type", ""), table.get("title", "")))
135 data = input_data.filter_data(table)
136 data = input_data.merge_data(data)
137 data.sort_index(inplace=True)
139 logging.info(" Creating the data set for the {0} '{1}'.".
140 format(table.get("type", ""), table.get("title", "")))
141 suites = input_data.filter_data(table, data_set="suites")
142 suites = input_data.merge_data(suites)
144 # Prepare the header of the tables
146 for column in table["columns"]:
147 header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
149 for _, suite in suites.iteritems():
151 suite_name = suite["name"]
153 for test in data.keys():
154 if data[test]["parent"] in suite_name:
156 for column in table["columns"]:
158 col_data = str(data[test][column["data"].
159 split(" ")[1]]).replace('"', '""')
160 if column["data"].split(" ")[1] in ("vat-history",
162 col_data = replace(col_data, " |br| ", "",
164 col_data = " |prein| {0} |preout| ".\
165 format(col_data[:-5])
166 row_lst.append('"{0}"'.format(col_data))
168 row_lst.append("No data")
169 table_lst.append(row_lst)
171 # Write the data to file
173 file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
174 table["output-file-ext"])
175 logging.info(" Writing file: '{}'".format(file_name))
176 with open(file_name, "w") as file_handler:
177 file_handler.write(",".join(header) + "\n")
178 for item in table_lst:
179 file_handler.write(",".join(item) + "\n")
181 logging.info(" Done.")
184 def table_performance_improvements(table, input_data):
185 """Generate the table(s) with algorithm: table_performance_improvements
186 specified in the specification file.
188 :param table: Table to generate.
189 :param input_data: Data to process.
190 :type table: pandas.Series
191 :type input_data: InputData
194 def _write_line_to_file(file_handler, data):
195 """Write a line to the .csv file.
197 :param file_handler: File handler for the csv file. It must be open for
199 :param data: Item to be written to the file.
200 :type file_handler: BinaryIO
206 if isinstance(item["data"], str):
207 # Remove -?drdisc from the end
208 if item["data"].endswith("drdisc"):
209 item["data"] = item["data"][:-8]
210 line_lst.append(item["data"])
211 elif isinstance(item["data"], float):
212 line_lst.append("{:.1f}".format(item["data"]))
213 elif item["data"] is None:
215 file_handler.write(",".join(line_lst) + "\n")
217 logging.info(" Generating the table {0} ...".
218 format(table.get("title", "")))
221 file_name = table.get("template", None)
224 tmpl = _read_csv_template(file_name)
225 except PresentationError:
226 logging.error(" The template '{0}' does not exist. Skipping the "
227 "table.".format(file_name))
230 logging.error("The template is not defined. Skipping the table.")
234 logging.info(" Creating the data set for the {0} '{1}'.".
235 format(table.get("type", ""), table.get("title", "")))
236 data = input_data.filter_data(table)
238 # Prepare the header of the tables
240 for column in table["columns"]:
241 header.append(column["title"])
243 # Generate the data for the table according to the model in the table
246 for tmpl_item in tmpl:
248 for column in table["columns"]:
249 cmd = column["data"].split(" ")[0]
250 args = column["data"].split(" ")[1:]
251 if cmd == "template":
253 val = float(tmpl_item[int(args[0])])
255 val = tmpl_item[int(args[0])]
256 tbl_item.append({"data": val})
262 for build in data[job]:
264 data_lst.append(float(build[tmpl_item[0]]
265 ["throughput"]["value"]))
266 except (KeyError, TypeError):
270 tbl_item.append({"data": (eval(operation)(data_lst)) /
273 tbl_item.append({"data": None})
274 elif cmd == "operation":
277 nr1 = float(tbl_item[int(args[1])]["data"])
278 nr2 = float(tbl_item[int(args[2])]["data"])
280 tbl_item.append({"data": eval(operation)(nr1, nr2)})
282 tbl_item.append({"data": None})
283 except (IndexError, ValueError, TypeError):
284 logging.error("No data for {0}".format(tbl_item[0]["data"]))
285 tbl_item.append({"data": None})
288 logging.error("Not supported command {0}. Skipping the table.".
291 tbl_lst.append(tbl_item)
293 # Sort the table according to the relative change
294 tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True)
296 # Create the tables and write them to the files
298 "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]),
299 "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]),
300 "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]),
301 "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"])
304 for file_name in file_names:
305 logging.info(" Writing the file '{0}'".format(file_name))
306 with open(file_name, "w") as file_handler:
307 file_handler.write(",".join(header) + "\n")
309 if isinstance(item[-1]["data"], float):
310 rel_change = round(item[-1]["data"], 1)
312 rel_change = item[-1]["data"]
313 if "ndr_top" in file_name \
314 and "ndr" in item[0]["data"] \
315 and rel_change >= 10.0:
316 _write_line_to_file(file_handler, item)
317 elif "pdr_top" in file_name \
318 and "pdr" in item[0]["data"] \
319 and rel_change >= 10.0:
320 _write_line_to_file(file_handler, item)
321 elif "ndr_low" in file_name \
322 and "ndr" in item[0]["data"] \
323 and rel_change < 10.0:
324 _write_line_to_file(file_handler, item)
325 elif "pdr_low" in file_name \
326 and "pdr" in item[0]["data"] \
327 and rel_change < 10.0:
328 _write_line_to_file(file_handler, item)
330 logging.info(" Done.")
333 def _read_csv_template(file_name):
334 """Read the template from a .csv file.
336 :param file_name: Name / full path / relative path of the file to read.
338 :returns: Data from the template as list (lines) of lists (items on line).
340 :raises: PresentationError if it is not possible to read the file.
344 with open(file_name, 'r') as csv_file:
346 for line in csv_file:
347 tmpl_data.append(line[:-1].split(","))
349 except IOError as err:
350 raise PresentationError(str(err), level="ERROR")
353 def table_performance_comparison(table, input_data):
354 """Generate the table(s) with algorithm: table_performance_comparison
355 specified in the specification file.
357 :param table: Table to generate.
358 :param input_data: Data to process.
359 :type table: pandas.Series
360 :type input_data: InputData
363 logging.info(" Generating the table {0} ...".
364 format(table.get("title", "")))
367 logging.info(" Creating the data set for the {0} '{1}'.".
368 format(table.get("type", ""), table.get("title", "")))
369 data = input_data.filter_data(table, continue_on_error=True)
371 # Prepare the header of the tables
373 header = ["Test case", ]
375 history = table.get("history", None)
379 ["{0} Throughput [Mpps]".format(item["title"]),
380 "{0} Stdev [Mpps]".format(item["title"])])
382 ["{0} Throughput [Mpps]".format(table["reference"]["title"]),
383 "{0} Stdev [Mpps]".format(table["reference"]["title"]),
384 "{0} Throughput [Mpps]".format(table["compare"]["title"]),
385 "{0} Stdev [Mpps]".format(table["compare"]["title"]),
387 header_str = ",".join(header) + "\n"
388 except (AttributeError, KeyError) as err:
389 logging.error("The model is invalid, missing parameter: {0}".
393 # Prepare data to the table:
395 for job, builds in table["reference"]["data"].items():
397 for tst_name, tst_data in data[job][str(build)].iteritems():
398 if tbl_dict.get(tst_name, None) is None:
399 name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
400 "-".join(tst_data["name"].
402 tbl_dict[tst_name] = {"name": name,
406 tbl_dict[tst_name]["ref-data"].\
407 append(tst_data["throughput"]["value"])
409 pass # No data in output.xml for this test
411 for job, builds in table["compare"]["data"].items():
413 for tst_name, tst_data in data[job][str(build)].iteritems():
415 tbl_dict[tst_name]["cmp-data"].\
416 append(tst_data["throughput"]["value"])
420 tbl_dict.pop(tst_name, None)
423 for job, builds in item["data"].items():
425 for tst_name, tst_data in data[job][str(build)].iteritems():
426 if tbl_dict.get(tst_name, None) is None:
428 if tbl_dict[tst_name].get("history", None) is None:
429 tbl_dict[tst_name]["history"] = OrderedDict()
430 if tbl_dict[tst_name]["history"].get(item["title"],
432 tbl_dict[tst_name]["history"][item["title"]] = \
435 tbl_dict[tst_name]["history"][item["title"]].\
436 append(tst_data["throughput"]["value"])
437 except (TypeError, KeyError):
441 for tst_name in tbl_dict.keys():
442 item = [tbl_dict[tst_name]["name"], ]
444 if tbl_dict[tst_name].get("history", None) is not None:
445 for hist_data in tbl_dict[tst_name]["history"].values():
447 item.append(round(mean(hist_data) / 1000000, 2))
448 item.append(round(stdev(hist_data) / 1000000, 2))
450 item.extend([None, None])
452 item.extend([None, None])
453 data_t = tbl_dict[tst_name]["ref-data"]
455 item.append(round(mean(data_t) / 1000000, 2))
456 item.append(round(stdev(data_t) / 1000000, 2))
458 item.extend([None, None])
459 data_t = tbl_dict[tst_name]["cmp-data"]
461 item.append(round(mean(data_t) / 1000000, 2))
462 item.append(round(stdev(data_t) / 1000000, 2))
464 item.extend([None, None])
465 if item[-4] is not None and item[-2] is not None and item[-4] != 0:
466 item.append(int(relative_change(float(item[-4]), float(item[-2]))))
467 if len(item) == len(header):
470 # Sort the table according to the relative change
471 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
475 tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"],
476 table["output-file-ext"]),
477 "{0}-ndr-2t2c-full{1}".format(table["output-file"],
478 table["output-file-ext"]),
479 "{0}-ndr-4t4c-full{1}".format(table["output-file"],
480 table["output-file-ext"]),
481 "{0}-pdr-1t1c-full{1}".format(table["output-file"],
482 table["output-file-ext"]),
483 "{0}-pdr-2t2c-full{1}".format(table["output-file"],
484 table["output-file-ext"]),
485 "{0}-pdr-4t4c-full{1}".format(table["output-file"],
486 table["output-file-ext"])
488 for file_name in tbl_names:
489 logging.info(" Writing file: '{0}'".format(file_name))
490 with open(file_name, "w") as file_handler:
491 file_handler.write(header_str)
493 if (file_name.split("-")[-3] in test[0] and # NDR vs PDR
494 file_name.split("-")[-2] in test[0]): # cores
495 test[0] = "-".join(test[0].split("-")[:-1])
496 file_handler.write(",".join([str(item) for item in test]) +
500 tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]),
501 "{0}-ndr-2t2c-full.txt".format(table["output-file"]),
502 "{0}-ndr-4t4c-full.txt".format(table["output-file"]),
503 "{0}-pdr-1t1c-full.txt".format(table["output-file"]),
504 "{0}-pdr-2t2c-full.txt".format(table["output-file"]),
505 "{0}-pdr-4t4c-full.txt".format(table["output-file"])
508 for i, txt_name in enumerate(tbl_names_txt):
509 logging.info(" Writing file: '{0}'".format(txt_name))
510 convert_csv_to_pretty_txt(tbl_names[i], txt_name)
512 # Selected tests in csv:
513 input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
514 table["output-file-ext"])
515 with open(input_file, "r") as in_file:
520 output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
521 table["output-file-ext"])
522 logging.info(" Writing file: '{0}'".format(output_file))
523 with open(output_file, "w") as out_file:
524 out_file.write(header_str)
525 for i, line in enumerate(lines[1:]):
526 if i == table["nr-of-tests-shown"]:
530 output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
531 table["output-file-ext"])
532 logging.info(" Writing file: '{0}'".format(output_file))
533 with open(output_file, "w") as out_file:
534 out_file.write(header_str)
535 for i, line in enumerate(lines[-1:0:-1]):
536 if i == table["nr-of-tests-shown"]:
540 input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
541 table["output-file-ext"])
542 with open(input_file, "r") as in_file:
547 output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
548 table["output-file-ext"])
549 logging.info(" Writing file: '{0}'".format(output_file))
550 with open(output_file, "w") as out_file:
551 out_file.write(header_str)
552 for i, line in enumerate(lines[1:]):
553 if i == table["nr-of-tests-shown"]:
557 output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
558 table["output-file-ext"])
559 logging.info(" Writing file: '{0}'".format(output_file))
560 with open(output_file, "w") as out_file:
561 out_file.write(header_str)
562 for i, line in enumerate(lines[-1:0:-1]):
563 if i == table["nr-of-tests-shown"]:
568 def table_performance_comparison_mrr(table, input_data):
569 """Generate the table(s) with algorithm: table_performance_comparison_mrr
570 specified in the specification file.
572 :param table: Table to generate.
573 :param input_data: Data to process.
574 :type table: pandas.Series
575 :type input_data: InputData
578 logging.info(" Generating the table {0} ...".
579 format(table.get("title", "")))
582 logging.info(" Creating the data set for the {0} '{1}'.".
583 format(table.get("type", ""), table.get("title", "")))
584 data = input_data.filter_data(table, continue_on_error=True)
586 # Prepare the header of the tables
588 header = ["Test case",
589 "{0} Throughput [Mpps]".format(table["reference"]["title"]),
590 "{0} stdev [Mpps]".format(table["reference"]["title"]),
591 "{0} Throughput [Mpps]".format(table["compare"]["title"]),
592 "{0} stdev [Mpps]".format(table["compare"]["title"]),
594 header_str = ",".join(header) + "\n"
595 except (AttributeError, KeyError) as err:
596 logging.error("The model is invalid, missing parameter: {0}".
600 # Prepare data to the table:
602 for job, builds in table["reference"]["data"].items():
604 for tst_name, tst_data in data[job][str(build)].iteritems():
605 if tbl_dict.get(tst_name, None) is None:
606 name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
607 "-".join(tst_data["name"].
609 tbl_dict[tst_name] = {"name": name,
613 tbl_dict[tst_name]["ref-data"].\
614 append(tst_data["result"]["throughput"])
616 pass # No data in output.xml for this test
618 for job, builds in table["compare"]["data"].items():
620 for tst_name, tst_data in data[job][str(build)].iteritems():
622 tbl_dict[tst_name]["cmp-data"].\
623 append(tst_data["result"]["throughput"])
627 tbl_dict.pop(tst_name, None)
630 for tst_name in tbl_dict.keys():
631 item = [tbl_dict[tst_name]["name"], ]
632 data_t = tbl_dict[tst_name]["ref-data"]
634 item.append(round(mean(data_t) / 1000000, 2))
635 item.append(round(stdev(data_t) / 1000000, 2))
637 item.extend([None, None])
638 data_t = tbl_dict[tst_name]["cmp-data"]
640 item.append(round(mean(data_t) / 1000000, 2))
641 item.append(round(stdev(data_t) / 1000000, 2))
643 item.extend([None, None])
644 if item[1] is not None and item[3] is not None and item[1] != 0:
645 item.append(int(relative_change(float(item[1]), float(item[3]))))
649 # Sort the table according to the relative change
650 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
654 tbl_names = ["{0}-1t1c-full{1}".format(table["output-file"],
655 table["output-file-ext"]),
656 "{0}-2t2c-full{1}".format(table["output-file"],
657 table["output-file-ext"]),
658 "{0}-4t4c-full{1}".format(table["output-file"],
659 table["output-file-ext"])
661 for file_name in tbl_names:
662 logging.info(" Writing file: '{0}'".format(file_name))
663 with open(file_name, "w") as file_handler:
664 file_handler.write(header_str)
666 if file_name.split("-")[-2] in test[0]: # cores
667 test[0] = "-".join(test[0].split("-")[:-1])
668 file_handler.write(",".join([str(item) for item in test]) +
672 tbl_names_txt = ["{0}-1t1c-full.txt".format(table["output-file"]),
673 "{0}-2t2c-full.txt".format(table["output-file"]),
674 "{0}-4t4c-full.txt".format(table["output-file"])
677 for i, txt_name in enumerate(tbl_names_txt):
678 logging.info(" Writing file: '{0}'".format(txt_name))
679 convert_csv_to_pretty_txt(tbl_names[i], txt_name)
682 def table_performance_trending_dashboard(table, input_data):
683 """Generate the table(s) with algorithm:
684 table_performance_trending_dashboard
685 specified in the specification file.
687 :param table: Table to generate.
688 :param input_data: Data to process.
689 :type table: pandas.Series
690 :type input_data: InputData
693 logging.info(" Generating the table {0} ...".
694 format(table.get("title", "")))
697 logging.info(" Creating the data set for the {0} '{1}'.".
698 format(table.get("type", ""), table.get("title", "")))
699 data = input_data.filter_data(table, continue_on_error=True)
701 # Prepare the header of the tables
702 header = ["Test Case",
704 "Short-Term Change [%]",
705 "Long-Term Change [%]",
709 header_str = ",".join(header) + "\n"
711 # Prepare data to the table:
713 for job, builds in table["data"].items():
715 for tst_name, tst_data in data[job][str(build)].iteritems():
716 if tst_name.lower() in table["ignore-list"]:
718 if tbl_dict.get(tst_name, None) is None:
719 name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
720 "-".join(tst_data["name"].
722 tbl_dict[tst_name] = {"name": name,
723 "data": OrderedDict()}
725 tbl_dict[tst_name]["data"][str(build)] = \
726 tst_data["result"]["throughput"]
727 except (TypeError, KeyError):
728 pass # No data in output.xml for this test
731 for tst_name in tbl_dict.keys():
732 if len(tbl_dict[tst_name]["data"]) < 2:
735 data_t = pd.Series(tbl_dict[tst_name]["data"])
737 classification_lst, avgs = classify_anomalies(data_t)
739 win_size = min(data_t.size, table["window"])
740 long_win_size = min(data_t.size, table["long-trend-window"])
743 [x for x in avgs[-long_win_size:-win_size]
748 avg_week_ago = avgs[max(-win_size, -len(avgs))]
750 if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
751 rel_change_last = nan
753 rel_change_last = round(
754 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
756 if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
757 rel_change_long = nan
759 rel_change_long = round(
760 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
762 if classification_lst:
763 if isnan(rel_change_last) and isnan(rel_change_long):
766 [tbl_dict[tst_name]["name"],
767 '-' if isnan(last_avg) else
768 round(last_avg / 1000000, 2),
769 '-' if isnan(rel_change_last) else rel_change_last,
770 '-' if isnan(rel_change_long) else rel_change_long,
771 classification_lst[-win_size:].count("regression"),
772 classification_lst[-win_size:].count("progression")])
774 tbl_lst.sort(key=lambda rel: rel[0])
777 for nrr in range(table["window"], -1, -1):
778 tbl_reg = [item for item in tbl_lst if item[4] == nrr]
779 for nrp in range(table["window"], -1, -1):
780 tbl_out = [item for item in tbl_reg if item[5] == nrp]
781 tbl_out.sort(key=lambda rel: rel[2])
782 tbl_sorted.extend(tbl_out)
784 file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
786 logging.info(" Writing file: '{0}'".format(file_name))
787 with open(file_name, "w") as file_handler:
788 file_handler.write(header_str)
789 for test in tbl_sorted:
790 file_handler.write(",".join([str(item) for item in test]) + '\n')
792 txt_file_name = "{0}.txt".format(table["output-file"])
793 logging.info(" Writing file: '{0}'".format(txt_file_name))
794 convert_csv_to_pretty_txt(file_name, txt_file_name)
797 def _generate_url(base, test_name):
798 """Generate URL to a trending plot from the name of the test case.
800 :param base: The base part of URL common to all test cases.
801 :param test_name: The name of the test case.
804 :returns: The URL to the plot with the trending data for the given test
814 if "lbdpdk" in test_name or "lbvpp" in test_name:
815 file_name = "link_bonding.html"
817 elif "testpmd" in test_name or "l3fwd" in test_name:
818 file_name = "dpdk.html"
820 elif "memif" in test_name:
821 file_name = "container_memif.html"
823 elif "srv6" in test_name:
824 file_name = "srv6.html"
826 elif "vhost" in test_name:
827 if "l2xcbase" in test_name or "l2bdbasemaclrn" in test_name:
828 file_name = "vm_vhost_l2.html"
829 elif "ip4base" in test_name:
830 file_name = "vm_vhost_ip4.html"
832 elif "ipsec" in test_name:
833 file_name = "ipsec.html"
835 elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name:
836 file_name = "ip4_tunnels.html"
838 elif "ip4base" in test_name or "ip4scale" in test_name:
839 file_name = "ip4.html"
840 if "iacl" in test_name or "snat" in test_name or "cop" in test_name:
841 feature = "-features"
843 elif "ip6base" in test_name or "ip6scale" in test_name:
844 file_name = "ip6.html"
846 elif "l2xcbase" in test_name or "l2xcscale" in test_name \
847 or "l2bdbasemaclrn" in test_name or "l2bdscale" in test_name \
848 or "l2dbbasemaclrn" in test_name or "l2dbscale" in test_name:
849 file_name = "l2.html"
850 if "iacl" in test_name:
851 feature = "-features"
853 if "x520" in test_name:
855 elif "x710" in test_name:
857 elif "xl710" in test_name:
860 if "64b" in test_name:
862 elif "78b" in test_name:
864 elif "imix" in test_name:
866 elif "9000b" in test_name:
868 elif "1518" in test_name:
871 if "1t1c" in test_name:
873 elif "2t2c" in test_name:
875 elif "4t4c" in test_name:
878 return url + file_name + anchor + feature
881 def table_performance_trending_dashboard_html(table, input_data):
882 """Generate the table(s) with algorithm:
883 table_performance_trending_dashboard_html specified in the specification
886 :param table: Table to generate.
887 :param input_data: Data to process.
888 :type table: pandas.Series
889 :type input_data: InputData
892 logging.info(" Generating the table {0} ...".
893 format(table.get("title", "")))
896 with open(table["input-file"], 'rb') as csv_file:
897 csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
898 csv_lst = [item for item in csv_content]
900 logging.warning("The input file is not defined.")
902 except csv.Error as err:
903 logging.warning("Not possible to process the file '{0}'.\n{1}".
904 format(table["input-file"], err))
908 dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
911 tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
912 for idx, item in enumerate(csv_lst[0]):
913 alignment = "left" if idx == 0 else "center"
914 th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
918 colors = {"regression": ("#ffcccc", "#ff9999"),
919 "progression": ("#c6ecc6", "#9fdf9f"),
920 "normal": ("#e9f1fb", "#d4e4f7")}
921 for r_idx, row in enumerate(csv_lst[1:]):
925 color = "progression"
928 background = colors[color][r_idx % 2]
929 tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
932 for c_idx, item in enumerate(row):
933 alignment = "left" if c_idx == 0 else "center"
934 td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
937 url = _generate_url("../trending/", item)
938 ref = ET.SubElement(td, "a", attrib=dict(href=url))
943 with open(table["output-file"], 'w') as html_file:
944 logging.info(" Writing file: '{0}'".format(table["output-file"]))
945 html_file.write(".. raw:: html\n\n\t")
946 html_file.write(ET.tostring(dashboard))
947 html_file.write("\n\t<p><br><br></p>\n")
949 logging.warning("The output file is not defined.")
953 def table_failed_tests(table, input_data):
954 """Generate the table(s) with algorithm: table_failed_tests
955 specified in the specification file.
957 :param table: Table to generate.
958 :param input_data: Data to process.
959 :type table: pandas.Series
960 :type input_data: InputData
963 logging.info(" Generating the table {0} ...".
964 format(table.get("title", "")))
967 logging.info(" Creating the data set for the {0} '{1}'.".
968 format(table.get("type", ""), table.get("title", "")))
969 data = input_data.filter_data(table, continue_on_error=True)
971 # Prepare the header of the tables
972 header = ["Test Case",
974 "Last Fail [Timestamp]",
975 "Last Fail [VPP Build]",
976 "Last Fail [CSIT Build]"]
978 # Generate the data for the table according to the model in the table
981 for job, builds in table["data"].items():
984 for tst_name, tst_data in data[job][build].iteritems():
985 if tst_name.lower() in table["ignore-list"]:
987 if tbl_dict.get(tst_name, None) is None:
988 name = "{0}-{1}".format(tst_data["parent"].split("-")[0],
989 "-".join(tst_data["name"].
991 tbl_dict[tst_name] = {"name": name,
992 "data": OrderedDict()}
994 tbl_dict[tst_name]["data"][build] = (
996 input_data.metadata(job, build).get("generated", ""),
997 input_data.metadata(job, build).get("version", ""),
999 except (TypeError, KeyError):
1000 pass # No data in output.xml for this test
1003 for tst_data in tbl_dict.values():
1004 win_size = min(len(tst_data["data"]), table["window"])
1006 for val in tst_data["data"].values()[-win_size:]:
1007 if val[0] == "FAIL":
1009 fails_last_date = val[1]
1010 fails_last_vpp = val[2]
1011 fails_last_csit = val[3]
1013 tbl_lst.append([tst_data["name"],
1017 "mrr-daily-build-{0}".format(fails_last_csit)])
1019 tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1021 for nrf in range(table["window"], -1, -1):
1022 tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1023 tbl_sorted.extend(tbl_fails)
1024 file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1026 logging.info(" Writing file: '{0}'".format(file_name))
1027 with open(file_name, "w") as file_handler:
1028 file_handler.write(",".join(header) + "\n")
1029 for test in tbl_sorted:
1030 file_handler.write(",".join([str(item) for item in test]) + '\n')
1032 txt_file_name = "{0}.txt".format(table["output-file"])
1033 logging.info(" Writing file: '{0}'".format(txt_file_name))
1034 convert_csv_to_pretty_txt(file_name, txt_file_name)
1037 def table_failed_tests_html(table, input_data):
1038 """Generate the table(s) with algorithm: table_failed_tests_html
1039 specified in the specification file.
1041 :param table: Table to generate.
1042 :param input_data: Data to process.
1043 :type table: pandas.Series
1044 :type input_data: InputData
1047 logging.info(" Generating the table {0} ...".
1048 format(table.get("title", "")))
1051 with open(table["input-file"], 'rb') as csv_file:
1052 csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
1053 csv_lst = [item for item in csv_content]
1055 logging.warning("The input file is not defined.")
1057 except csv.Error as err:
1058 logging.warning("Not possible to process the file '{0}'.\n{1}".
1059 format(table["input-file"], err))
1063 failed_tests = ET.Element("table", attrib=dict(width="100%", border='0'))
1066 tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor="#7eade7"))
1067 for idx, item in enumerate(csv_lst[0]):
1068 alignment = "left" if idx == 0 else "center"
1069 th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
1073 colors = {"very-bad": ("#ffcccc", "#ff9999"),
1074 "bad": ("#e9f1fb", "#d4e4f7")}
1075 for r_idx, row in enumerate(csv_lst[1:]):
1080 background = colors[color][r_idx % 2]
1081 tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor=background))
1084 for c_idx, item in enumerate(row):
1085 alignment = "left" if c_idx == 0 else "center"
1086 td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1089 url = _generate_url("../trending/", item)
1090 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1095 with open(table["output-file"], 'w') as html_file:
1096 logging.info(" Writing file: '{0}'".format(table["output-file"]))
1097 html_file.write(".. raw:: html\n\n\t")
1098 html_file.write(ET.tostring(failed_tests))
1099 html_file.write("\n\t<p><br><br></p>\n")
1101 logging.warning("The output file is not defined.")