1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Algorithms to generate tables.
22 from collections import OrderedDict
23 from xml.etree import ElementTree as ET
24 from datetime import datetime as dt
25 from datetime import timedelta
26 from copy import deepcopy
28 import plotly.graph_objects as go
29 import plotly.offline as ploff
32 from numpy import nan, isnan
33 from yaml import load, FullLoader, YAMLError
35 from pal_utils import mean, stdev, classify_anomalies, \
36 convert_csv_to_pretty_txt, relative_change_stdev, relative_change
39 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
42 def generate_tables(spec, data):
43 """Generate all tables specified in the specification file.
45 :param spec: Specification read from the specification file.
46 :param data: Data to process.
47 :type spec: Specification
52 u"table_merged_details": table_merged_details,
53 u"table_soak_vs_ndr": table_soak_vs_ndr,
54 u"table_perf_trending_dash": table_perf_trending_dash,
55 u"table_perf_trending_dash_html": table_perf_trending_dash_html,
56 u"table_last_failed_tests": table_last_failed_tests,
57 u"table_failed_tests": table_failed_tests,
58 u"table_failed_tests_html": table_failed_tests_html,
59 u"table_oper_data_html": table_oper_data_html,
60 u"table_comparison": table_comparison,
61 u"table_weekly_comparison": table_weekly_comparison
64 logging.info(u"Generating the tables ...")
65 for table in spec.tables:
67 if table[u"algorithm"] == u"table_weekly_comparison":
68 table[u"testbeds"] = spec.environment.get(u"testbeds", None)
69 generator[table[u"algorithm"]](table, data)
70 except NameError as err:
72 f"Probably algorithm {table[u'algorithm']} is not defined: "
75 logging.info(u"Done.")
78 def table_oper_data_html(table, input_data):
79 """Generate the table(s) with algorithm: html_table_oper_data
80 specified in the specification file.
82 :param table: Table to generate.
83 :param input_data: Data to process.
84 :type table: pandas.Series
85 :type input_data: InputData
88 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
91 f" Creating the data set for the {table.get(u'type', u'')} "
92 f"{table.get(u'title', u'')}."
94 data = input_data.filter_data(
96 params=[u"name", u"parent", u"show-run", u"type"],
97 continue_on_error=True
101 data = input_data.merge_data(data)
103 sort_tests = table.get(u"sort", None)
107 ascending=(sort_tests == u"ascending")
109 data.sort_index(**args)
111 suites = input_data.filter_data(
113 continue_on_error=True,
118 suites = input_data.merge_data(suites)
120 def _generate_html_table(tst_data):
121 """Generate an HTML table with operational data for the given test.
123 :param tst_data: Test data to be used to generate the table.
124 :type tst_data: pandas.Series
125 :returns: HTML table with operational data.
130 u"header": u"#7eade7",
131 u"empty": u"#ffffff",
132 u"body": (u"#e9f1fb", u"#d4e4f7")
135 tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
137 trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
138 thead = ET.SubElement(
139 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
141 thead.text = tst_data[u"name"]
143 trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
144 thead = ET.SubElement(
145 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
149 if tst_data.get(u"show-run", u"No Data") == u"No Data":
150 trow = ET.SubElement(
151 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
153 tcol = ET.SubElement(
154 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
156 tcol.text = u"No Data"
158 trow = ET.SubElement(
159 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
161 thead = ET.SubElement(
162 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
164 font = ET.SubElement(
165 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
168 return str(ET.tostring(tbl, encoding=u"unicode"))
175 u"Cycles per Packet",
176 u"Average Vector Size"
179 for dut_data in tst_data[u"show-run"].values():
180 trow = ET.SubElement(
181 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
183 tcol = ET.SubElement(
184 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
186 if dut_data.get(u"threads", None) is None:
187 tcol.text = u"No Data"
190 bold = ET.SubElement(tcol, u"b")
192 f"Host IP: {dut_data.get(u'host', '')}, "
193 f"Socket: {dut_data.get(u'socket', '')}"
195 trow = ET.SubElement(
196 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
198 thead = ET.SubElement(
199 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
203 for thread_nr, thread in dut_data[u"threads"].items():
204 trow = ET.SubElement(
205 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
207 tcol = ET.SubElement(
208 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
210 bold = ET.SubElement(tcol, u"b")
211 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
212 trow = ET.SubElement(
213 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
215 for idx, col in enumerate(tbl_hdr):
216 tcol = ET.SubElement(
218 attrib=dict(align=u"right" if idx else u"left")
220 font = ET.SubElement(
221 tcol, u"font", attrib=dict(size=u"2")
223 bold = ET.SubElement(font, u"b")
225 for row_nr, row in enumerate(thread):
226 trow = ET.SubElement(
228 attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
230 for idx, col in enumerate(row):
231 tcol = ET.SubElement(
233 attrib=dict(align=u"right" if idx else u"left")
235 font = ET.SubElement(
236 tcol, u"font", attrib=dict(size=u"2")
238 if isinstance(col, float):
239 font.text = f"{col:.2f}"
242 trow = ET.SubElement(
243 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
245 thead = ET.SubElement(
246 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
250 trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
251 thead = ET.SubElement(
252 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
254 font = ET.SubElement(
255 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
259 return str(ET.tostring(tbl, encoding=u"unicode"))
261 for suite in suites.values:
263 for test_data in data.values:
264 if test_data[u"parent"] not in suite[u"name"]:
266 html_table += _generate_html_table(test_data)
270 file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
271 with open(f"{file_name}", u'w') as html_file:
272 logging.info(f" Writing file: {file_name}")
273 html_file.write(u".. raw:: html\n\n\t")
274 html_file.write(html_table)
275 html_file.write(u"\n\t<p><br><br></p>\n")
277 logging.warning(u"The output file is not defined.")
279 logging.info(u" Done.")
282 def table_merged_details(table, input_data):
283 """Generate the table(s) with algorithm: table_merged_details
284 specified in the specification file.
286 :param table: Table to generate.
287 :param input_data: Data to process.
288 :type table: pandas.Series
289 :type input_data: InputData
292 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
296 f" Creating the data set for the {table.get(u'type', u'')} "
297 f"{table.get(u'title', u'')}."
299 data = input_data.filter_data(table, continue_on_error=True)
300 data = input_data.merge_data(data)
302 sort_tests = table.get(u"sort", None)
306 ascending=(sort_tests == u"ascending")
308 data.sort_index(**args)
310 suites = input_data.filter_data(
311 table, continue_on_error=True, data_set=u"suites")
312 suites = input_data.merge_data(suites)
314 # Prepare the header of the tables
316 for column in table[u"columns"]:
318 u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
321 for suite in suites.values:
323 suite_name = suite[u"name"]
325 for test in data.keys():
326 if data[test][u"parent"] not in suite_name:
329 for column in table[u"columns"]:
331 col_data = str(data[test][column[
332 u"data"].split(u" ")[1]]).replace(u'"', u'""')
333 # Do not include tests with "Test Failed" in test message
334 if u"Test Failed" in col_data:
336 col_data = col_data.replace(
337 u"No Data", u"Not Captured "
339 if column[u"data"].split(u" ")[1] in (u"name", ):
340 if len(col_data) > 30:
341 col_data_lst = col_data.split(u"-")
342 half = int(len(col_data_lst) / 2)
343 col_data = f"{u'-'.join(col_data_lst[:half])}" \
345 f"{u'-'.join(col_data_lst[half:])}"
346 col_data = f" |prein| {col_data} |preout| "
347 elif column[u"data"].split(u" ")[1] in (u"msg", ):
348 # Temporary solution: remove NDR results from message:
349 if bool(table.get(u'remove-ndr', False)):
351 col_data = col_data.split(u" |br| ", 1)[1]
354 col_data = f" |prein| {col_data} |preout| "
355 elif column[u"data"].split(u" ")[1] in \
356 (u"conf-history", u"show-run"):
357 col_data = col_data.replace(u" |br| ", u"", 1)
358 col_data = f" |prein| {col_data[:-5]} |preout| "
359 row_lst.append(f'"{col_data}"')
361 row_lst.append(u'"Not captured"')
362 if len(row_lst) == len(table[u"columns"]):
363 table_lst.append(row_lst)
365 # Write the data to file
367 separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
368 file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
369 logging.info(f" Writing file: {file_name}")
370 with open(file_name, u"wt") as file_handler:
371 file_handler.write(u",".join(header) + u"\n")
372 for item in table_lst:
373 file_handler.write(u",".join(item) + u"\n")
375 logging.info(u" Done.")
378 def _tpc_modify_test_name(test_name, ignore_nic=False):
379 """Modify a test name by replacing its parts.
381 :param test_name: Test name to be modified.
382 :param ignore_nic: If True, NIC is removed from TC name.
384 :type ignore_nic: bool
385 :returns: Modified test name.
388 test_name_mod = test_name.\
389 replace(u"-ndrpdrdisc", u""). \
390 replace(u"-ndrpdr", u"").\
391 replace(u"-pdrdisc", u""). \
392 replace(u"-ndrdisc", u"").\
393 replace(u"-pdr", u""). \
394 replace(u"-ndr", u""). \
395 replace(u"1t1c", u"1c").\
396 replace(u"2t1c", u"1c"). \
397 replace(u"2t2c", u"2c").\
398 replace(u"4t2c", u"2c"). \
399 replace(u"4t4c", u"4c").\
400 replace(u"8t4c", u"4c")
403 return re.sub(REGEX_NIC, u"", test_name_mod)
407 def _tpc_modify_displayed_test_name(test_name):
408 """Modify a test name which is displayed in a table by replacing its parts.
410 :param test_name: Test name to be modified.
412 :returns: Modified test name.
416 replace(u"1t1c", u"1c").\
417 replace(u"2t1c", u"1c"). \
418 replace(u"2t2c", u"2c").\
419 replace(u"4t2c", u"2c"). \
420 replace(u"4t4c", u"4c").\
421 replace(u"8t4c", u"4c")
424 def _tpc_insert_data(target, src, include_tests):
425 """Insert src data to the target structure.
427 :param target: Target structure where the data is placed.
428 :param src: Source data to be placed into the target stucture.
429 :param include_tests: Which results will be included (MRR, NDR, PDR).
432 :type include_tests: str
435 if include_tests == u"MRR":
436 target[u"mean"] = src[u"result"][u"receive-rate"]
437 target[u"stdev"] = src[u"result"][u"receive-stdev"]
438 elif include_tests == u"PDR":
439 target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
440 elif include_tests == u"NDR":
441 target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
442 except (KeyError, TypeError):
446 def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
447 footnote=u"", sort_data=True, title=u"",
449 """Generate html table from input data with simple sorting possibility.
451 :param header: Table header.
452 :param data: Input data to be included in the table. It is a list of lists.
453 Inner lists are rows in the table. All inner lists must be of the same
454 length. The length of these lists must be the same as the length of the
456 :param out_file_name: The name (relative or full path) where the
457 generated html table is written.
458 :param legend: The legend to display below the table.
459 :param footnote: The footnote to display below the table (and legend).
460 :param sort_data: If True the data sorting is enabled.
461 :param title: The table (and file) title.
462 :param generate_rst: If True, wrapping rst file is generated.
464 :type data: list of lists
465 :type out_file_name: str
468 :type sort_data: bool
470 :type generate_rst: bool
474 idx = header.index(u"Test Case")
480 [u"left", u"left", u"right"],
481 [u"left", u"left", u"left", u"right"]
485 [u"left", u"left", u"right"],
486 [u"left", u"left", u"left", u"right"]
488 u"width": ([15, 9], [4, 24, 10], [4, 4, 32, 10])
491 df_data = pd.DataFrame(data, columns=header)
494 df_sorted = [df_data.sort_values(
495 by=[key, header[idx]], ascending=[True, True]
496 if key != header[idx] else [False, True]) for key in header]
497 df_sorted_rev = [df_data.sort_values(
498 by=[key, header[idx]], ascending=[False, True]
499 if key != header[idx] else [True, True]) for key in header]
500 df_sorted.extend(df_sorted_rev)
504 fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
505 for idx in range(len(df_data))]]
507 values=[f"<b>{item.replace(u',', u',<br>')}</b>" for item in header],
508 fill_color=u"#7eade7",
509 align=params[u"align-hdr"][idx],
511 family=u"Courier New",
519 for table in df_sorted:
520 columns = [table.get(col) for col in header]
523 columnwidth=params[u"width"][idx],
527 fill_color=fill_color,
528 align=params[u"align-itm"][idx],
530 family=u"Courier New",
538 menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
539 menu_items.extend([f"<b>{itm}</b> (descending)" for itm in header])
540 for idx, hdr in enumerate(menu_items):
541 visible = [False, ] * len(menu_items)
545 label=hdr.replace(u" [Mpps]", u""),
547 args=[{u"visible": visible}],
553 go.layout.Updatemenu(
560 active=len(menu_items) - 1,
561 buttons=list(buttons)
568 columnwidth=params[u"width"][idx],
571 values=[df_sorted.get(col) for col in header],
572 fill_color=fill_color,
573 align=params[u"align-itm"][idx],
575 family=u"Courier New",
586 filename=f"{out_file_name}_in.html"
592 file_name = out_file_name.split(u"/")[-1]
593 if u"vpp" in out_file_name:
594 path = u"_tmp/src/vpp_performance_tests/comparisons/"
596 path = u"_tmp/src/dpdk_performance_tests/comparisons/"
597 with open(f"{path}{file_name}.rst", u"wt") as rst_file:
600 u".. |br| raw:: html\n\n <br />\n\n\n"
601 u".. |prein| raw:: html\n\n <pre>\n\n\n"
602 u".. |preout| raw:: html\n\n </pre>\n\n"
605 rst_file.write(f"{title}\n")
606 rst_file.write(f"{u'`' * len(title)}\n\n")
609 f' <iframe frameborder="0" scrolling="no" '
610 f'width="1600" height="1200" '
611 f'src="../..{out_file_name.replace(u"_build", u"")}_in.html">'
615 # TODO: Use html (rst) list for legend and footnote
617 rst_file.write(legend[1:].replace(u"\n", u" |br| "))
619 rst_file.write(footnote.replace(u"\n", u" |br| ")[1:])
622 def table_soak_vs_ndr(table, input_data):
623 """Generate the table(s) with algorithm: table_soak_vs_ndr
624 specified in the specification file.
626 :param table: Table to generate.
627 :param input_data: Data to process.
628 :type table: pandas.Series
629 :type input_data: InputData
632 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
636 f" Creating the data set for the {table.get(u'type', u'')} "
637 f"{table.get(u'title', u'')}."
639 data = input_data.filter_data(table, continue_on_error=True)
641 # Prepare the header of the table
645 f"Avg({table[u'reference'][u'title']})",
646 f"Stdev({table[u'reference'][u'title']})",
647 f"Avg({table[u'compare'][u'title']})",
648 f"Stdev{table[u'compare'][u'title']})",
652 header_str = u";".join(header) + u"\n"
655 f"Avg({table[u'reference'][u'title']}): "
656 f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
657 f"from a series of runs of the listed tests.\n"
658 f"Stdev({table[u'reference'][u'title']}): "
659 f"Standard deviation value of {table[u'reference'][u'title']} "
660 f"[Mpps] computed from a series of runs of the listed tests.\n"
661 f"Avg({table[u'compare'][u'title']}): "
662 f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
663 f"a series of runs of the listed tests.\n"
664 f"Stdev({table[u'compare'][u'title']}): "
665 f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
666 f"computed from a series of runs of the listed tests.\n"
667 f"Diff({table[u'reference'][u'title']},"
668 f"{table[u'compare'][u'title']}): "
669 f"Percentage change calculated for mean values.\n"
671 u"Standard deviation of percentage change calculated for mean "
674 except (AttributeError, KeyError) as err:
675 logging.error(f"The model is invalid, missing parameter: {repr(err)}")
678 # Create a list of available SOAK test results:
680 for job, builds in table[u"compare"][u"data"].items():
682 for tst_name, tst_data in data[job][str(build)].items():
683 if tst_data[u"type"] == u"SOAK":
684 tst_name_mod = tst_name.replace(u"-soak", u"")
685 if tbl_dict.get(tst_name_mod, None) is None:
686 groups = re.search(REGEX_NIC, tst_data[u"parent"])
687 nic = groups.group(0) if groups else u""
690 f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
692 tbl_dict[tst_name_mod] = {
698 tbl_dict[tst_name_mod][u"cmp-data"].append(
699 tst_data[u"throughput"][u"LOWER"])
700 except (KeyError, TypeError):
702 tests_lst = tbl_dict.keys()
704 # Add corresponding NDR test results:
705 for job, builds in table[u"reference"][u"data"].items():
707 for tst_name, tst_data in data[job][str(build)].items():
708 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
709 replace(u"-mrr", u"")
710 if tst_name_mod not in tests_lst:
713 if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
715 if table[u"include-tests"] == u"MRR":
716 result = (tst_data[u"result"][u"receive-rate"],
717 tst_data[u"result"][u"receive-stdev"])
718 elif table[u"include-tests"] == u"PDR":
720 tst_data[u"throughput"][u"PDR"][u"LOWER"]
721 elif table[u"include-tests"] == u"NDR":
723 tst_data[u"throughput"][u"NDR"][u"LOWER"]
726 if result is not None:
727 tbl_dict[tst_name_mod][u"ref-data"].append(
729 except (KeyError, TypeError):
733 for tst_name in tbl_dict:
734 item = [tbl_dict[tst_name][u"name"], ]
735 data_r = tbl_dict[tst_name][u"ref-data"]
737 if table[u"include-tests"] == u"MRR":
738 data_r_mean = data_r[0][0]
739 data_r_stdev = data_r[0][1]
741 data_r_mean = mean(data_r)
742 data_r_stdev = stdev(data_r)
743 item.append(round(data_r_mean / 1e6, 1))
744 item.append(round(data_r_stdev / 1e6, 1))
748 item.extend([None, None])
749 data_c = tbl_dict[tst_name][u"cmp-data"]
751 if table[u"include-tests"] == u"MRR":
752 data_c_mean = data_c[0][0]
753 data_c_stdev = data_c[0][1]
755 data_c_mean = mean(data_c)
756 data_c_stdev = stdev(data_c)
757 item.append(round(data_c_mean / 1e6, 1))
758 item.append(round(data_c_stdev / 1e6, 1))
762 item.extend([None, None])
763 if data_r_mean is not None and data_c_mean is not None:
764 delta, d_stdev = relative_change_stdev(
765 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
767 item.append(round(delta))
771 item.append(round(d_stdev))
776 # Sort the table according to the relative change
777 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
779 # Generate csv tables:
780 csv_file = f"{table[u'output-file']}.csv"
781 with open(csv_file, u"wt") as file_handler:
782 file_handler.write(header_str)
784 file_handler.write(u";".join([str(item) for item in test]) + u"\n")
786 convert_csv_to_pretty_txt(
787 csv_file, f"{table[u'output-file']}.txt", delimiter=u";"
789 with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
790 txt_file.write(legend)
792 # Generate html table:
793 _tpc_generate_html_table(
796 table[u'output-file'],
798 title=table.get(u"title", u"")
802 def table_perf_trending_dash(table, input_data):
803 """Generate the table(s) with algorithm:
804 table_perf_trending_dash
805 specified in the specification file.
807 :param table: Table to generate.
808 :param input_data: Data to process.
809 :type table: pandas.Series
810 :type input_data: InputData
813 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
817 f" Creating the data set for the {table.get(u'type', u'')} "
818 f"{table.get(u'title', u'')}."
820 data = input_data.filter_data(table, continue_on_error=True)
822 # Prepare the header of the tables
826 u"Short-Term Change [%]",
827 u"Long-Term Change [%]",
831 header_str = u",".join(header) + u"\n"
833 incl_tests = table.get(u"include-tests", u"MRR")
835 # Prepare data to the table:
837 for job, builds in table[u"data"].items():
839 for tst_name, tst_data in data[job][str(build)].items():
840 if tst_name.lower() in table.get(u"ignore-list", list()):
842 if tbl_dict.get(tst_name, None) is None:
843 groups = re.search(REGEX_NIC, tst_data[u"parent"])
846 nic = groups.group(0)
847 tbl_dict[tst_name] = {
848 u"name": f"{nic}-{tst_data[u'name']}",
849 u"data": OrderedDict()
852 if incl_tests == u"MRR":
853 tbl_dict[tst_name][u"data"][str(build)] = \
854 tst_data[u"result"][u"receive-rate"]
855 elif incl_tests == u"NDR":
856 tbl_dict[tst_name][u"data"][str(build)] = \
857 tst_data[u"throughput"][u"NDR"][u"LOWER"]
858 elif incl_tests == u"PDR":
859 tbl_dict[tst_name][u"data"][str(build)] = \
860 tst_data[u"throughput"][u"PDR"][u"LOWER"]
861 except (TypeError, KeyError):
862 pass # No data in output.xml for this test
865 for tst_name in tbl_dict:
866 data_t = tbl_dict[tst_name][u"data"]
870 classification_lst, avgs, _ = classify_anomalies(data_t)
872 win_size = min(len(data_t), table[u"window"])
873 long_win_size = min(len(data_t), table[u"long-trend-window"])
877 [x for x in avgs[-long_win_size:-win_size]
882 avg_week_ago = avgs[max(-win_size, -len(avgs))]
884 if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
885 rel_change_last = nan
887 rel_change_last = round(
888 ((last_avg - avg_week_ago) / avg_week_ago) * 1e2, 2)
890 if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
891 rel_change_long = nan
893 rel_change_long = round(
894 ((last_avg - max_long_avg) / max_long_avg) * 1e2, 2)
896 if classification_lst:
897 if isnan(rel_change_last) and isnan(rel_change_long):
899 if isnan(last_avg) or isnan(rel_change_last) or \
900 isnan(rel_change_long):
903 [tbl_dict[tst_name][u"name"],
904 round(last_avg / 1e6, 2),
907 classification_lst[-win_size+1:].count(u"regression"),
908 classification_lst[-win_size+1:].count(u"progression")])
910 tbl_lst.sort(key=lambda rel: rel[0])
911 tbl_lst.sort(key=lambda rel: rel[3])
912 tbl_lst.sort(key=lambda rel: rel[2])
915 for nrr in range(table[u"window"], -1, -1):
916 tbl_reg = [item for item in tbl_lst if item[4] == nrr]
917 for nrp in range(table[u"window"], -1, -1):
918 tbl_out = [item for item in tbl_reg if item[5] == nrp]
919 tbl_sorted.extend(tbl_out)
921 file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
923 logging.info(f" Writing file: {file_name}")
924 with open(file_name, u"wt") as file_handler:
925 file_handler.write(header_str)
926 for test in tbl_sorted:
927 file_handler.write(u",".join([str(item) for item in test]) + u'\n')
929 logging.info(f" Writing file: {table[u'output-file']}.txt")
930 convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
933 def _generate_url(testbed, test_name):
934 """Generate URL to a trending plot from the name of the test case.
936 :param testbed: The testbed used for testing.
937 :param test_name: The name of the test case.
940 :returns: The URL to the plot with the trending data for the given test
945 if u"x520" in test_name:
947 elif u"x710" in test_name:
949 elif u"xl710" in test_name:
951 elif u"xxv710" in test_name:
953 elif u"vic1227" in test_name:
955 elif u"vic1385" in test_name:
957 elif u"x553" in test_name:
959 elif u"cx556" in test_name or u"cx556a" in test_name:
964 if u"64b" in test_name:
966 elif u"78b" in test_name:
968 elif u"imix" in test_name:
970 elif u"9000b" in test_name:
971 frame_size = u"9000b"
972 elif u"1518b" in test_name:
973 frame_size = u"1518b"
974 elif u"114b" in test_name:
979 if u"1t1c" in test_name or \
980 (u"-1c-" in test_name and
981 testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
983 elif u"2t2c" in test_name or \
984 (u"-2c-" in test_name and
985 testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
987 elif u"4t4c" in test_name or \
988 (u"-4c-" in test_name and
989 testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
991 elif u"2t1c" in test_name or \
992 (u"-1c-" in test_name and
993 testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
995 elif u"4t2c" in test_name or \
996 (u"-2c-" in test_name and
997 testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
999 elif u"8t4c" in test_name or \
1000 (u"-4c-" in test_name and
1001 testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1006 if u"testpmd" in test_name:
1008 elif u"l3fwd" in test_name:
1010 elif u"avf" in test_name:
1012 elif u"rdma" in test_name:
1014 elif u"dnv" in testbed or u"tsh" in testbed:
1019 if u"macip-iacl1s" in test_name:
1020 bsf = u"features-macip-iacl1"
1021 elif u"macip-iacl10s" in test_name:
1022 bsf = u"features-macip-iacl01"
1023 elif u"macip-iacl50s" in test_name:
1024 bsf = u"features-macip-iacl50"
1025 elif u"iacl1s" in test_name:
1026 bsf = u"features-iacl1"
1027 elif u"iacl10s" in test_name:
1028 bsf = u"features-iacl10"
1029 elif u"iacl50s" in test_name:
1030 bsf = u"features-iacl50"
1031 elif u"oacl1s" in test_name:
1032 bsf = u"features-oacl1"
1033 elif u"oacl10s" in test_name:
1034 bsf = u"features-oacl10"
1035 elif u"oacl50s" in test_name:
1036 bsf = u"features-oacl50"
1037 elif u"udpsrcscale" in test_name:
1038 bsf = u"features-udp"
1039 elif u"iacl" in test_name:
1041 elif u"policer" in test_name:
1043 elif u"cop" in test_name:
1045 elif u"nat" in test_name:
1047 elif u"macip" in test_name:
1049 elif u"scale" in test_name:
1051 elif u"base" in test_name:
1056 if u"114b" in test_name and u"vhost" in test_name:
1058 elif u"testpmd" in test_name or u"l3fwd" in test_name:
1060 elif u"memif" in test_name:
1061 domain = u"container_memif"
1062 elif u"srv6" in test_name:
1064 elif u"vhost" in test_name:
1066 if u"vppl2xc" in test_name:
1069 driver += u"-testpmd"
1070 if u"lbvpplacp" in test_name:
1071 bsf += u"-link-bonding"
1072 elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1073 domain = u"nf_service_density_vnfc"
1074 elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1075 domain = u"nf_service_density_cnfc"
1076 elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1077 domain = u"nf_service_density_cnfp"
1078 elif u"ipsec" in test_name:
1080 if u"sw" in test_name:
1082 elif u"hw" in test_name:
1084 elif u"ethip4vxlan" in test_name:
1085 domain = u"ip4_tunnels"
1086 elif u"ip4base" in test_name or u"ip4scale" in test_name:
1088 elif u"ip6base" in test_name or u"ip6scale" in test_name:
1090 elif u"l2xcbase" in test_name or \
1091 u"l2xcscale" in test_name or \
1092 u"l2bdbasemaclrn" in test_name or \
1093 u"l2bdscale" in test_name or \
1094 u"l2patch" in test_name:
1099 file_name = u"-".join((domain, testbed, nic)) + u".html#"
1100 anchor_name = u"-".join((frame_size, cores, bsf, driver))
1102 return file_name + anchor_name
1105 def table_perf_trending_dash_html(table, input_data):
1106 """Generate the table(s) with algorithm:
1107 table_perf_trending_dash_html specified in the specification
1110 :param table: Table to generate.
1111 :param input_data: Data to process.
1113 :type input_data: InputData
1118 if not table.get(u"testbed", None):
1120 f"The testbed is not defined for the table "
1121 f"{table.get(u'title', u'')}. Skipping."
1125 test_type = table.get(u"test-type", u"MRR")
1126 if test_type not in (u"MRR", u"NDR", u"PDR"):
1128 f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
1133 if test_type in (u"NDR", u"PDR"):
1134 lnk_dir = u"../ndrpdr_trending/"
1135 lnk_sufix = f"-{test_type.lower()}"
1137 lnk_dir = u"../trending/"
1140 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1143 with open(table[u"input-file"], u'rt') as csv_file:
1144 csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1146 logging.warning(u"The input file is not defined.")
1148 except csv.Error as err:
1150 f"Not possible to process the file {table[u'input-file']}.\n"
1156 dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1159 trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1160 for idx, item in enumerate(csv_lst[0]):
1161 alignment = u"left" if idx == 0 else u"center"
1162 thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1180 for r_idx, row in enumerate(csv_lst[1:]):
1182 color = u"regression"
1184 color = u"progression"
1187 trow = ET.SubElement(
1188 dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
1192 for c_idx, item in enumerate(row):
1193 tdata = ET.SubElement(
1196 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1199 if c_idx == 0 and table.get(u"add-links", True):
1200 ref = ET.SubElement(
1205 f"{_generate_url(table.get(u'testbed', ''), item)}"
1213 with open(table[u"output-file"], u'w') as html_file:
1214 logging.info(f" Writing file: {table[u'output-file']}")
1215 html_file.write(u".. raw:: html\n\n\t")
1216 html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
1217 html_file.write(u"\n\t<p><br><br></p>\n")
1219 logging.warning(u"The output file is not defined.")
1223 def table_last_failed_tests(table, input_data):
1224 """Generate the table(s) with algorithm: table_last_failed_tests
1225 specified in the specification file.
1227 :param table: Table to generate.
1228 :param input_data: Data to process.
1229 :type table: pandas.Series
1230 :type input_data: InputData
1233 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1235 # Transform the data
1237 f" Creating the data set for the {table.get(u'type', u'')} "
1238 f"{table.get(u'title', u'')}."
1241 data = input_data.filter_data(table, continue_on_error=True)
1243 if data is None or data.empty:
1245 f" No data for the {table.get(u'type', u'')} "
1246 f"{table.get(u'title', u'')}."
1251 for job, builds in table[u"data"].items():
1252 for build in builds:
1255 version = input_data.metadata(job, build).get(u"version", u"")
1257 logging.error(f"Data for {job}: {build} is not present.")
1259 tbl_list.append(build)
1260 tbl_list.append(version)
1261 failed_tests = list()
1264 for tst_data in data[job][build].values:
1265 if tst_data[u"status"] != u"FAIL":
1269 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1272 nic = groups.group(0)
1273 failed_tests.append(f"{nic}-{tst_data[u'name']}")
1274 tbl_list.append(str(passed))
1275 tbl_list.append(str(failed))
1276 tbl_list.extend(failed_tests)
1278 file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1279 logging.info(f" Writing file: {file_name}")
1280 with open(file_name, u"wt") as file_handler:
1281 for test in tbl_list:
1282 file_handler.write(test + u'\n')
1285 def table_failed_tests(table, input_data):
1286 """Generate the table(s) with algorithm: table_failed_tests
1287 specified in the specification file.
1289 :param table: Table to generate.
1290 :param input_data: Data to process.
1291 :type table: pandas.Series
1292 :type input_data: InputData
1295 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1297 # Transform the data
1299 f" Creating the data set for the {table.get(u'type', u'')} "
1300 f"{table.get(u'title', u'')}."
1302 data = input_data.filter_data(table, continue_on_error=True)
1305 if u"NDRPDR" in table.get(u"filter", list()):
1306 test_type = u"NDRPDR"
1308 # Prepare the header of the tables
1312 u"Last Failure [Time]",
1313 u"Last Failure [VPP-Build-Id]",
1314 u"Last Failure [CSIT-Job-Build-Id]"
1317 # Generate the data for the table according to the model in the table
1321 timeperiod = timedelta(int(table.get(u"window", 7)))
1324 for job, builds in table[u"data"].items():
1325 for build in builds:
1327 for tst_name, tst_data in data[job][build].items():
1328 if tst_name.lower() in table.get(u"ignore-list", list()):
1330 if tbl_dict.get(tst_name, None) is None:
1331 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1334 nic = groups.group(0)
1335 tbl_dict[tst_name] = {
1336 u"name": f"{nic}-{tst_data[u'name']}",
1337 u"data": OrderedDict()
1340 generated = input_data.metadata(job, build).\
1341 get(u"generated", u"")
1344 then = dt.strptime(generated, u"%Y%m%d %H:%M")
1345 if (now - then) <= timeperiod:
1346 tbl_dict[tst_name][u"data"][build] = (
1347 tst_data[u"status"],
1349 input_data.metadata(job, build).get(u"version",
1353 except (TypeError, KeyError) as err:
1354 logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
1358 for tst_data in tbl_dict.values():
1360 fails_last_date = u""
1361 fails_last_vpp = u""
1362 fails_last_csit = u""
1363 for val in tst_data[u"data"].values():
1364 if val[0] == u"FAIL":
1366 fails_last_date = val[1]
1367 fails_last_vpp = val[2]
1368 fails_last_csit = val[3]
1370 max_fails = fails_nr if fails_nr > max_fails else max_fails
1376 f"{u'mrr-daily' if test_type == u'MRR' else u'ndrpdr-weekly'}"
1377 f"-build-{fails_last_csit}"
1380 tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1382 for nrf in range(max_fails, -1, -1):
1383 tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1384 tbl_sorted.extend(tbl_fails)
1386 file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1387 logging.info(f" Writing file: {file_name}")
1388 with open(file_name, u"wt") as file_handler:
1389 file_handler.write(u",".join(header) + u"\n")
1390 for test in tbl_sorted:
1391 file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1393 logging.info(f" Writing file: {table[u'output-file']}.txt")
1394 convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1397 def table_failed_tests_html(table, input_data):
1398 """Generate the table(s) with algorithm: table_failed_tests_html
1399 specified in the specification file.
1401 :param table: Table to generate.
1402 :param input_data: Data to process.
1403 :type table: pandas.Series
1404 :type input_data: InputData
1409 if not table.get(u"testbed", None):
1411 f"The testbed is not defined for the table "
1412 f"{table.get(u'title', u'')}. Skipping."
1416 test_type = table.get(u"test-type", u"MRR")
1417 if test_type not in (u"MRR", u"NDR", u"PDR", u"NDRPDR"):
1419 f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
1424 if test_type in (u"NDRPDR", u"NDR", u"PDR"):
1425 lnk_dir = u"../ndrpdr_trending/"
1428 lnk_dir = u"../trending/"
1431 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1434 with open(table[u"input-file"], u'rt') as csv_file:
1435 csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1437 logging.warning(u"The input file is not defined.")
1439 except csv.Error as err:
1441 f"Not possible to process the file {table[u'input-file']}.\n"
1447 failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1450 trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1451 for idx, item in enumerate(csv_lst[0]):
1452 alignment = u"left" if idx == 0 else u"center"
1453 thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1457 colors = (u"#e9f1fb", u"#d4e4f7")
1458 for r_idx, row in enumerate(csv_lst[1:]):
1459 background = colors[r_idx % 2]
1460 trow = ET.SubElement(
1461 failed_tests, u"tr", attrib=dict(bgcolor=background)
1465 for c_idx, item in enumerate(row):
1466 tdata = ET.SubElement(
1469 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1472 if c_idx == 0 and table.get(u"add-links", True):
1473 ref = ET.SubElement(
1478 f"{_generate_url(table.get(u'testbed', ''), item)}"
1486 with open(table[u"output-file"], u'w') as html_file:
1487 logging.info(f" Writing file: {table[u'output-file']}")
1488 html_file.write(u".. raw:: html\n\n\t")
1489 html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
1490 html_file.write(u"\n\t<p><br><br></p>\n")
1492 logging.warning(u"The output file is not defined.")
1496 def table_comparison(table, input_data):
1497 """Generate the table(s) with algorithm: table_comparison
1498 specified in the specification file.
1500 :param table: Table to generate.
1501 :param input_data: Data to process.
1502 :type table: pandas.Series
1503 :type input_data: InputData
1505 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1507 # Transform the data
1509 f" Creating the data set for the {table.get(u'type', u'')} "
1510 f"{table.get(u'title', u'')}."
1513 columns = table.get(u"columns", None)
1516 f"No columns specified for {table.get(u'title', u'')}. Skipping."
1521 for idx, col in enumerate(columns):
1522 if col.get(u"data-set", None) is None:
1523 logging.warning(f"No data for column {col.get(u'title', u'')}")
1525 tag = col.get(u"tag", None)
1526 data = input_data.filter_data(
1528 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1529 data=col[u"data-set"],
1530 continue_on_error=True
1533 u"title": col.get(u"title", f"Column{idx}"),
1536 for builds in data.values:
1537 for build in builds:
1538 for tst_name, tst_data in build.items():
1539 if tag and tag not in tst_data[u"tags"]:
1542 _tpc_modify_test_name(tst_name, ignore_nic=True).\
1543 replace(u"2n1l-", u"")
1544 if col_data[u"data"].get(tst_name_mod, None) is None:
1545 name = tst_data[u'name'].rsplit(u'-', 1)[0]
1546 if u"across testbeds" in table[u"title"].lower() or \
1547 u"across topologies" in table[u"title"].lower():
1548 name = _tpc_modify_displayed_test_name(name)
1549 col_data[u"data"][tst_name_mod] = {
1557 target=col_data[u"data"][tst_name_mod],
1559 include_tests=table[u"include-tests"]
1562 replacement = col.get(u"data-replacement", None)
1564 rpl_data = input_data.filter_data(
1566 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1568 continue_on_error=True
1570 for builds in rpl_data.values:
1571 for build in builds:
1572 for tst_name, tst_data in build.items():
1573 if tag and tag not in tst_data[u"tags"]:
1576 _tpc_modify_test_name(tst_name, ignore_nic=True).\
1577 replace(u"2n1l-", u"")
1578 if col_data[u"data"].get(tst_name_mod, None) is None:
1579 name = tst_data[u'name'].rsplit(u'-', 1)[0]
1580 if u"across testbeds" in table[u"title"].lower() \
1581 or u"across topologies" in \
1582 table[u"title"].lower():
1583 name = _tpc_modify_displayed_test_name(name)
1584 col_data[u"data"][tst_name_mod] = {
1591 if col_data[u"data"][tst_name_mod][u"replace"]:
1592 col_data[u"data"][tst_name_mod][u"replace"] = False
1593 col_data[u"data"][tst_name_mod][u"data"] = list()
1595 target=col_data[u"data"][tst_name_mod],
1597 include_tests=table[u"include-tests"]
1600 if table[u"include-tests"] in (u"NDR", u"PDR"):
1601 for tst_name, tst_data in col_data[u"data"].items():
1602 if tst_data[u"data"]:
1603 tst_data[u"mean"] = mean(tst_data[u"data"])
1604 tst_data[u"stdev"] = stdev(tst_data[u"data"])
1606 cols.append(col_data)
1610 for tst_name, tst_data in col[u"data"].items():
1611 if tbl_dict.get(tst_name, None) is None:
1612 tbl_dict[tst_name] = {
1613 "name": tst_data[u"name"]
1615 tbl_dict[tst_name][col[u"title"]] = {
1616 u"mean": tst_data[u"mean"],
1617 u"stdev": tst_data[u"stdev"]
1621 logging.warning(f"No data for table {table.get(u'title', u'')}!")
1625 for tst_data in tbl_dict.values():
1626 row = [tst_data[u"name"], ]
1628 row.append(tst_data.get(col[u"title"], None))
1631 comparisons = table.get(u"comparisons", None)
1632 if comparisons and isinstance(comparisons, list):
1633 for idx, comp in enumerate(comparisons):
1635 col_ref = int(comp[u"reference"])
1636 col_cmp = int(comp[u"compare"])
1638 logging.warning(u"Comparison: No references defined! Skipping.")
1639 comparisons.pop(idx)
1641 if not (0 < col_ref <= len(cols) and
1642 0 < col_cmp <= len(cols)) or \
1644 logging.warning(f"Wrong values of reference={col_ref} "
1645 f"and/or compare={col_cmp}. Skipping.")
1646 comparisons.pop(idx)
1649 tbl_cmp_lst = list()
1652 new_row = deepcopy(row)
1654 for comp in comparisons:
1655 ref_itm = row[int(comp[u"reference"])]
1656 if ref_itm is None and \
1657 comp.get(u"reference-alt", None) is not None:
1658 ref_itm = row[int(comp[u"reference-alt"])]
1659 cmp_itm = row[int(comp[u"compare"])]
1660 if ref_itm is not None and cmp_itm is not None and \
1661 ref_itm[u"mean"] is not None and \
1662 cmp_itm[u"mean"] is not None and \
1663 ref_itm[u"stdev"] is not None and \
1664 cmp_itm[u"stdev"] is not None:
1665 delta, d_stdev = relative_change_stdev(
1666 ref_itm[u"mean"], cmp_itm[u"mean"],
1667 ref_itm[u"stdev"], cmp_itm[u"stdev"]
1671 u"mean": delta * 1e6,
1672 u"stdev": d_stdev * 1e6
1677 new_row.append(None)
1679 tbl_cmp_lst.append(new_row)
1681 tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
1682 tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
1685 rca_in = table.get(u"rca", None)
1686 if rca_in and isinstance(rca_in, list):
1687 for idx, itm in enumerate(rca_in):
1689 with open(itm.get(u"data", u""), u"r") as rca_file:
1692 u"title": itm.get(u"title", f"RCA{idx}"),
1693 u"data": load(rca_file, Loader=FullLoader)
1696 except (YAMLError, IOError) as err:
1698 f"The RCA file {itm.get(u'data', u'')} does not exist or "
1701 logging.debug(repr(err))
1703 tbl_for_csv = list()
1704 for line in tbl_cmp_lst:
1706 for idx, itm in enumerate(line[1:]):
1707 if itm is None or not isinstance(itm, dict) or\
1708 itm.get(u'mean', None) is None or \
1709 itm.get(u'stdev', None) is None:
1713 row.append(round(float(itm[u'mean']) / 1e6, 3))
1714 row.append(round(float(itm[u'stdev']) / 1e6, 3))
1716 rca_nr = rca[u"data"].get(row[0], u"-")
1717 row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1718 tbl_for_csv.append(row)
1720 header_csv = [u"Test Case", ]
1722 header_csv.append(f"Avg({col[u'title']})")
1723 header_csv.append(f"Stdev({col[u'title']})")
1724 for comp in comparisons:
1726 f"Avg({comp.get(u'title', u'')})"
1729 f"Stdev({comp.get(u'title', u'')})"
1731 header_csv.extend([rca[u"title"] for rca in rcas])
1733 legend_lst = table.get(u"legend", None)
1734 if legend_lst is None:
1737 legend = u"\n" + u"\n".join(legend_lst) + u"\n"
1741 footnote += f"\n{rca[u'title']}:\n"
1742 footnote += rca[u"data"].get(u"footnote", u"")
1744 csv_file = f"{table[u'output-file']}-csv.csv"
1745 with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1747 u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
1749 for test in tbl_for_csv:
1751 u",".join([f'"{item}"' for item in test]) + u"\n"
1754 for item in legend_lst:
1755 file_handler.write(f'"{item}"\n')
1757 for itm in footnote.split(u"\n"):
1758 file_handler.write(f'"{itm}"\n')
1761 max_lens = [0, ] * len(tbl_cmp_lst[0])
1762 for line in tbl_cmp_lst:
1764 for idx, itm in enumerate(line[1:]):
1765 if itm is None or not isinstance(itm, dict) or \
1766 itm.get(u'mean', None) is None or \
1767 itm.get(u'stdev', None) is None:
1772 f"{round(float(itm[u'mean']) / 1e6, 1)} "
1773 f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1774 replace(u"nan", u"NaN")
1778 f"{round(float(itm[u'mean']) / 1e6, 1):+} "
1779 f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1780 replace(u"nan", u"NaN")
1782 if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]:
1783 max_lens[idx] = len(new_itm.rsplit(u" ", 1)[-1])
1789 for line in tbl_tmp:
1791 for idx, itm in enumerate(line[1:]):
1792 if itm in (u"NT", u"NaN"):
1795 itm_lst = itm.rsplit(u"\u00B1", 1)
1797 f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
1798 row.append(u"\u00B1".join(itm_lst))
1800 rca_nr = rca[u"data"].get(row[0], u"-")
1801 row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1803 tbl_final.append(row)
1805 header = [u"Test Case", ]
1806 header.extend([col[u"title"] for col in cols])
1807 header.extend([comp.get(u"title", u"") for comp in comparisons])
1808 header.extend([rca[u"title"] for rca in rcas])
1810 # Generate csv tables:
1811 csv_file = f"{table[u'output-file']}.csv"
1812 with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1813 file_handler.write(u";".join(header) + u"\n")
1814 for test in tbl_final:
1815 file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1817 # Generate txt table:
1818 txt_file_name = f"{table[u'output-file']}.txt"
1819 convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
1821 with open(txt_file_name, u'a', encoding='utf-8') as txt_file:
1822 txt_file.write(legend)
1823 txt_file.write(footnote)
1825 # Generate html table:
1826 _tpc_generate_html_table(
1829 table[u'output-file'],
1833 title=table.get(u"title", u"")
1837 def table_weekly_comparison(table, in_data):
1838 """Generate the table(s) with algorithm: table_weekly_comparison
1839 specified in the specification file.
1841 :param table: Table to generate.
1842 :param in_data: Data to process.
1843 :type table: pandas.Series
1844 :type in_data: InputData
1846 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1848 # Transform the data
1850 f" Creating the data set for the {table.get(u'type', u'')} "
1851 f"{table.get(u'title', u'')}."
1854 incl_tests = table.get(u"include-tests", None)
1855 if incl_tests not in (u"NDR", u"PDR"):
1856 logging.error(f"Wrong tests to include specified ({incl_tests}).")
1859 nr_cols = table.get(u"nr-of-data-columns", None)
1860 if not nr_cols or nr_cols < 2:
1862 f"No columns specified for {table.get(u'title', u'')}. Skipping."
1866 data = in_data.filter_data(
1868 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1869 continue_on_error=True
1874 [u"Start Timestamp", ],
1880 tb_tbl = table.get(u"testbeds", None)
1881 for job_name, job_data in data.items():
1882 for build_nr, build in job_data.items():
1888 tb_ip = in_data.metadata(job_name, build_nr).get(u"testbed", u"")
1889 if tb_ip and tb_tbl:
1890 testbed = tb_tbl.get(tb_ip, u"")
1893 header[2].insert(1, build_nr)
1894 header[3].insert(1, testbed)
1896 1, in_data.metadata(job_name, build_nr).get(u"generated", u"")
1899 1, in_data.metadata(job_name, build_nr).get(u"version", u"")
1902 for tst_name, tst_data in build.items():
1904 _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
1905 if not tbl_dict.get(tst_name_mod, None):
1906 tbl_dict[tst_name_mod] = dict(
1907 name=tst_data[u'name'].rsplit(u'-', 1)[0],
1910 tbl_dict[tst_name_mod][-idx - 1] = \
1911 tst_data[u"throughput"][incl_tests][u"LOWER"]
1912 except (TypeError, IndexError, KeyError, ValueError):
1917 logging.error(u"Not enough data to build the table! Skipping")
1921 for idx, cmp in enumerate(table.get(u"comparisons", list())):
1922 idx_ref = cmp.get(u"reference", None)
1923 idx_cmp = cmp.get(u"compare", None)
1924 if idx_ref is None or idx_cmp is None:
1927 f"Diff({header[0][idx_ref - idx].split(u'~')[-1]} vs "
1928 f"{header[0][idx_cmp - idx].split(u'~')[-1]})"
1930 header[1].append(u"")
1931 header[2].append(u"")
1932 header[3].append(u"")
1933 for tst_name, tst_data in tbl_dict.items():
1934 if not cmp_dict.get(tst_name, None):
1935 cmp_dict[tst_name] = list()
1936 ref_data = tst_data.get(idx_ref, None)
1937 cmp_data = tst_data.get(idx_cmp, None)
1938 if ref_data is None or cmp_data is None:
1939 cmp_dict[tst_name].append(float(u'nan'))
1941 cmp_dict[tst_name].append(
1942 relative_change(ref_data, cmp_data)
1945 tbl_lst_none = list()
1947 for tst_name, tst_data in tbl_dict.items():
1948 itm_lst = [tst_data[u"name"], ]
1949 for idx in range(nr_cols):
1950 item = tst_data.get(-idx - 1, None)
1952 itm_lst.insert(1, None)
1954 itm_lst.insert(1, round(item / 1e6, 1))
1957 None if itm is None else round(itm, 1)
1958 for itm in cmp_dict[tst_name]
1961 if str(itm_lst[-1]) == u"nan" or itm_lst[-1] is None:
1962 tbl_lst_none.append(itm_lst)
1964 tbl_lst.append(itm_lst)
1966 tbl_lst_none.sort(key=lambda rel: rel[0], reverse=False)
1967 tbl_lst.sort(key=lambda rel: rel[0], reverse=False)
1968 tbl_lst.sort(key=lambda rel: rel[-1], reverse=False)
1969 tbl_lst.extend(tbl_lst_none)
1971 # Generate csv table:
1972 csv_file = f"{table[u'output-file']}.csv"
1973 logging.info(f" Writing the file {csv_file}")
1974 with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1976 file_handler.write(u",".join(hdr) + u"\n")
1977 for test in tbl_lst:
1978 file_handler.write(u",".join(
1980 str(item).replace(u"None", u"-").replace(u"nan", u"-").
1981 replace(u"null", u"-") for item in test
1985 txt_file = f"{table[u'output-file']}.txt"
1986 logging.info(f" Writing the file {txt_file}")
1987 convert_csv_to_pretty_txt(csv_file, txt_file, delimiter=u",")
1989 # Reorganize header in txt table
1991 with open(txt_file, u"rt", encoding='utf-8') as file_handler:
1992 for line in file_handler:
1993 txt_table.append(line)
1995 txt_table.insert(5, txt_table.pop(2))
1996 with open(txt_file, u"wt", encoding='utf-8') as file_handler:
1997 file_handler.writelines(txt_table)
2001 # Generate html table:
2003 u"<br>".join(row) for row in zip(*header)
2005 _tpc_generate_html_table(
2008 table[u'output-file'],
2010 title=table.get(u"title", u""),