1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Algorithms to generate tables.
22 from collections import OrderedDict
23 from xml.etree import ElementTree as ET
24 from datetime import datetime as dt
25 from datetime import timedelta
26 from copy import deepcopy
28 import plotly.graph_objects as go
29 import plotly.offline as ploff
32 from numpy import nan, isnan
33 from yaml import load, FullLoader, YAMLError
35 from pal_utils import mean, stdev, classify_anomalies, \
36 convert_csv_to_pretty_txt, relative_change_stdev, relative_change
39 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
42 def generate_tables(spec, data):
43 """Generate all tables specified in the specification file.
45 :param spec: Specification read from the specification file.
46 :param data: Data to process.
47 :type spec: Specification
52 u"table_merged_details": table_merged_details,
53 u"table_soak_vs_ndr": table_soak_vs_ndr,
54 u"table_perf_trending_dash": table_perf_trending_dash,
55 u"table_perf_trending_dash_html": table_perf_trending_dash_html,
56 u"table_last_failed_tests": table_last_failed_tests,
57 u"table_failed_tests": table_failed_tests,
58 u"table_failed_tests_html": table_failed_tests_html,
59 u"table_oper_data_html": table_oper_data_html,
60 u"table_comparison": table_comparison,
61 u"table_weekly_comparison": table_weekly_comparison
64 logging.info(u"Generating the tables ...")
65 for table in spec.tables:
67 if table[u"algorithm"] == u"table_weekly_comparison":
68 table[u"testbeds"] = spec.environment.get(u"testbeds", None)
69 generator[table[u"algorithm"]](table, data)
70 except NameError as err:
72 f"Probably algorithm {table[u'algorithm']} is not defined: "
75 logging.info(u"Done.")
78 def table_oper_data_html(table, input_data):
79 """Generate the table(s) with algorithm: html_table_oper_data
80 specified in the specification file.
82 :param table: Table to generate.
83 :param input_data: Data to process.
84 :type table: pandas.Series
85 :type input_data: InputData
88 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
91 f" Creating the data set for the {table.get(u'type', u'')} "
92 f"{table.get(u'title', u'')}."
94 data = input_data.filter_data(
96 params=[u"name", u"parent", u"show-run", u"type"],
97 continue_on_error=True
101 data = input_data.merge_data(data)
103 sort_tests = table.get(u"sort", None)
107 ascending=(sort_tests == u"ascending")
109 data.sort_index(**args)
111 suites = input_data.filter_data(
113 continue_on_error=True,
118 suites = input_data.merge_data(suites)
120 def _generate_html_table(tst_data):
121 """Generate an HTML table with operational data for the given test.
123 :param tst_data: Test data to be used to generate the table.
124 :type tst_data: pandas.Series
125 :returns: HTML table with operational data.
130 u"header": u"#7eade7",
131 u"empty": u"#ffffff",
132 u"body": (u"#e9f1fb", u"#d4e4f7")
135 tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
137 trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
138 thead = ET.SubElement(
139 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
141 thead.text = tst_data[u"name"]
143 trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
144 thead = ET.SubElement(
145 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
149 if tst_data.get(u"show-run", u"No Data") == u"No Data":
150 trow = ET.SubElement(
151 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
153 tcol = ET.SubElement(
154 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
156 tcol.text = u"No Data"
158 trow = ET.SubElement(
159 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
161 thead = ET.SubElement(
162 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
164 font = ET.SubElement(
165 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
168 return str(ET.tostring(tbl, encoding=u"unicode"))
175 u"Cycles per Packet",
176 u"Average Vector Size"
179 for dut_data in tst_data[u"show-run"].values():
180 trow = ET.SubElement(
181 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
183 tcol = ET.SubElement(
184 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
186 if dut_data.get(u"threads", None) is None:
187 tcol.text = u"No Data"
190 bold = ET.SubElement(tcol, u"b")
192 f"Host IP: {dut_data.get(u'host', '')}, "
193 f"Socket: {dut_data.get(u'socket', '')}"
195 trow = ET.SubElement(
196 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
198 thead = ET.SubElement(
199 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
203 for thread_nr, thread in dut_data[u"threads"].items():
204 trow = ET.SubElement(
205 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
207 tcol = ET.SubElement(
208 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
210 bold = ET.SubElement(tcol, u"b")
211 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
212 trow = ET.SubElement(
213 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
215 for idx, col in enumerate(tbl_hdr):
216 tcol = ET.SubElement(
218 attrib=dict(align=u"right" if idx else u"left")
220 font = ET.SubElement(
221 tcol, u"font", attrib=dict(size=u"2")
223 bold = ET.SubElement(font, u"b")
225 for row_nr, row in enumerate(thread):
226 trow = ET.SubElement(
228 attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
230 for idx, col in enumerate(row):
231 tcol = ET.SubElement(
233 attrib=dict(align=u"right" if idx else u"left")
235 font = ET.SubElement(
236 tcol, u"font", attrib=dict(size=u"2")
238 if isinstance(col, float):
239 font.text = f"{col:.2f}"
242 trow = ET.SubElement(
243 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
245 thead = ET.SubElement(
246 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
250 trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
251 thead = ET.SubElement(
252 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
254 font = ET.SubElement(
255 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
259 return str(ET.tostring(tbl, encoding=u"unicode"))
261 for suite in suites.values:
263 for test_data in data.values:
264 if test_data[u"parent"] not in suite[u"name"]:
266 html_table += _generate_html_table(test_data)
270 file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
271 with open(f"{file_name}", u'w') as html_file:
272 logging.info(f" Writing file: {file_name}")
273 html_file.write(u".. raw:: html\n\n\t")
274 html_file.write(html_table)
275 html_file.write(u"\n\t<p><br><br></p>\n")
277 logging.warning(u"The output file is not defined.")
279 logging.info(u" Done.")
282 def table_merged_details(table, input_data):
283 """Generate the table(s) with algorithm: table_merged_details
284 specified in the specification file.
286 :param table: Table to generate.
287 :param input_data: Data to process.
288 :type table: pandas.Series
289 :type input_data: InputData
292 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
296 f" Creating the data set for the {table.get(u'type', u'')} "
297 f"{table.get(u'title', u'')}."
299 data = input_data.filter_data(table, continue_on_error=True)
300 data = input_data.merge_data(data)
302 sort_tests = table.get(u"sort", None)
306 ascending=(sort_tests == u"ascending")
308 data.sort_index(**args)
310 suites = input_data.filter_data(
311 table, continue_on_error=True, data_set=u"suites")
312 suites = input_data.merge_data(suites)
314 # Prepare the header of the tables
316 for column in table[u"columns"]:
318 u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
321 for suite in suites.values:
323 suite_name = suite[u"name"]
325 for test in data.keys():
326 if data[test][u"parent"] not in suite_name:
329 for column in table[u"columns"]:
331 col_data = str(data[test][column[
332 u"data"].split(u" ")[1]]).replace(u'"', u'""')
333 # Do not include tests with "Test Failed" in test message
334 if u"Test Failed" in col_data:
336 col_data = col_data.replace(
337 u"No Data", u"Not Captured "
339 if column[u"data"].split(u" ")[1] in (u"name", ):
340 if len(col_data) > 30:
341 col_data_lst = col_data.split(u"-")
342 half = int(len(col_data_lst) / 2)
343 col_data = f"{u'-'.join(col_data_lst[:half])}" \
345 f"{u'-'.join(col_data_lst[half:])}"
346 col_data = f" |prein| {col_data} |preout| "
347 elif column[u"data"].split(u" ")[1] in (u"msg", ):
348 # Temporary solution: remove NDR results from message:
349 if bool(table.get(u'remove-ndr', False)):
351 col_data = col_data.split(u" |br| ", 1)[1]
354 col_data = f" |prein| {col_data} |preout| "
355 elif column[u"data"].split(u" ")[1] in \
356 (u"conf-history", u"show-run"):
357 col_data = col_data.replace(u" |br| ", u"", 1)
358 col_data = f" |prein| {col_data[:-5]} |preout| "
359 row_lst.append(f'"{col_data}"')
361 row_lst.append(u'"Not captured"')
362 if len(row_lst) == len(table[u"columns"]):
363 table_lst.append(row_lst)
365 # Write the data to file
367 separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
368 file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
369 logging.info(f" Writing file: {file_name}")
370 with open(file_name, u"wt") as file_handler:
371 file_handler.write(u",".join(header) + u"\n")
372 for item in table_lst:
373 file_handler.write(u",".join(item) + u"\n")
375 logging.info(u" Done.")
378 def _tpc_modify_test_name(test_name, ignore_nic=False):
379 """Modify a test name by replacing its parts.
381 :param test_name: Test name to be modified.
382 :param ignore_nic: If True, NIC is removed from TC name.
384 :type ignore_nic: bool
385 :returns: Modified test name.
388 test_name_mod = test_name.\
389 replace(u"-ndrpdrdisc", u""). \
390 replace(u"-ndrpdr", u"").\
391 replace(u"-pdrdisc", u""). \
392 replace(u"-ndrdisc", u"").\
393 replace(u"-pdr", u""). \
394 replace(u"-ndr", u""). \
395 replace(u"1t1c", u"1c").\
396 replace(u"2t1c", u"1c"). \
397 replace(u"2t2c", u"2c").\
398 replace(u"4t2c", u"2c"). \
399 replace(u"4t4c", u"4c").\
400 replace(u"8t4c", u"4c")
403 return re.sub(REGEX_NIC, u"", test_name_mod)
407 def _tpc_modify_displayed_test_name(test_name):
408 """Modify a test name which is displayed in a table by replacing its parts.
410 :param test_name: Test name to be modified.
412 :returns: Modified test name.
416 replace(u"1t1c", u"1c").\
417 replace(u"2t1c", u"1c"). \
418 replace(u"2t2c", u"2c").\
419 replace(u"4t2c", u"2c"). \
420 replace(u"4t4c", u"4c").\
421 replace(u"8t4c", u"4c")
424 def _tpc_insert_data(target, src, include_tests):
425 """Insert src data to the target structure.
427 :param target: Target structure where the data is placed.
428 :param src: Source data to be placed into the target stucture.
429 :param include_tests: Which results will be included (MRR, NDR, PDR).
432 :type include_tests: str
435 if include_tests == u"MRR":
436 target[u"mean"] = src[u"result"][u"receive-rate"]
437 target[u"stdev"] = src[u"result"][u"receive-stdev"]
438 elif include_tests == u"PDR":
439 target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
440 elif include_tests == u"NDR":
441 target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
442 except (KeyError, TypeError):
446 def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
447 footnote=u"", sort_data=True, title=u"",
449 """Generate html table from input data with simple sorting possibility.
451 :param header: Table header.
452 :param data: Input data to be included in the table. It is a list of lists.
453 Inner lists are rows in the table. All inner lists must be of the same
454 length. The length of these lists must be the same as the length of the
456 :param out_file_name: The name (relative or full path) where the
457 generated html table is written.
458 :param legend: The legend to display below the table.
459 :param footnote: The footnote to display below the table (and legend).
460 :param sort_data: If True the data sorting is enabled.
461 :param title: The table (and file) title.
462 :param generate_rst: If True, wrapping rst file is generated.
464 :type data: list of lists
465 :type out_file_name: str
468 :type sort_data: bool
470 :type generate_rst: bool
474 idx = header.index(u"Test Case")
480 [u"left", u"left", u"right"],
481 [u"left", u"left", u"left", u"right"]
485 [u"left", u"left", u"right"],
486 [u"left", u"left", u"left", u"right"]
488 u"width": ([15, 9], [4, 24, 10], [4, 4, 32, 10])
491 df_data = pd.DataFrame(data, columns=header)
494 df_sorted = [df_data.sort_values(
495 by=[key, header[idx]], ascending=[True, True]
496 if key != header[idx] else [False, True]) for key in header]
497 df_sorted_rev = [df_data.sort_values(
498 by=[key, header[idx]], ascending=[False, True]
499 if key != header[idx] else [True, True]) for key in header]
500 df_sorted.extend(df_sorted_rev)
504 fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
505 for idx in range(len(df_data))]]
507 values=[f"<b>{item.replace(u',', u',<br>')}</b>" for item in header],
508 fill_color=u"#7eade7",
509 align=params[u"align-hdr"][idx],
511 family=u"Courier New",
519 for table in df_sorted:
520 columns = [table.get(col) for col in header]
523 columnwidth=params[u"width"][idx],
527 fill_color=fill_color,
528 align=params[u"align-itm"][idx],
530 family=u"Courier New",
538 menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
539 menu_items.extend([f"<b>{itm}</b> (descending)" for itm in header])
540 for idx, hdr in enumerate(menu_items):
541 visible = [False, ] * len(menu_items)
545 label=hdr.replace(u" [Mpps]", u""),
547 args=[{u"visible": visible}],
553 go.layout.Updatemenu(
560 active=len(menu_items) - 1,
561 buttons=list(buttons)
568 columnwidth=params[u"width"][idx],
571 values=[df_sorted.get(col) for col in header],
572 fill_color=fill_color,
573 align=params[u"align-itm"][idx],
575 family=u"Courier New",
586 filename=f"{out_file_name}_in.html"
592 file_name = out_file_name.split(u"/")[-1]
593 if u"vpp" in out_file_name:
594 path = u"_tmp/src/vpp_performance_tests/comparisons/"
596 path = u"_tmp/src/dpdk_performance_tests/comparisons/"
597 with open(f"{path}{file_name}.rst", u"wt") as rst_file:
600 u".. |br| raw:: html\n\n <br />\n\n\n"
601 u".. |prein| raw:: html\n\n <pre>\n\n\n"
602 u".. |preout| raw:: html\n\n </pre>\n\n"
605 rst_file.write(f"{title}\n")
606 rst_file.write(f"{u'`' * len(title)}\n\n")
609 f' <iframe frameborder="0" scrolling="no" '
610 f'width="1600" height="1200" '
611 f'src="../..{out_file_name.replace(u"_build", u"")}_in.html">'
615 # TODO: Use html (rst) list for legend and footnote
617 rst_file.write(legend[1:].replace(u"\n", u" |br| "))
619 rst_file.write(footnote.replace(u"\n", u" |br| ")[1:])
622 def table_soak_vs_ndr(table, input_data):
623 """Generate the table(s) with algorithm: table_soak_vs_ndr
624 specified in the specification file.
626 :param table: Table to generate.
627 :param input_data: Data to process.
628 :type table: pandas.Series
629 :type input_data: InputData
632 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
636 f" Creating the data set for the {table.get(u'type', u'')} "
637 f"{table.get(u'title', u'')}."
639 data = input_data.filter_data(table, continue_on_error=True)
641 # Prepare the header of the table
645 f"Avg({table[u'reference'][u'title']})",
646 f"Stdev({table[u'reference'][u'title']})",
647 f"Avg({table[u'compare'][u'title']})",
648 f"Stdev{table[u'compare'][u'title']})",
652 header_str = u";".join(header) + u"\n"
655 f"Avg({table[u'reference'][u'title']}): "
656 f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
657 f"from a series of runs of the listed tests.\n"
658 f"Stdev({table[u'reference'][u'title']}): "
659 f"Standard deviation value of {table[u'reference'][u'title']} "
660 f"[Mpps] computed from a series of runs of the listed tests.\n"
661 f"Avg({table[u'compare'][u'title']}): "
662 f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
663 f"a series of runs of the listed tests.\n"
664 f"Stdev({table[u'compare'][u'title']}): "
665 f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
666 f"computed from a series of runs of the listed tests.\n"
667 f"Diff({table[u'reference'][u'title']},"
668 f"{table[u'compare'][u'title']}): "
669 f"Percentage change calculated for mean values.\n"
671 u"Standard deviation of percentage change calculated for mean "
674 except (AttributeError, KeyError) as err:
675 logging.error(f"The model is invalid, missing parameter: {repr(err)}")
678 # Create a list of available SOAK test results:
680 for job, builds in table[u"compare"][u"data"].items():
682 for tst_name, tst_data in data[job][str(build)].items():
683 if tst_data[u"type"] == u"SOAK":
684 tst_name_mod = tst_name.replace(u"-soak", u"")
685 if tbl_dict.get(tst_name_mod, None) is None:
686 groups = re.search(REGEX_NIC, tst_data[u"parent"])
687 nic = groups.group(0) if groups else u""
690 f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
692 tbl_dict[tst_name_mod] = {
698 tbl_dict[tst_name_mod][u"cmp-data"].append(
699 tst_data[u"throughput"][u"LOWER"])
700 except (KeyError, TypeError):
702 tests_lst = tbl_dict.keys()
704 # Add corresponding NDR test results:
705 for job, builds in table[u"reference"][u"data"].items():
707 for tst_name, tst_data in data[job][str(build)].items():
708 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
709 replace(u"-mrr", u"")
710 if tst_name_mod not in tests_lst:
713 if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
715 if table[u"include-tests"] == u"MRR":
716 result = (tst_data[u"result"][u"receive-rate"],
717 tst_data[u"result"][u"receive-stdev"])
718 elif table[u"include-tests"] == u"PDR":
720 tst_data[u"throughput"][u"PDR"][u"LOWER"]
721 elif table[u"include-tests"] == u"NDR":
723 tst_data[u"throughput"][u"NDR"][u"LOWER"]
726 if result is not None:
727 tbl_dict[tst_name_mod][u"ref-data"].append(
729 except (KeyError, TypeError):
733 for tst_name in tbl_dict:
734 item = [tbl_dict[tst_name][u"name"], ]
735 data_r = tbl_dict[tst_name][u"ref-data"]
737 if table[u"include-tests"] == u"MRR":
738 data_r_mean = data_r[0][0]
739 data_r_stdev = data_r[0][1]
741 data_r_mean = mean(data_r)
742 data_r_stdev = stdev(data_r)
743 item.append(round(data_r_mean / 1e6, 1))
744 item.append(round(data_r_stdev / 1e6, 1))
748 item.extend([None, None])
749 data_c = tbl_dict[tst_name][u"cmp-data"]
751 if table[u"include-tests"] == u"MRR":
752 data_c_mean = data_c[0][0]
753 data_c_stdev = data_c[0][1]
755 data_c_mean = mean(data_c)
756 data_c_stdev = stdev(data_c)
757 item.append(round(data_c_mean / 1e6, 1))
758 item.append(round(data_c_stdev / 1e6, 1))
762 item.extend([None, None])
763 if data_r_mean is not None and data_c_mean is not None:
764 delta, d_stdev = relative_change_stdev(
765 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
767 item.append(round(delta))
771 item.append(round(d_stdev))
776 # Sort the table according to the relative change
777 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
779 # Generate csv tables:
780 csv_file = f"{table[u'output-file']}.csv"
781 with open(csv_file, u"wt") as file_handler:
782 file_handler.write(header_str)
784 file_handler.write(u";".join([str(item) for item in test]) + u"\n")
786 convert_csv_to_pretty_txt(
787 csv_file, f"{table[u'output-file']}.txt", delimiter=u";"
789 with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
790 txt_file.write(legend)
792 # Generate html table:
793 _tpc_generate_html_table(
796 table[u'output-file'],
798 title=table.get(u"title", u"")
802 def table_perf_trending_dash(table, input_data):
803 """Generate the table(s) with algorithm:
804 table_perf_trending_dash
805 specified in the specification file.
807 :param table: Table to generate.
808 :param input_data: Data to process.
809 :type table: pandas.Series
810 :type input_data: InputData
813 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
817 f" Creating the data set for the {table.get(u'type', u'')} "
818 f"{table.get(u'title', u'')}."
820 data = input_data.filter_data(table, continue_on_error=True)
822 # Prepare the header of the tables
826 u"Short-Term Change [%]",
827 u"Long-Term Change [%]",
831 header_str = u",".join(header) + u"\n"
833 incl_tests = table.get(u"include-tests", u"MRR")
835 # Prepare data to the table:
837 for job, builds in table[u"data"].items():
839 for tst_name, tst_data in data[job][str(build)].items():
840 if tst_name.lower() in table.get(u"ignore-list", list()):
842 if tbl_dict.get(tst_name, None) is None:
843 groups = re.search(REGEX_NIC, tst_data[u"parent"])
846 nic = groups.group(0)
847 tbl_dict[tst_name] = {
848 u"name": f"{nic}-{tst_data[u'name']}",
849 u"data": OrderedDict()
852 if incl_tests == u"MRR":
853 tbl_dict[tst_name][u"data"][str(build)] = \
854 tst_data[u"result"][u"receive-rate"]
855 elif incl_tests == u"NDR":
856 tbl_dict[tst_name][u"data"][str(build)] = \
857 tst_data[u"throughput"][u"NDR"][u"LOWER"]
858 elif incl_tests == u"PDR":
859 tbl_dict[tst_name][u"data"][str(build)] = \
860 tst_data[u"throughput"][u"PDR"][u"LOWER"]
861 except (TypeError, KeyError):
862 pass # No data in output.xml for this test
865 for tst_name in tbl_dict:
866 data_t = tbl_dict[tst_name][u"data"]
870 classification_lst, avgs, _ = classify_anomalies(data_t)
872 win_size = min(len(data_t), table[u"window"])
873 long_win_size = min(len(data_t), table[u"long-trend-window"])
877 [x for x in avgs[-long_win_size:-win_size]
882 avg_week_ago = avgs[max(-win_size, -len(avgs))]
884 if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
885 rel_change_last = nan
887 rel_change_last = round(
888 ((last_avg - avg_week_ago) / avg_week_ago) * 1e2, 2)
890 if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
891 rel_change_long = nan
893 rel_change_long = round(
894 ((last_avg - max_long_avg) / max_long_avg) * 1e2, 2)
896 if classification_lst:
897 if isnan(rel_change_last) and isnan(rel_change_long):
899 if isnan(last_avg) or isnan(rel_change_last) or \
900 isnan(rel_change_long):
903 [tbl_dict[tst_name][u"name"],
904 round(last_avg / 1e6, 2),
907 classification_lst[-win_size+1:].count(u"regression"),
908 classification_lst[-win_size+1:].count(u"progression")])
910 tbl_lst.sort(key=lambda rel: rel[0])
913 for nrr in range(table[u"window"], -1, -1):
914 tbl_reg = [item for item in tbl_lst if item[4] == nrr]
915 for nrp in range(table[u"window"], -1, -1):
916 tbl_out = [item for item in tbl_reg if item[5] == nrp]
917 tbl_out.sort(key=lambda rel: rel[2])
918 tbl_sorted.extend(tbl_out)
920 file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
922 logging.info(f" Writing file: {file_name}")
923 with open(file_name, u"wt") as file_handler:
924 file_handler.write(header_str)
925 for test in tbl_sorted:
926 file_handler.write(u",".join([str(item) for item in test]) + u'\n')
928 logging.info(f" Writing file: {table[u'output-file']}.txt")
929 convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
932 def _generate_url(testbed, test_name):
933 """Generate URL to a trending plot from the name of the test case.
935 :param testbed: The testbed used for testing.
936 :param test_name: The name of the test case.
939 :returns: The URL to the plot with the trending data for the given test
944 if u"x520" in test_name:
946 elif u"x710" in test_name:
948 elif u"xl710" in test_name:
950 elif u"xxv710" in test_name:
952 elif u"vic1227" in test_name:
954 elif u"vic1385" in test_name:
956 elif u"x553" in test_name:
958 elif u"cx556" in test_name or u"cx556a" in test_name:
963 if u"64b" in test_name:
965 elif u"78b" in test_name:
967 elif u"imix" in test_name:
969 elif u"9000b" in test_name:
970 frame_size = u"9000b"
971 elif u"1518b" in test_name:
972 frame_size = u"1518b"
973 elif u"114b" in test_name:
978 if u"1t1c" in test_name or \
979 (u"-1c-" in test_name and
980 testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
982 elif u"2t2c" in test_name or \
983 (u"-2c-" in test_name and
984 testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
986 elif u"4t4c" in test_name or \
987 (u"-4c-" in test_name and
988 testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
990 elif u"2t1c" in test_name or \
991 (u"-1c-" in test_name and
992 testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
994 elif u"4t2c" in test_name or \
995 (u"-2c-" in test_name and
996 testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
998 elif u"8t4c" in test_name or \
999 (u"-4c-" in test_name and
1000 testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1005 if u"testpmd" in test_name:
1007 elif u"l3fwd" in test_name:
1009 elif u"avf" in test_name:
1011 elif u"rdma" in test_name:
1013 elif u"dnv" in testbed or u"tsh" in testbed:
1018 if u"macip-iacl1s" in test_name:
1019 bsf = u"features-macip-iacl1"
1020 elif u"macip-iacl10s" in test_name:
1021 bsf = u"features-macip-iacl01"
1022 elif u"macip-iacl50s" in test_name:
1023 bsf = u"features-macip-iacl50"
1024 elif u"iacl1s" in test_name:
1025 bsf = u"features-iacl1"
1026 elif u"iacl10s" in test_name:
1027 bsf = u"features-iacl10"
1028 elif u"iacl50s" in test_name:
1029 bsf = u"features-iacl50"
1030 elif u"oacl1s" in test_name:
1031 bsf = u"features-oacl1"
1032 elif u"oacl10s" in test_name:
1033 bsf = u"features-oacl10"
1034 elif u"oacl50s" in test_name:
1035 bsf = u"features-oacl50"
1036 elif u"udpsrcscale" in test_name:
1037 bsf = u"features-udp"
1038 elif u"iacl" in test_name:
1040 elif u"policer" in test_name:
1042 elif u"cop" in test_name:
1044 elif u"nat" in test_name:
1046 elif u"macip" in test_name:
1048 elif u"scale" in test_name:
1050 elif u"base" in test_name:
1055 if u"114b" in test_name and u"vhost" in test_name:
1057 elif u"testpmd" in test_name or u"l3fwd" in test_name:
1059 elif u"memif" in test_name:
1060 domain = u"container_memif"
1061 elif u"srv6" in test_name:
1063 elif u"vhost" in test_name:
1065 if u"vppl2xc" in test_name:
1068 driver += u"-testpmd"
1069 if u"lbvpplacp" in test_name:
1070 bsf += u"-link-bonding"
1071 elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1072 domain = u"nf_service_density_vnfc"
1073 elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1074 domain = u"nf_service_density_cnfc"
1075 elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1076 domain = u"nf_service_density_cnfp"
1077 elif u"ipsec" in test_name:
1079 if u"sw" in test_name:
1081 elif u"hw" in test_name:
1083 elif u"ethip4vxlan" in test_name:
1084 domain = u"ip4_tunnels"
1085 elif u"ip4base" in test_name or u"ip4scale" in test_name:
1087 elif u"ip6base" in test_name or u"ip6scale" in test_name:
1089 elif u"l2xcbase" in test_name or \
1090 u"l2xcscale" in test_name or \
1091 u"l2bdbasemaclrn" in test_name or \
1092 u"l2bdscale" in test_name or \
1093 u"l2patch" in test_name:
1098 file_name = u"-".join((domain, testbed, nic)) + u".html#"
1099 anchor_name = u"-".join((frame_size, cores, bsf, driver))
1101 return file_name + anchor_name
1104 def table_perf_trending_dash_html(table, input_data):
1105 """Generate the table(s) with algorithm:
1106 table_perf_trending_dash_html specified in the specification
1109 :param table: Table to generate.
1110 :param input_data: Data to process.
1112 :type input_data: InputData
1117 if not table.get(u"testbed", None):
1119 f"The testbed is not defined for the table "
1120 f"{table.get(u'title', u'')}. Skipping."
1124 test_type = table.get(u"test-type", u"MRR")
1125 if test_type not in (u"MRR", u"NDR", u"PDR"):
1127 f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
1132 if test_type in (u"NDR", u"PDR"):
1133 lnk_dir = u"../ndrpdr_trending/"
1134 lnk_sufix = f"-{test_type.lower()}"
1136 lnk_dir = u"../trending/"
1139 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1142 with open(table[u"input-file"], u'rt') as csv_file:
1143 csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1145 logging.warning(u"The input file is not defined.")
1147 except csv.Error as err:
1149 f"Not possible to process the file {table[u'input-file']}.\n"
1155 dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1158 trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1159 for idx, item in enumerate(csv_lst[0]):
1160 alignment = u"left" if idx == 0 else u"center"
1161 thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1179 for r_idx, row in enumerate(csv_lst[1:]):
1181 color = u"regression"
1183 color = u"progression"
1186 trow = ET.SubElement(
1187 dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
1191 for c_idx, item in enumerate(row):
1192 tdata = ET.SubElement(
1195 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1198 if c_idx == 0 and table.get(u"add-links", True):
1199 ref = ET.SubElement(
1204 f"{_generate_url(table.get(u'testbed', ''), item)}"
1212 with open(table[u"output-file"], u'w') as html_file:
1213 logging.info(f" Writing file: {table[u'output-file']}")
1214 html_file.write(u".. raw:: html\n\n\t")
1215 html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
1216 html_file.write(u"\n\t<p><br><br></p>\n")
1218 logging.warning(u"The output file is not defined.")
1222 def table_last_failed_tests(table, input_data):
1223 """Generate the table(s) with algorithm: table_last_failed_tests
1224 specified in the specification file.
1226 :param table: Table to generate.
1227 :param input_data: Data to process.
1228 :type table: pandas.Series
1229 :type input_data: InputData
1232 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1234 # Transform the data
1236 f" Creating the data set for the {table.get(u'type', u'')} "
1237 f"{table.get(u'title', u'')}."
1240 data = input_data.filter_data(table, continue_on_error=True)
1242 if data is None or data.empty:
1244 f" No data for the {table.get(u'type', u'')} "
1245 f"{table.get(u'title', u'')}."
1250 for job, builds in table[u"data"].items():
1251 for build in builds:
1254 version = input_data.metadata(job, build).get(u"version", u"")
1256 logging.error(f"Data for {job}: {build} is not present.")
1258 tbl_list.append(build)
1259 tbl_list.append(version)
1260 failed_tests = list()
1263 for tst_data in data[job][build].values:
1264 if tst_data[u"status"] != u"FAIL":
1268 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1271 nic = groups.group(0)
1272 failed_tests.append(f"{nic}-{tst_data[u'name']}")
1273 tbl_list.append(str(passed))
1274 tbl_list.append(str(failed))
1275 tbl_list.extend(failed_tests)
1277 file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1278 logging.info(f" Writing file: {file_name}")
1279 with open(file_name, u"wt") as file_handler:
1280 for test in tbl_list:
1281 file_handler.write(test + u'\n')
1284 def table_failed_tests(table, input_data):
1285 """Generate the table(s) with algorithm: table_failed_tests
1286 specified in the specification file.
1288 :param table: Table to generate.
1289 :param input_data: Data to process.
1290 :type table: pandas.Series
1291 :type input_data: InputData
1294 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1296 # Transform the data
1298 f" Creating the data set for the {table.get(u'type', u'')} "
1299 f"{table.get(u'title', u'')}."
1301 data = input_data.filter_data(table, continue_on_error=True)
1304 if u"NDRPDR" in table.get(u"filter", list()):
1305 test_type = u"NDRPDR"
1307 # Prepare the header of the tables
1311 u"Last Failure [Time]",
1312 u"Last Failure [VPP-Build-Id]",
1313 u"Last Failure [CSIT-Job-Build-Id]"
1316 # Generate the data for the table according to the model in the table
1320 timeperiod = timedelta(int(table.get(u"window", 7)))
1323 for job, builds in table[u"data"].items():
1324 for build in builds:
1326 for tst_name, tst_data in data[job][build].items():
1327 if tst_name.lower() in table.get(u"ignore-list", list()):
1329 if tbl_dict.get(tst_name, None) is None:
1330 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1333 nic = groups.group(0)
1334 tbl_dict[tst_name] = {
1335 u"name": f"{nic}-{tst_data[u'name']}",
1336 u"data": OrderedDict()
1339 generated = input_data.metadata(job, build).\
1340 get(u"generated", u"")
1343 then = dt.strptime(generated, u"%Y%m%d %H:%M")
1344 if (now - then) <= timeperiod:
1345 tbl_dict[tst_name][u"data"][build] = (
1346 tst_data[u"status"],
1348 input_data.metadata(job, build).get(u"version",
1352 except (TypeError, KeyError) as err:
1353 logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
1357 for tst_data in tbl_dict.values():
1359 fails_last_date = u""
1360 fails_last_vpp = u""
1361 fails_last_csit = u""
1362 for val in tst_data[u"data"].values():
1363 if val[0] == u"FAIL":
1365 fails_last_date = val[1]
1366 fails_last_vpp = val[2]
1367 fails_last_csit = val[3]
1369 max_fails = fails_nr if fails_nr > max_fails else max_fails
1375 f"{u'mrr-daily' if test_type == u'MRR' else u'ndrpdr-weekly'}"
1376 f"-build-{fails_last_csit}"
1379 tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1381 for nrf in range(max_fails, -1, -1):
1382 tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1383 tbl_sorted.extend(tbl_fails)
1385 file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1386 logging.info(f" Writing file: {file_name}")
1387 with open(file_name, u"wt") as file_handler:
1388 file_handler.write(u",".join(header) + u"\n")
1389 for test in tbl_sorted:
1390 file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1392 logging.info(f" Writing file: {table[u'output-file']}.txt")
1393 convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1396 def table_failed_tests_html(table, input_data):
1397 """Generate the table(s) with algorithm: table_failed_tests_html
1398 specified in the specification file.
1400 :param table: Table to generate.
1401 :param input_data: Data to process.
1402 :type table: pandas.Series
1403 :type input_data: InputData
1408 if not table.get(u"testbed", None):
1410 f"The testbed is not defined for the table "
1411 f"{table.get(u'title', u'')}. Skipping."
1415 test_type = table.get(u"test-type", u"MRR")
1416 if test_type not in (u"MRR", u"NDR", u"PDR", u"NDRPDR"):
1418 f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
1423 if test_type in (u"NDRPDR", u"NDR", u"PDR"):
1424 lnk_dir = u"../ndrpdr_trending/"
1427 lnk_dir = u"../trending/"
1430 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1433 with open(table[u"input-file"], u'rt') as csv_file:
1434 csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1436 logging.warning(u"The input file is not defined.")
1438 except csv.Error as err:
1440 f"Not possible to process the file {table[u'input-file']}.\n"
1446 failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1449 trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1450 for idx, item in enumerate(csv_lst[0]):
1451 alignment = u"left" if idx == 0 else u"center"
1452 thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1456 colors = (u"#e9f1fb", u"#d4e4f7")
1457 for r_idx, row in enumerate(csv_lst[1:]):
1458 background = colors[r_idx % 2]
1459 trow = ET.SubElement(
1460 failed_tests, u"tr", attrib=dict(bgcolor=background)
1464 for c_idx, item in enumerate(row):
1465 tdata = ET.SubElement(
1468 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1471 if c_idx == 0 and table.get(u"add-links", True):
1472 ref = ET.SubElement(
1477 f"{_generate_url(table.get(u'testbed', ''), item)}"
1485 with open(table[u"output-file"], u'w') as html_file:
1486 logging.info(f" Writing file: {table[u'output-file']}")
1487 html_file.write(u".. raw:: html\n\n\t")
1488 html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
1489 html_file.write(u"\n\t<p><br><br></p>\n")
1491 logging.warning(u"The output file is not defined.")
1495 def table_comparison(table, input_data):
1496 """Generate the table(s) with algorithm: table_comparison
1497 specified in the specification file.
1499 :param table: Table to generate.
1500 :param input_data: Data to process.
1501 :type table: pandas.Series
1502 :type input_data: InputData
1504 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1506 # Transform the data
1508 f" Creating the data set for the {table.get(u'type', u'')} "
1509 f"{table.get(u'title', u'')}."
1512 columns = table.get(u"columns", None)
1515 f"No columns specified for {table.get(u'title', u'')}. Skipping."
1520 for idx, col in enumerate(columns):
1521 if col.get(u"data-set", None) is None:
1522 logging.warning(f"No data for column {col.get(u'title', u'')}")
1524 tag = col.get(u"tag", None)
1525 data = input_data.filter_data(
1527 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1528 data=col[u"data-set"],
1529 continue_on_error=True
1532 u"title": col.get(u"title", f"Column{idx}"),
1535 for builds in data.values:
1536 for build in builds:
1537 for tst_name, tst_data in build.items():
1538 if tag and tag not in tst_data[u"tags"]:
1541 _tpc_modify_test_name(tst_name, ignore_nic=True).\
1542 replace(u"2n1l-", u"")
1543 if col_data[u"data"].get(tst_name_mod, None) is None:
1544 name = tst_data[u'name'].rsplit(u'-', 1)[0]
1545 if u"across testbeds" in table[u"title"].lower() or \
1546 u"across topologies" in table[u"title"].lower():
1547 name = _tpc_modify_displayed_test_name(name)
1548 col_data[u"data"][tst_name_mod] = {
1556 target=col_data[u"data"][tst_name_mod],
1558 include_tests=table[u"include-tests"]
1561 replacement = col.get(u"data-replacement", None)
1563 rpl_data = input_data.filter_data(
1565 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1567 continue_on_error=True
1569 for builds in rpl_data.values:
1570 for build in builds:
1571 for tst_name, tst_data in build.items():
1572 if tag and tag not in tst_data[u"tags"]:
1575 _tpc_modify_test_name(tst_name, ignore_nic=True).\
1576 replace(u"2n1l-", u"")
1577 if col_data[u"data"].get(tst_name_mod, None) is None:
1578 name = tst_data[u'name'].rsplit(u'-', 1)[0]
1579 if u"across testbeds" in table[u"title"].lower() \
1580 or u"across topologies" in \
1581 table[u"title"].lower():
1582 name = _tpc_modify_displayed_test_name(name)
1583 col_data[u"data"][tst_name_mod] = {
1590 if col_data[u"data"][tst_name_mod][u"replace"]:
1591 col_data[u"data"][tst_name_mod][u"replace"] = False
1592 col_data[u"data"][tst_name_mod][u"data"] = list()
1594 target=col_data[u"data"][tst_name_mod],
1596 include_tests=table[u"include-tests"]
1599 if table[u"include-tests"] in (u"NDR", u"PDR"):
1600 for tst_name, tst_data in col_data[u"data"].items():
1601 if tst_data[u"data"]:
1602 tst_data[u"mean"] = mean(tst_data[u"data"])
1603 tst_data[u"stdev"] = stdev(tst_data[u"data"])
1605 cols.append(col_data)
1609 for tst_name, tst_data in col[u"data"].items():
1610 if tbl_dict.get(tst_name, None) is None:
1611 tbl_dict[tst_name] = {
1612 "name": tst_data[u"name"]
1614 tbl_dict[tst_name][col[u"title"]] = {
1615 u"mean": tst_data[u"mean"],
1616 u"stdev": tst_data[u"stdev"]
1620 logging.warning(f"No data for table {table.get(u'title', u'')}!")
1624 for tst_data in tbl_dict.values():
1625 row = [tst_data[u"name"], ]
1627 row.append(tst_data.get(col[u"title"], None))
1630 comparisons = table.get(u"comparisons", None)
1631 if comparisons and isinstance(comparisons, list):
1632 for idx, comp in enumerate(comparisons):
1634 col_ref = int(comp[u"reference"])
1635 col_cmp = int(comp[u"compare"])
1637 logging.warning(u"Comparison: No references defined! Skipping.")
1638 comparisons.pop(idx)
1640 if not (0 < col_ref <= len(cols) and
1641 0 < col_cmp <= len(cols)) or \
1643 logging.warning(f"Wrong values of reference={col_ref} "
1644 f"and/or compare={col_cmp}. Skipping.")
1645 comparisons.pop(idx)
1648 tbl_cmp_lst = list()
1651 new_row = deepcopy(row)
1653 for comp in comparisons:
1654 ref_itm = row[int(comp[u"reference"])]
1655 if ref_itm is None and \
1656 comp.get(u"reference-alt", None) is not None:
1657 ref_itm = row[int(comp[u"reference-alt"])]
1658 cmp_itm = row[int(comp[u"compare"])]
1659 if ref_itm is not None and cmp_itm is not None and \
1660 ref_itm[u"mean"] is not None and \
1661 cmp_itm[u"mean"] is not None and \
1662 ref_itm[u"stdev"] is not None and \
1663 cmp_itm[u"stdev"] is not None:
1664 delta, d_stdev = relative_change_stdev(
1665 ref_itm[u"mean"], cmp_itm[u"mean"],
1666 ref_itm[u"stdev"], cmp_itm[u"stdev"]
1670 u"mean": delta * 1e6,
1671 u"stdev": d_stdev * 1e6
1676 new_row.append(None)
1678 tbl_cmp_lst.append(new_row)
1680 tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
1681 tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
1684 rca_in = table.get(u"rca", None)
1685 if rca_in and isinstance(rca_in, list):
1686 for idx, itm in enumerate(rca_in):
1688 with open(itm.get(u"data", u""), u"r") as rca_file:
1691 u"title": itm.get(u"title", f"RCA{idx}"),
1692 u"data": load(rca_file, Loader=FullLoader)
1695 except (YAMLError, IOError) as err:
1697 f"The RCA file {itm.get(u'data', u'')} does not exist or "
1700 logging.debug(repr(err))
1702 tbl_for_csv = list()
1703 for line in tbl_cmp_lst:
1705 for idx, itm in enumerate(line[1:]):
1710 row.append(round(float(itm[u'mean']) / 1e6, 3))
1711 row.append(round(float(itm[u'stdev']) / 1e6, 3))
1713 rca_nr = rca[u"data"].get(row[0], u"-")
1714 row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1715 tbl_for_csv.append(row)
1717 header_csv = [u"Test Case", ]
1719 header_csv.append(f"Avg({col[u'title']})")
1720 header_csv.append(f"Stdev({col[u'title']})")
1721 for comp in comparisons:
1723 f"Avg({comp.get(u'title', u'')})"
1726 f"Stdev({comp.get(u'title', u'')})"
1728 header_csv.extend([rca[u"title"] for rca in rcas])
1730 legend_lst = table.get(u"legend", None)
1731 if legend_lst is None:
1734 legend = u"\n" + u"\n".join(legend_lst) + u"\n"
1738 footnote += f"\n{rca[u'title']}:\n"
1739 footnote += rca[u"data"].get(u"footnote", u"")
1741 csv_file = f"{table[u'output-file']}-csv.csv"
1742 with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1744 u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
1746 for test in tbl_for_csv:
1748 u",".join([f'"{item}"' for item in test]) + u"\n"
1751 for item in legend_lst:
1752 file_handler.write(f'"{item}"\n')
1754 for itm in footnote.split(u"\n"):
1755 file_handler.write(f'"{itm}"\n')
1758 max_lens = [0, ] * len(tbl_cmp_lst[0])
1759 for line in tbl_cmp_lst:
1761 for idx, itm in enumerate(line[1:]):
1767 f"{round(float(itm[u'mean']) / 1e6, 1)} "
1768 f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1769 replace(u"nan", u"NaN")
1773 f"{round(float(itm[u'mean']) / 1e6, 1):+} "
1774 f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1775 replace(u"nan", u"NaN")
1777 if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]:
1778 max_lens[idx] = len(new_itm.rsplit(u" ", 1)[-1])
1784 for line in tbl_tmp:
1786 for idx, itm in enumerate(line[1:]):
1787 if itm in (u"NT", u"NaN"):
1790 itm_lst = itm.rsplit(u"\u00B1", 1)
1792 f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
1793 row.append(u"\u00B1".join(itm_lst))
1795 rca_nr = rca[u"data"].get(row[0], u"-")
1796 row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1798 tbl_final.append(row)
1800 header = [u"Test Case", ]
1801 header.extend([col[u"title"] for col in cols])
1802 header.extend([comp.get(u"title", u"") for comp in comparisons])
1803 header.extend([rca[u"title"] for rca in rcas])
1805 # Generate csv tables:
1806 csv_file = f"{table[u'output-file']}.csv"
1807 with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1808 file_handler.write(u";".join(header) + u"\n")
1809 for test in tbl_final:
1810 file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1812 # Generate txt table:
1813 txt_file_name = f"{table[u'output-file']}.txt"
1814 convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
1816 with open(txt_file_name, u'a', encoding='utf-8') as txt_file:
1817 txt_file.write(legend)
1818 txt_file.write(footnote)
1820 # Generate html table:
1821 _tpc_generate_html_table(
1824 table[u'output-file'],
1828 title=table.get(u"title", u"")
1832 def table_weekly_comparison(table, in_data):
1833 """Generate the table(s) with algorithm: table_weekly_comparison
1834 specified in the specification file.
1836 :param table: Table to generate.
1837 :param in_data: Data to process.
1838 :type table: pandas.Series
1839 :type in_data: InputData
1841 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1843 # Transform the data
1845 f" Creating the data set for the {table.get(u'type', u'')} "
1846 f"{table.get(u'title', u'')}."
1849 incl_tests = table.get(u"include-tests", None)
1850 if incl_tests not in (u"NDR", u"PDR"):
1851 logging.error(f"Wrong tests to include specified ({incl_tests}).")
1854 nr_cols = table.get(u"nr-of-data-columns", None)
1855 if not nr_cols or nr_cols < 2:
1857 f"No columns specified for {table.get(u'title', u'')}. Skipping."
1861 data = in_data.filter_data(
1863 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1864 continue_on_error=True
1869 [u"Start Timestamp", ],
1875 tb_tbl = table.get(u"testbeds", None)
1876 for job_name, job_data in data.items():
1877 for build_nr, build in job_data.items():
1883 tb_ip = in_data.metadata(job_name, build_nr).get(u"testbed", u"")
1884 if tb_ip and tb_tbl:
1885 testbed = tb_tbl.get(tb_ip, u"")
1888 header[2].insert(1, build_nr)
1889 header[3].insert(1, testbed)
1891 1, in_data.metadata(job_name, build_nr).get(u"generated", u"")
1894 1, in_data.metadata(job_name, build_nr).get(u"version", u"")
1897 for tst_name, tst_data in build.items():
1899 _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
1900 if not tbl_dict.get(tst_name_mod, None):
1901 tbl_dict[tst_name_mod] = dict(
1902 name=tst_data[u'name'].rsplit(u'-', 1)[0],
1905 tbl_dict[tst_name_mod][-idx - 1] = \
1906 tst_data[u"throughput"][incl_tests][u"LOWER"]
1907 except (TypeError, IndexError, KeyError, ValueError):
1912 logging.error(u"Not enough data to build the table! Skipping")
1916 for idx, cmp in enumerate(table.get(u"comparisons", list())):
1917 idx_ref = cmp.get(u"reference", None)
1918 idx_cmp = cmp.get(u"compare", None)
1919 if idx_ref is None or idx_cmp is None:
1922 f"Diff({header[0][idx_ref - idx].split(u'~')[-1]} vs "
1923 f"{header[0][idx_cmp - idx].split(u'~')[-1]})"
1925 header[1].append(u"")
1926 header[2].append(u"")
1927 header[3].append(u"")
1928 for tst_name, tst_data in tbl_dict.items():
1929 if not cmp_dict.get(tst_name, None):
1930 cmp_dict[tst_name] = list()
1931 ref_data = tst_data.get(idx_ref, None)
1932 cmp_data = tst_data.get(idx_cmp, None)
1933 if ref_data is None or cmp_data is None:
1934 cmp_dict[tst_name].append(float('nan'))
1936 cmp_dict[tst_name].append(
1937 relative_change(ref_data, cmp_data)
1941 for tst_name, tst_data in tbl_dict.items():
1942 itm_lst = [tst_data[u"name"], ]
1943 for idx in range(nr_cols):
1944 item = tst_data.get(-idx - 1, None)
1946 itm_lst.insert(1, None)
1948 itm_lst.insert(1, round(item / 1e6, 1))
1951 None if itm is None else round(itm, 1)
1952 for itm in cmp_dict[tst_name]
1955 tbl_lst.append(itm_lst)
1957 tbl_lst.sort(key=lambda rel: rel[0], reverse=False)
1958 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1960 # Generate csv table:
1961 csv_file = f"{table[u'output-file']}.csv"
1962 with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1964 file_handler.write(u",".join(hdr) + u"\n")
1965 for test in tbl_lst:
1966 file_handler.write(u",".join(
1968 str(item).replace(u"None", u"-").replace(u"nan", u"-").
1969 replace(u"null", u"-") for item in test
1973 txt_file = f"{table[u'output-file']}.txt"
1974 convert_csv_to_pretty_txt(csv_file, txt_file, delimiter=u",")
1976 # Reorganize header in txt table
1978 with open(txt_file, u"rt", encoding='utf-8') as file_handler:
1979 for line in file_handler:
1980 txt_table.append(line)
1982 txt_table.insert(5, txt_table.pop(2))
1983 with open(txt_file, u"wt", encoding='utf-8') as file_handler:
1984 file_handler.writelines(txt_table)
1988 # Generate html table:
1990 u"<br>".join(row) for row in zip(*header)
1992 _tpc_generate_html_table(
1995 table[u'output-file'],
1997 title=table.get(u"title", u""),