1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Algorithms to generate tables.
22 from collections import OrderedDict
23 from xml.etree import ElementTree as ET
24 from datetime import datetime as dt
25 from datetime import timedelta
26 from copy import deepcopy
28 import plotly.graph_objects as go
29 import plotly.offline as ploff
32 from numpy import nan, isnan
33 from yaml import load, FullLoader, YAMLError
35 from pal_utils import mean, stdev, classify_anomalies, \
36 convert_csv_to_pretty_txt, relative_change_stdev, relative_change
39 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
42 def generate_tables(spec, data):
43 """Generate all tables specified in the specification file.
45 :param spec: Specification read from the specification file.
46 :param data: Data to process.
47 :type spec: Specification
52 u"table_merged_details": table_merged_details,
53 u"table_soak_vs_ndr": table_soak_vs_ndr,
54 u"table_perf_trending_dash": table_perf_trending_dash,
55 u"table_perf_trending_dash_html": table_perf_trending_dash_html,
56 u"table_last_failed_tests": table_last_failed_tests,
57 u"table_failed_tests": table_failed_tests,
58 u"table_failed_tests_html": table_failed_tests_html,
59 u"table_oper_data_html": table_oper_data_html,
60 u"table_comparison": table_comparison,
61 u"table_weekly_comparison": table_weekly_comparison
64 logging.info(u"Generating the tables ...")
65 for table in spec.tables:
67 if table[u"algorithm"] == u"table_weekly_comparison":
68 table[u"testbeds"] = spec.environment.get(u"testbeds", None)
69 generator[table[u"algorithm"]](table, data)
70 except NameError as err:
72 f"Probably algorithm {table[u'algorithm']} is not defined: "
75 logging.info(u"Done.")
78 def table_oper_data_html(table, input_data):
79 """Generate the table(s) with algorithm: html_table_oper_data
80 specified in the specification file.
82 :param table: Table to generate.
83 :param input_data: Data to process.
84 :type table: pandas.Series
85 :type input_data: InputData
88 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
91 f" Creating the data set for the {table.get(u'type', u'')} "
92 f"{table.get(u'title', u'')}."
94 data = input_data.filter_data(
96 params=[u"name", u"parent", u"show-run", u"type"],
97 continue_on_error=True
101 data = input_data.merge_data(data)
103 sort_tests = table.get(u"sort", None)
107 ascending=(sort_tests == u"ascending")
109 data.sort_index(**args)
111 suites = input_data.filter_data(
113 continue_on_error=True,
118 suites = input_data.merge_data(suites)
120 def _generate_html_table(tst_data):
121 """Generate an HTML table with operational data for the given test.
123 :param tst_data: Test data to be used to generate the table.
124 :type tst_data: pandas.Series
125 :returns: HTML table with operational data.
130 u"header": u"#7eade7",
131 u"empty": u"#ffffff",
132 u"body": (u"#e9f1fb", u"#d4e4f7")
135 tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
137 trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
138 thead = ET.SubElement(
139 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
141 thead.text = tst_data[u"name"]
143 trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
144 thead = ET.SubElement(
145 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
149 if tst_data.get(u"show-run", u"No Data") == u"No Data":
150 trow = ET.SubElement(
151 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
153 tcol = ET.SubElement(
154 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
156 tcol.text = u"No Data"
158 trow = ET.SubElement(
159 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
161 thead = ET.SubElement(
162 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
164 font = ET.SubElement(
165 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
168 return str(ET.tostring(tbl, encoding=u"unicode"))
175 u"Cycles per Packet",
176 u"Average Vector Size"
179 for dut_data in tst_data[u"show-run"].values():
180 trow = ET.SubElement(
181 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
183 tcol = ET.SubElement(
184 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
186 if dut_data.get(u"threads", None) is None:
187 tcol.text = u"No Data"
190 bold = ET.SubElement(tcol, u"b")
192 f"Host IP: {dut_data.get(u'host', '')}, "
193 f"Socket: {dut_data.get(u'socket', '')}"
195 trow = ET.SubElement(
196 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
198 thead = ET.SubElement(
199 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
203 for thread_nr, thread in dut_data[u"threads"].items():
204 trow = ET.SubElement(
205 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
207 tcol = ET.SubElement(
208 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
210 bold = ET.SubElement(tcol, u"b")
211 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
212 trow = ET.SubElement(
213 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
215 for idx, col in enumerate(tbl_hdr):
216 tcol = ET.SubElement(
218 attrib=dict(align=u"right" if idx else u"left")
220 font = ET.SubElement(
221 tcol, u"font", attrib=dict(size=u"2")
223 bold = ET.SubElement(font, u"b")
225 for row_nr, row in enumerate(thread):
226 trow = ET.SubElement(
228 attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
230 for idx, col in enumerate(row):
231 tcol = ET.SubElement(
233 attrib=dict(align=u"right" if idx else u"left")
235 font = ET.SubElement(
236 tcol, u"font", attrib=dict(size=u"2")
238 if isinstance(col, float):
239 font.text = f"{col:.2f}"
242 trow = ET.SubElement(
243 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
245 thead = ET.SubElement(
246 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
250 trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
251 thead = ET.SubElement(
252 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
254 font = ET.SubElement(
255 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
259 return str(ET.tostring(tbl, encoding=u"unicode"))
261 for suite in suites.values:
263 for test_data in data.values:
264 if test_data[u"parent"] not in suite[u"name"]:
266 html_table += _generate_html_table(test_data)
270 file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
271 with open(f"{file_name}", u'w') as html_file:
272 logging.info(f" Writing file: {file_name}")
273 html_file.write(u".. raw:: html\n\n\t")
274 html_file.write(html_table)
275 html_file.write(u"\n\t<p><br><br></p>\n")
277 logging.warning(u"The output file is not defined.")
279 logging.info(u" Done.")
282 def table_merged_details(table, input_data):
283 """Generate the table(s) with algorithm: table_merged_details
284 specified in the specification file.
286 :param table: Table to generate.
287 :param input_data: Data to process.
288 :type table: pandas.Series
289 :type input_data: InputData
292 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
296 f" Creating the data set for the {table.get(u'type', u'')} "
297 f"{table.get(u'title', u'')}."
299 data = input_data.filter_data(table, continue_on_error=True)
300 data = input_data.merge_data(data)
302 sort_tests = table.get(u"sort", None)
306 ascending=(sort_tests == u"ascending")
308 data.sort_index(**args)
310 suites = input_data.filter_data(
311 table, continue_on_error=True, data_set=u"suites")
312 suites = input_data.merge_data(suites)
314 # Prepare the header of the tables
316 for column in table[u"columns"]:
318 u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
321 for suite in suites.values:
323 suite_name = suite[u"name"]
325 for test in data.keys():
326 if data[test][u"parent"] not in suite_name:
329 for column in table[u"columns"]:
331 col_data = str(data[test][column[
332 u"data"].split(u" ")[1]]).replace(u'"', u'""')
333 # Do not include tests with "Test Failed" in test message
334 if u"Test Failed" in col_data:
336 col_data = col_data.replace(
337 u"No Data", u"Not Captured "
339 if column[u"data"].split(u" ")[1] in (u"name", ):
340 if len(col_data) > 30:
341 col_data_lst = col_data.split(u"-")
342 half = int(len(col_data_lst) / 2)
343 col_data = f"{u'-'.join(col_data_lst[:half])}" \
345 f"{u'-'.join(col_data_lst[half:])}"
346 col_data = f" |prein| {col_data} |preout| "
347 elif column[u"data"].split(u" ")[1] in (u"msg", ):
348 # Temporary solution: remove NDR results from message:
349 if bool(table.get(u'remove-ndr', False)):
351 col_data = col_data.split(u" |br| ", 1)[1]
354 col_data = f" |prein| {col_data} |preout| "
355 elif column[u"data"].split(u" ")[1] in \
356 (u"conf-history", u"show-run"):
357 col_data = col_data.replace(u" |br| ", u"", 1)
358 col_data = f" |prein| {col_data[:-5]} |preout| "
359 row_lst.append(f'"{col_data}"')
361 row_lst.append(u'"Not captured"')
362 if len(row_lst) == len(table[u"columns"]):
363 table_lst.append(row_lst)
365 # Write the data to file
367 separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
368 file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
369 logging.info(f" Writing file: {file_name}")
370 with open(file_name, u"wt") as file_handler:
371 file_handler.write(u",".join(header) + u"\n")
372 for item in table_lst:
373 file_handler.write(u",".join(item) + u"\n")
375 logging.info(u" Done.")
378 def _tpc_modify_test_name(test_name, ignore_nic=False):
379 """Modify a test name by replacing its parts.
381 :param test_name: Test name to be modified.
382 :param ignore_nic: If True, NIC is removed from TC name.
384 :type ignore_nic: bool
385 :returns: Modified test name.
388 test_name_mod = test_name.\
389 replace(u"-ndrpdrdisc", u""). \
390 replace(u"-ndrpdr", u"").\
391 replace(u"-pdrdisc", u""). \
392 replace(u"-ndrdisc", u"").\
393 replace(u"-pdr", u""). \
394 replace(u"-ndr", u""). \
395 replace(u"1t1c", u"1c").\
396 replace(u"2t1c", u"1c"). \
397 replace(u"2t2c", u"2c").\
398 replace(u"4t2c", u"2c"). \
399 replace(u"4t4c", u"4c").\
400 replace(u"8t4c", u"4c")
403 return re.sub(REGEX_NIC, u"", test_name_mod)
407 def _tpc_modify_displayed_test_name(test_name):
408 """Modify a test name which is displayed in a table by replacing its parts.
410 :param test_name: Test name to be modified.
412 :returns: Modified test name.
416 replace(u"1t1c", u"1c").\
417 replace(u"2t1c", u"1c"). \
418 replace(u"2t2c", u"2c").\
419 replace(u"4t2c", u"2c"). \
420 replace(u"4t4c", u"4c").\
421 replace(u"8t4c", u"4c")
424 def _tpc_insert_data(target, src, include_tests):
425 """Insert src data to the target structure.
427 :param target: Target structure where the data is placed.
428 :param src: Source data to be placed into the target stucture.
429 :param include_tests: Which results will be included (MRR, NDR, PDR).
432 :type include_tests: str
435 if include_tests == u"MRR":
436 target[u"mean"] = src[u"result"][u"receive-rate"]
437 target[u"stdev"] = src[u"result"][u"receive-stdev"]
438 elif include_tests == u"PDR":
439 target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
440 elif include_tests == u"NDR":
441 target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
442 except (KeyError, TypeError):
446 def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
447 footnote=u"", sort_data=True, title=u"",
449 """Generate html table from input data with simple sorting possibility.
451 :param header: Table header.
452 :param data: Input data to be included in the table. It is a list of lists.
453 Inner lists are rows in the table. All inner lists must be of the same
454 length. The length of these lists must be the same as the length of the
456 :param out_file_name: The name (relative or full path) where the
457 generated html table is written.
458 :param legend: The legend to display below the table.
459 :param footnote: The footnote to display below the table (and legend).
460 :param sort_data: If True the data sorting is enabled.
461 :param title: The table (and file) title.
462 :param generate_rst: If True, wrapping rst file is generated.
464 :type data: list of lists
465 :type out_file_name: str
468 :type sort_data: bool
470 :type generate_rst: bool
474 idx = header.index(u"Test Case")
480 [u"left", u"left", u"right"],
481 [u"left", u"left", u"left", u"right"]
485 [u"left", u"left", u"right"],
486 [u"left", u"left", u"left", u"right"]
488 u"width": ([15, 9], [4, 24, 10], [4, 4, 32, 10])
491 df_data = pd.DataFrame(data, columns=header)
494 df_sorted = [df_data.sort_values(
495 by=[key, header[idx]], ascending=[True, True]
496 if key != header[idx] else [False, True]) for key in header]
497 df_sorted_rev = [df_data.sort_values(
498 by=[key, header[idx]], ascending=[False, True]
499 if key != header[idx] else [True, True]) for key in header]
500 df_sorted.extend(df_sorted_rev)
504 fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
505 for idx in range(len(df_data))]]
507 values=[f"<b>{item.replace(u',', u',<br>')}</b>" for item in header],
508 fill_color=u"#7eade7",
509 align=params[u"align-hdr"][idx],
511 family=u"Courier New",
519 for table in df_sorted:
520 columns = [table.get(col) for col in header]
523 columnwidth=params[u"width"][idx],
527 fill_color=fill_color,
528 align=params[u"align-itm"][idx],
530 family=u"Courier New",
538 menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
539 menu_items.extend([f"<b>{itm}</b> (descending)" for itm in header])
540 for idx, hdr in enumerate(menu_items):
541 visible = [False, ] * len(menu_items)
545 label=hdr.replace(u" [Mpps]", u""),
547 args=[{u"visible": visible}],
553 go.layout.Updatemenu(
560 active=len(menu_items) - 1,
561 buttons=list(buttons)
568 columnwidth=params[u"width"][idx],
571 values=[df_sorted.get(col) for col in header],
572 fill_color=fill_color,
573 align=params[u"align-itm"][idx],
575 family=u"Courier New",
586 filename=f"{out_file_name}_in.html"
592 file_name = out_file_name.split(u"/")[-1]
593 if u"vpp" in out_file_name:
594 path = u"_tmp/src/vpp_performance_tests/comparisons/"
596 path = u"_tmp/src/dpdk_performance_tests/comparisons/"
597 with open(f"{path}{file_name}.rst", u"wt") as rst_file:
600 u".. |br| raw:: html\n\n <br />\n\n\n"
601 u".. |prein| raw:: html\n\n <pre>\n\n\n"
602 u".. |preout| raw:: html\n\n </pre>\n\n"
605 rst_file.write(f"{title}\n")
606 rst_file.write(f"{u'`' * len(title)}\n\n")
609 f' <iframe frameborder="0" scrolling="no" '
610 f'width="1600" height="1200" '
611 f'src="../..{out_file_name.replace(u"_build", u"")}_in.html">'
615 rst_file.write(legend[1:].replace(u"\n", u" |br| "))
617 rst_file.write(footnote.replace(u"\n", u" |br| ")[1:])
620 def table_soak_vs_ndr(table, input_data):
621 """Generate the table(s) with algorithm: table_soak_vs_ndr
622 specified in the specification file.
624 :param table: Table to generate.
625 :param input_data: Data to process.
626 :type table: pandas.Series
627 :type input_data: InputData
630 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
634 f" Creating the data set for the {table.get(u'type', u'')} "
635 f"{table.get(u'title', u'')}."
637 data = input_data.filter_data(table, continue_on_error=True)
639 # Prepare the header of the table
643 f"Avg({table[u'reference'][u'title']})",
644 f"Stdev({table[u'reference'][u'title']})",
645 f"Avg({table[u'compare'][u'title']})",
646 f"Stdev{table[u'compare'][u'title']})",
650 header_str = u";".join(header) + u"\n"
653 f"Avg({table[u'reference'][u'title']}): "
654 f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
655 f"from a series of runs of the listed tests.\n"
656 f"Stdev({table[u'reference'][u'title']}): "
657 f"Standard deviation value of {table[u'reference'][u'title']} "
658 f"[Mpps] computed from a series of runs of the listed tests.\n"
659 f"Avg({table[u'compare'][u'title']}): "
660 f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
661 f"a series of runs of the listed tests.\n"
662 f"Stdev({table[u'compare'][u'title']}): "
663 f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
664 f"computed from a series of runs of the listed tests.\n"
665 f"Diff({table[u'reference'][u'title']},"
666 f"{table[u'compare'][u'title']}): "
667 f"Percentage change calculated for mean values.\n"
669 u"Standard deviation of percentage change calculated for mean "
673 except (AttributeError, KeyError) as err:
674 logging.error(f"The model is invalid, missing parameter: {repr(err)}")
677 # Create a list of available SOAK test results:
679 for job, builds in table[u"compare"][u"data"].items():
681 for tst_name, tst_data in data[job][str(build)].items():
682 if tst_data[u"type"] == u"SOAK":
683 tst_name_mod = tst_name.replace(u"-soak", u"")
684 if tbl_dict.get(tst_name_mod, None) is None:
685 groups = re.search(REGEX_NIC, tst_data[u"parent"])
686 nic = groups.group(0) if groups else u""
689 f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
691 tbl_dict[tst_name_mod] = {
697 tbl_dict[tst_name_mod][u"cmp-data"].append(
698 tst_data[u"throughput"][u"LOWER"])
699 except (KeyError, TypeError):
701 tests_lst = tbl_dict.keys()
703 # Add corresponding NDR test results:
704 for job, builds in table[u"reference"][u"data"].items():
706 for tst_name, tst_data in data[job][str(build)].items():
707 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
708 replace(u"-mrr", u"")
709 if tst_name_mod not in tests_lst:
712 if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
714 if table[u"include-tests"] == u"MRR":
715 result = (tst_data[u"result"][u"receive-rate"],
716 tst_data[u"result"][u"receive-stdev"])
717 elif table[u"include-tests"] == u"PDR":
719 tst_data[u"throughput"][u"PDR"][u"LOWER"]
720 elif table[u"include-tests"] == u"NDR":
722 tst_data[u"throughput"][u"NDR"][u"LOWER"]
725 if result is not None:
726 tbl_dict[tst_name_mod][u"ref-data"].append(
728 except (KeyError, TypeError):
732 for tst_name in tbl_dict:
733 item = [tbl_dict[tst_name][u"name"], ]
734 data_r = tbl_dict[tst_name][u"ref-data"]
736 if table[u"include-tests"] == u"MRR":
737 data_r_mean = data_r[0][0]
738 data_r_stdev = data_r[0][1]
740 data_r_mean = mean(data_r)
741 data_r_stdev = stdev(data_r)
742 item.append(round(data_r_mean / 1e6, 1))
743 item.append(round(data_r_stdev / 1e6, 1))
747 item.extend([None, None])
748 data_c = tbl_dict[tst_name][u"cmp-data"]
750 if table[u"include-tests"] == u"MRR":
751 data_c_mean = data_c[0][0]
752 data_c_stdev = data_c[0][1]
754 data_c_mean = mean(data_c)
755 data_c_stdev = stdev(data_c)
756 item.append(round(data_c_mean / 1e6, 1))
757 item.append(round(data_c_stdev / 1e6, 1))
761 item.extend([None, None])
762 if data_r_mean is not None and data_c_mean is not None:
763 delta, d_stdev = relative_change_stdev(
764 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
766 item.append(round(delta))
770 item.append(round(d_stdev))
775 # Sort the table according to the relative change
776 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
778 # Generate csv tables:
779 csv_file = f"{table[u'output-file']}.csv"
780 with open(csv_file, u"wt") as file_handler:
781 file_handler.write(header_str)
783 file_handler.write(u";".join([str(item) for item in test]) + u"\n")
785 convert_csv_to_pretty_txt(
786 csv_file, f"{table[u'output-file']}.txt", delimiter=u";"
788 with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
789 txt_file.write(legend)
791 # Generate html table:
792 _tpc_generate_html_table(
795 table[u'output-file'],
797 title=table.get(u"title", u"")
801 def table_perf_trending_dash(table, input_data):
802 """Generate the table(s) with algorithm:
803 table_perf_trending_dash
804 specified in the specification file.
806 :param table: Table to generate.
807 :param input_data: Data to process.
808 :type table: pandas.Series
809 :type input_data: InputData
812 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
816 f" Creating the data set for the {table.get(u'type', u'')} "
817 f"{table.get(u'title', u'')}."
819 data = input_data.filter_data(table, continue_on_error=True)
821 # Prepare the header of the tables
825 u"Short-Term Change [%]",
826 u"Long-Term Change [%]",
830 header_str = u",".join(header) + u"\n"
832 incl_tests = table.get(u"include-tests", u"MRR")
834 # Prepare data to the table:
836 for job, builds in table[u"data"].items():
838 for tst_name, tst_data in data[job][str(build)].items():
839 if tst_name.lower() in table.get(u"ignore-list", list()):
841 if tbl_dict.get(tst_name, None) is None:
842 groups = re.search(REGEX_NIC, tst_data[u"parent"])
845 nic = groups.group(0)
846 tbl_dict[tst_name] = {
847 u"name": f"{nic}-{tst_data[u'name']}",
848 u"data": OrderedDict()
851 if incl_tests == u"MRR":
852 tbl_dict[tst_name][u"data"][str(build)] = \
853 tst_data[u"result"][u"receive-rate"]
854 elif incl_tests == u"NDR":
855 tbl_dict[tst_name][u"data"][str(build)] = \
856 tst_data[u"throughput"][u"NDR"][u"LOWER"]
857 elif incl_tests == u"PDR":
858 tbl_dict[tst_name][u"data"][str(build)] = \
859 tst_data[u"throughput"][u"PDR"][u"LOWER"]
860 except (TypeError, KeyError):
861 pass # No data in output.xml for this test
864 for tst_name in tbl_dict:
865 data_t = tbl_dict[tst_name][u"data"]
869 classification_lst, avgs, _ = classify_anomalies(data_t)
871 win_size = min(len(data_t), table[u"window"])
872 long_win_size = min(len(data_t), table[u"long-trend-window"])
876 [x for x in avgs[-long_win_size:-win_size]
881 avg_week_ago = avgs[max(-win_size, -len(avgs))]
883 if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
884 rel_change_last = nan
886 rel_change_last = round(
887 ((last_avg - avg_week_ago) / avg_week_ago) * 1e2, 2)
889 if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
890 rel_change_long = nan
892 rel_change_long = round(
893 ((last_avg - max_long_avg) / max_long_avg) * 1e2, 2)
895 if classification_lst:
896 if isnan(rel_change_last) and isnan(rel_change_long):
898 if isnan(last_avg) or isnan(rel_change_last) or \
899 isnan(rel_change_long):
902 [tbl_dict[tst_name][u"name"],
903 round(last_avg / 1e6, 2),
906 classification_lst[-win_size+1:].count(u"regression"),
907 classification_lst[-win_size+1:].count(u"progression")])
909 tbl_lst.sort(key=lambda rel: rel[0])
912 for nrr in range(table[u"window"], -1, -1):
913 tbl_reg = [item for item in tbl_lst if item[4] == nrr]
914 for nrp in range(table[u"window"], -1, -1):
915 tbl_out = [item for item in tbl_reg if item[5] == nrp]
916 tbl_out.sort(key=lambda rel: rel[2])
917 tbl_sorted.extend(tbl_out)
919 file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
921 logging.info(f" Writing file: {file_name}")
922 with open(file_name, u"wt") as file_handler:
923 file_handler.write(header_str)
924 for test in tbl_sorted:
925 file_handler.write(u",".join([str(item) for item in test]) + u'\n')
927 logging.info(f" Writing file: {table[u'output-file']}.txt")
928 convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
931 def _generate_url(testbed, test_name):
932 """Generate URL to a trending plot from the name of the test case.
934 :param testbed: The testbed used for testing.
935 :param test_name: The name of the test case.
938 :returns: The URL to the plot with the trending data for the given test
943 if u"x520" in test_name:
945 elif u"x710" in test_name:
947 elif u"xl710" in test_name:
949 elif u"xxv710" in test_name:
951 elif u"vic1227" in test_name:
953 elif u"vic1385" in test_name:
955 elif u"x553" in test_name:
957 elif u"cx556" in test_name or u"cx556a" in test_name:
962 if u"64b" in test_name:
964 elif u"78b" in test_name:
966 elif u"imix" in test_name:
968 elif u"9000b" in test_name:
969 frame_size = u"9000b"
970 elif u"1518b" in test_name:
971 frame_size = u"1518b"
972 elif u"114b" in test_name:
977 if u"1t1c" in test_name or \
978 (u"-1c-" in test_name and
979 testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
981 elif u"2t2c" in test_name or \
982 (u"-2c-" in test_name and
983 testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
985 elif u"4t4c" in test_name or \
986 (u"-4c-" in test_name and
987 testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
989 elif u"2t1c" in test_name or \
990 (u"-1c-" in test_name and
991 testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
993 elif u"4t2c" in test_name or \
994 (u"-2c-" in test_name and
995 testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
997 elif u"8t4c" in test_name or \
998 (u"-4c-" in test_name and
999 testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
1004 if u"testpmd" in test_name:
1006 elif u"l3fwd" in test_name:
1008 elif u"avf" in test_name:
1010 elif u"rdma" in test_name:
1012 elif u"dnv" in testbed or u"tsh" in testbed:
1017 if u"acl" in test_name or \
1018 u"macip" in test_name or \
1019 u"nat" in test_name or \
1020 u"policer" in test_name or \
1021 u"cop" in test_name:
1023 elif u"scale" in test_name:
1025 elif u"base" in test_name:
1030 if u"114b" in test_name and u"vhost" in test_name:
1032 elif u"testpmd" in test_name or u"l3fwd" in test_name:
1034 elif u"memif" in test_name:
1035 domain = u"container_memif"
1036 elif u"srv6" in test_name:
1038 elif u"vhost" in test_name:
1040 if u"vppl2xc" in test_name:
1043 driver += u"-testpmd"
1044 if u"lbvpplacp" in test_name:
1045 bsf += u"-link-bonding"
1046 elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1047 domain = u"nf_service_density_vnfc"
1048 elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1049 domain = u"nf_service_density_cnfc"
1050 elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1051 domain = u"nf_service_density_cnfp"
1052 elif u"ipsec" in test_name:
1054 if u"sw" in test_name:
1056 elif u"hw" in test_name:
1058 elif u"ethip4vxlan" in test_name:
1059 domain = u"ip4_tunnels"
1060 elif u"ip4base" in test_name or u"ip4scale" in test_name:
1062 elif u"ip6base" in test_name or u"ip6scale" in test_name:
1064 elif u"l2xcbase" in test_name or \
1065 u"l2xcscale" in test_name or \
1066 u"l2bdbasemaclrn" in test_name or \
1067 u"l2bdscale" in test_name or \
1068 u"l2patch" in test_name:
1073 file_name = u"-".join((domain, testbed, nic)) + u".html#"
1074 anchor_name = u"-".join((frame_size, cores, bsf, driver))
1076 return file_name + anchor_name
1079 def table_perf_trending_dash_html(table, input_data):
1080 """Generate the table(s) with algorithm:
1081 table_perf_trending_dash_html specified in the specification
1084 :param table: Table to generate.
1085 :param input_data: Data to process.
1087 :type input_data: InputData
1092 if not table.get(u"testbed", None):
1094 f"The testbed is not defined for the table "
1095 f"{table.get(u'title', u'')}."
1099 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1102 with open(table[u"input-file"], u'rt') as csv_file:
1103 csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1105 logging.warning(u"The input file is not defined.")
1107 except csv.Error as err:
1109 f"Not possible to process the file {table[u'input-file']}.\n"
1115 dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1118 trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1119 for idx, item in enumerate(csv_lst[0]):
1120 alignment = u"left" if idx == 0 else u"center"
1121 thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1139 for r_idx, row in enumerate(csv_lst[1:]):
1141 color = u"regression"
1143 color = u"progression"
1146 trow = ET.SubElement(
1147 dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
1151 for c_idx, item in enumerate(row):
1152 tdata = ET.SubElement(
1155 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1158 if c_idx == 0 and table.get(u"add-links", True):
1159 ref = ET.SubElement(
1163 href=f"../trending/"
1164 f"{_generate_url(table.get(u'testbed', ''), item)}"
1171 with open(table[u"output-file"], u'w') as html_file:
1172 logging.info(f" Writing file: {table[u'output-file']}")
1173 html_file.write(u".. raw:: html\n\n\t")
1174 html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
1175 html_file.write(u"\n\t<p><br><br></p>\n")
1177 logging.warning(u"The output file is not defined.")
1181 def table_last_failed_tests(table, input_data):
1182 """Generate the table(s) with algorithm: table_last_failed_tests
1183 specified in the specification file.
1185 :param table: Table to generate.
1186 :param input_data: Data to process.
1187 :type table: pandas.Series
1188 :type input_data: InputData
1191 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1193 # Transform the data
1195 f" Creating the data set for the {table.get(u'type', u'')} "
1196 f"{table.get(u'title', u'')}."
1199 data = input_data.filter_data(table, continue_on_error=True)
1201 if data is None or data.empty:
1203 f" No data for the {table.get(u'type', u'')} "
1204 f"{table.get(u'title', u'')}."
1209 for job, builds in table[u"data"].items():
1210 for build in builds:
1213 version = input_data.metadata(job, build).get(u"version", u"")
1215 logging.error(f"Data for {job}: {build} is not present.")
1217 tbl_list.append(build)
1218 tbl_list.append(version)
1219 failed_tests = list()
1222 for tst_data in data[job][build].values:
1223 if tst_data[u"status"] != u"FAIL":
1227 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1230 nic = groups.group(0)
1231 failed_tests.append(f"{nic}-{tst_data[u'name']}")
1232 tbl_list.append(str(passed))
1233 tbl_list.append(str(failed))
1234 tbl_list.extend(failed_tests)
1236 file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1237 logging.info(f" Writing file: {file_name}")
1238 with open(file_name, u"wt") as file_handler:
1239 for test in tbl_list:
1240 file_handler.write(test + u'\n')
1243 def table_failed_tests(table, input_data):
1244 """Generate the table(s) with algorithm: table_failed_tests
1245 specified in the specification file.
1247 :param table: Table to generate.
1248 :param input_data: Data to process.
1249 :type table: pandas.Series
1250 :type input_data: InputData
1253 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1255 # Transform the data
1257 f" Creating the data set for the {table.get(u'type', u'')} "
1258 f"{table.get(u'title', u'')}."
1260 data = input_data.filter_data(table, continue_on_error=True)
1262 # Prepare the header of the tables
1266 u"Last Failure [Time]",
1267 u"Last Failure [VPP-Build-Id]",
1268 u"Last Failure [CSIT-Job-Build-Id]"
1271 # Generate the data for the table according to the model in the table
1275 timeperiod = timedelta(int(table.get(u"window", 7)))
1278 for job, builds in table[u"data"].items():
1279 for build in builds:
1281 for tst_name, tst_data in data[job][build].items():
1282 if tst_name.lower() in table.get(u"ignore-list", list()):
1284 if tbl_dict.get(tst_name, None) is None:
1285 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1288 nic = groups.group(0)
1289 tbl_dict[tst_name] = {
1290 u"name": f"{nic}-{tst_data[u'name']}",
1291 u"data": OrderedDict()
1294 generated = input_data.metadata(job, build).\
1295 get(u"generated", u"")
1298 then = dt.strptime(generated, u"%Y%m%d %H:%M")
1299 if (now - then) <= timeperiod:
1300 tbl_dict[tst_name][u"data"][build] = (
1301 tst_data[u"status"],
1303 input_data.metadata(job, build).get(u"version",
1307 except (TypeError, KeyError) as err:
1308 logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
1312 for tst_data in tbl_dict.values():
1314 fails_last_date = u""
1315 fails_last_vpp = u""
1316 fails_last_csit = u""
1317 for val in tst_data[u"data"].values():
1318 if val[0] == u"FAIL":
1320 fails_last_date = val[1]
1321 fails_last_vpp = val[2]
1322 fails_last_csit = val[3]
1324 max_fails = fails_nr if fails_nr > max_fails else max_fails
1331 f"mrr-daily-build-{fails_last_csit}"
1335 tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1337 for nrf in range(max_fails, -1, -1):
1338 tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1339 tbl_sorted.extend(tbl_fails)
1341 file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1342 logging.info(f" Writing file: {file_name}")
1343 with open(file_name, u"wt") as file_handler:
1344 file_handler.write(u",".join(header) + u"\n")
1345 for test in tbl_sorted:
1346 file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1348 logging.info(f" Writing file: {table[u'output-file']}.txt")
1349 convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1352 def table_failed_tests_html(table, input_data):
1353 """Generate the table(s) with algorithm: table_failed_tests_html
1354 specified in the specification file.
1356 :param table: Table to generate.
1357 :param input_data: Data to process.
1358 :type table: pandas.Series
1359 :type input_data: InputData
1364 if not table.get(u"testbed", None):
1366 f"The testbed is not defined for the table "
1367 f"{table.get(u'title', u'')}."
1371 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1374 with open(table[u"input-file"], u'rt') as csv_file:
1375 csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1377 logging.warning(u"The input file is not defined.")
1379 except csv.Error as err:
1381 f"Not possible to process the file {table[u'input-file']}.\n"
1387 failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1390 trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1391 for idx, item in enumerate(csv_lst[0]):
1392 alignment = u"left" if idx == 0 else u"center"
1393 thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1397 colors = (u"#e9f1fb", u"#d4e4f7")
1398 for r_idx, row in enumerate(csv_lst[1:]):
1399 background = colors[r_idx % 2]
1400 trow = ET.SubElement(
1401 failed_tests, u"tr", attrib=dict(bgcolor=background)
1405 for c_idx, item in enumerate(row):
1406 tdata = ET.SubElement(
1409 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1413 ref = ET.SubElement(
1417 href=f"../trending/"
1418 f"{_generate_url(table.get(u'testbed', ''), item)}"
1425 with open(table[u"output-file"], u'w') as html_file:
1426 logging.info(f" Writing file: {table[u'output-file']}")
1427 html_file.write(u".. raw:: html\n\n\t")
1428 html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
1429 html_file.write(u"\n\t<p><br><br></p>\n")
1431 logging.warning(u"The output file is not defined.")
1435 def table_comparison(table, input_data):
1436 """Generate the table(s) with algorithm: table_comparison
1437 specified in the specification file.
1439 :param table: Table to generate.
1440 :param input_data: Data to process.
1441 :type table: pandas.Series
1442 :type input_data: InputData
1444 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1446 # Transform the data
1448 f" Creating the data set for the {table.get(u'type', u'')} "
1449 f"{table.get(u'title', u'')}."
1452 columns = table.get(u"columns", None)
1455 f"No columns specified for {table.get(u'title', u'')}. Skipping."
1460 for idx, col in enumerate(columns):
1461 if col.get(u"data-set", None) is None:
1462 logging.warning(f"No data for column {col.get(u'title', u'')}")
1464 tag = col.get(u"tag", None)
1465 data = input_data.filter_data(
1467 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1468 data=col[u"data-set"],
1469 continue_on_error=True
1472 u"title": col.get(u"title", f"Column{idx}"),
1475 for builds in data.values:
1476 for build in builds:
1477 for tst_name, tst_data in build.items():
1478 if tag and tag not in tst_data[u"tags"]:
1481 _tpc_modify_test_name(tst_name, ignore_nic=True).\
1482 replace(u"2n1l-", u"")
1483 if col_data[u"data"].get(tst_name_mod, None) is None:
1484 name = tst_data[u'name'].rsplit(u'-', 1)[0]
1485 if u"across testbeds" in table[u"title"].lower() or \
1486 u"across topologies" in table[u"title"].lower():
1487 name = _tpc_modify_displayed_test_name(name)
1488 col_data[u"data"][tst_name_mod] = {
1496 target=col_data[u"data"][tst_name_mod],
1498 include_tests=table[u"include-tests"]
1501 replacement = col.get(u"data-replacement", None)
1503 rpl_data = input_data.filter_data(
1505 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1507 continue_on_error=True
1509 for builds in rpl_data.values:
1510 for build in builds:
1511 for tst_name, tst_data in build.items():
1512 if tag and tag not in tst_data[u"tags"]:
1515 _tpc_modify_test_name(tst_name, ignore_nic=True).\
1516 replace(u"2n1l-", u"")
1517 if col_data[u"data"].get(tst_name_mod, None) is None:
1518 name = tst_data[u'name'].rsplit(u'-', 1)[0]
1519 if u"across testbeds" in table[u"title"].lower() \
1520 or u"across topologies" in \
1521 table[u"title"].lower():
1522 name = _tpc_modify_displayed_test_name(name)
1523 col_data[u"data"][tst_name_mod] = {
1530 if col_data[u"data"][tst_name_mod][u"replace"]:
1531 col_data[u"data"][tst_name_mod][u"replace"] = False
1532 col_data[u"data"][tst_name_mod][u"data"] = list()
1534 target=col_data[u"data"][tst_name_mod],
1536 include_tests=table[u"include-tests"]
1539 if table[u"include-tests"] in (u"NDR", u"PDR"):
1540 for tst_name, tst_data in col_data[u"data"].items():
1541 if tst_data[u"data"]:
1542 tst_data[u"mean"] = mean(tst_data[u"data"])
1543 tst_data[u"stdev"] = stdev(tst_data[u"data"])
1545 cols.append(col_data)
1549 for tst_name, tst_data in col[u"data"].items():
1550 if tbl_dict.get(tst_name, None) is None:
1551 tbl_dict[tst_name] = {
1552 "name": tst_data[u"name"]
1554 tbl_dict[tst_name][col[u"title"]] = {
1555 u"mean": tst_data[u"mean"],
1556 u"stdev": tst_data[u"stdev"]
1560 logging.warning(f"No data for table {table.get(u'title', u'')}!")
1564 for tst_data in tbl_dict.values():
1565 row = [tst_data[u"name"], ]
1567 row.append(tst_data.get(col[u"title"], None))
1570 comparisons = table.get(u"comparisons", None)
1571 if comparisons and isinstance(comparisons, list):
1572 for idx, comp in enumerate(comparisons):
1574 col_ref = int(comp[u"reference"])
1575 col_cmp = int(comp[u"compare"])
1577 logging.warning(u"Comparison: No references defined! Skipping.")
1578 comparisons.pop(idx)
1580 if not (0 < col_ref <= len(cols) and
1581 0 < col_cmp <= len(cols)) or \
1583 logging.warning(f"Wrong values of reference={col_ref} "
1584 f"and/or compare={col_cmp}. Skipping.")
1585 comparisons.pop(idx)
1588 tbl_cmp_lst = list()
1591 new_row = deepcopy(row)
1593 for comp in comparisons:
1594 ref_itm = row[int(comp[u"reference"])]
1595 if ref_itm is None and \
1596 comp.get(u"reference-alt", None) is not None:
1597 ref_itm = row[int(comp[u"reference-alt"])]
1598 cmp_itm = row[int(comp[u"compare"])]
1599 if ref_itm is not None and cmp_itm is not None and \
1600 ref_itm[u"mean"] is not None and \
1601 cmp_itm[u"mean"] is not None and \
1602 ref_itm[u"stdev"] is not None and \
1603 cmp_itm[u"stdev"] is not None:
1604 delta, d_stdev = relative_change_stdev(
1605 ref_itm[u"mean"], cmp_itm[u"mean"],
1606 ref_itm[u"stdev"], cmp_itm[u"stdev"]
1610 u"mean": delta * 1e6,
1611 u"stdev": d_stdev * 1e6
1616 new_row.append(None)
1618 tbl_cmp_lst.append(new_row)
1620 tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
1621 tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
1624 rca_in = table.get(u"rca", None)
1625 if rca_in and isinstance(rca_in, list):
1626 for idx, itm in enumerate(rca_in):
1628 with open(itm.get(u"data", u""), u"r") as rca_file:
1631 u"title": itm.get(u"title", f"RCA{idx}"),
1632 u"data": load(rca_file, Loader=FullLoader)
1635 except (YAMLError, IOError) as err:
1637 f"The RCA file {itm.get(u'data', u'')} does not exist or "
1640 logging.debug(repr(err))
1642 tbl_for_csv = list()
1643 for line in tbl_cmp_lst:
1645 for idx, itm in enumerate(line[1:]):
1650 row.append(round(float(itm[u'mean']) / 1e6, 3))
1651 row.append(round(float(itm[u'stdev']) / 1e6, 3))
1653 rca_nr = rca[u"data"].get(row[0], u"-")
1654 row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1655 tbl_for_csv.append(row)
1657 header_csv = [u"Test Case", ]
1659 header_csv.append(f"Avg({col[u'title']})")
1660 header_csv.append(f"Stdev({col[u'title']})")
1661 for comp in comparisons:
1663 f"Avg({comp.get(u'title', u'')})"
1666 f"Stdev({comp.get(u'title', u'')})"
1668 header_csv.extend([rca[u"title"] for rca in rcas])
1670 legend_lst = table.get(u"legend", None)
1671 if legend_lst is None:
1674 legend = u"\n" + u"\n".join(legend_lst) + u"\n"
1678 footnote += f"\n{rca[u'title']}:\n"
1679 footnote += rca[u"data"].get(u"footnote", u"")
1681 csv_file = f"{table[u'output-file']}-csv.csv"
1682 with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1684 u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
1686 for test in tbl_for_csv:
1688 u",".join([f'"{item}"' for item in test]) + u"\n"
1691 for item in legend_lst:
1692 file_handler.write(f'"{item}"\n')
1694 for itm in footnote.split(u"\n"):
1695 file_handler.write(f'"{itm}"\n')
1698 max_lens = [0, ] * len(tbl_cmp_lst[0])
1699 for line in tbl_cmp_lst:
1701 for idx, itm in enumerate(line[1:]):
1707 f"{round(float(itm[u'mean']) / 1e6, 1)} "
1708 f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1709 replace(u"nan", u"NaN")
1713 f"{round(float(itm[u'mean']) / 1e6, 1):+} "
1714 f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
1715 replace(u"nan", u"NaN")
1717 if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]:
1718 max_lens[idx] = len(new_itm.rsplit(u" ", 1)[-1])
1724 for line in tbl_tmp:
1726 for idx, itm in enumerate(line[1:]):
1727 if itm in (u"NT", u"NaN"):
1730 itm_lst = itm.rsplit(u"\u00B1", 1)
1732 f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
1733 row.append(u"\u00B1".join(itm_lst))
1735 rca_nr = rca[u"data"].get(row[0], u"-")
1736 row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1738 tbl_final.append(row)
1740 header = [u"Test Case", ]
1741 header.extend([col[u"title"] for col in cols])
1742 header.extend([comp.get(u"title", u"") for comp in comparisons])
1743 header.extend([rca[u"title"] for rca in rcas])
1745 # Generate csv tables:
1746 csv_file = f"{table[u'output-file']}.csv"
1747 with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1748 file_handler.write(u";".join(header) + u"\n")
1749 for test in tbl_final:
1750 file_handler.write(u";".join([str(item) for item in test]) + u"\n")
1752 # Generate txt table:
1753 txt_file_name = f"{table[u'output-file']}.txt"
1754 convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
1756 with open(txt_file_name, u'a', encoding='utf-8') as txt_file:
1757 txt_file.write(legend)
1758 txt_file.write(footnote)
1759 if legend or footnote:
1760 txt_file.write(u"\n:END")
1762 # Generate html table:
1763 _tpc_generate_html_table(
1766 table[u'output-file'],
1770 title=table.get(u"title", u"")
1774 def table_weekly_comparison(table, in_data):
1775 """Generate the table(s) with algorithm: table_weekly_comparison
1776 specified in the specification file.
1778 :param table: Table to generate.
1779 :param in_data: Data to process.
1780 :type table: pandas.Series
1781 :type in_data: InputData
1783 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1785 # Transform the data
1787 f" Creating the data set for the {table.get(u'type', u'')} "
1788 f"{table.get(u'title', u'')}."
1791 incl_tests = table.get(u"include-tests", None)
1792 if incl_tests not in (u"NDR", u"PDR"):
1793 logging.error(f"Wrong tests to include specified ({incl_tests}).")
1796 nr_cols = table.get(u"nr-of-data-columns", None)
1797 if not nr_cols or nr_cols < 2:
1799 f"No columns specified for {table.get(u'title', u'')}. Skipping."
1803 data = in_data.filter_data(
1805 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
1806 continue_on_error=True
1811 [u"Start Timestamp", ],
1817 tb_tbl = table.get(u"testbeds", None)
1818 for job_name, job_data in data.items():
1819 for build_nr, build in job_data.items():
1825 tb_ip = in_data.metadata(job_name, build_nr).get(u"testbed", u"")
1826 if tb_ip and tb_tbl:
1827 testbed = tb_tbl.get(tb_ip, u"")
1830 header[2].insert(1, build_nr)
1831 header[3].insert(1, testbed)
1833 1, in_data.metadata(job_name, build_nr).get(u"generated", u"")
1836 1, in_data.metadata(job_name, build_nr).get(u"version", u"")
1839 for tst_name, tst_data in build.items():
1841 _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
1842 if not tbl_dict.get(tst_name_mod, None):
1843 tbl_dict[tst_name_mod] = dict(
1844 name=tst_data[u'name'].rsplit(u'-', 1)[0],
1847 tbl_dict[tst_name_mod][-idx - 1] = \
1848 tst_data[u"throughput"][incl_tests][u"LOWER"]
1849 except (TypeError, IndexError, KeyError, ValueError):
1854 logging.error(u"Not enough data to build the table! Skipping")
1858 for idx, cmp in enumerate(table.get(u"comparisons", list())):
1859 idx_ref = cmp.get(u"reference", None)
1860 idx_cmp = cmp.get(u"compare", None)
1861 if idx_ref is None or idx_cmp is None:
1864 f"Diff({header[0][idx_ref - idx].split(u'~')[-1]} vs "
1865 f"{header[0][idx_cmp - idx].split(u'~')[-1]})"
1867 header[1].append(u"")
1868 header[2].append(u"")
1869 header[3].append(u"")
1870 for tst_name, tst_data in tbl_dict.items():
1871 if not cmp_dict.get(tst_name, None):
1872 cmp_dict[tst_name] = list()
1873 ref_data = tst_data.get(idx_ref, None)
1874 cmp_data = tst_data.get(idx_cmp, None)
1875 if ref_data is None or cmp_data is None:
1876 cmp_dict[tst_name].append(float('nan'))
1878 cmp_dict[tst_name].append(
1879 relative_change(ref_data, cmp_data)
1883 for tst_name, tst_data in tbl_dict.items():
1884 itm_lst = [tst_data[u"name"], ]
1885 for idx in range(nr_cols):
1886 item = tst_data.get(-idx - 1, None)
1888 itm_lst.insert(1, None)
1890 itm_lst.insert(1, round(item / 1e6, 1))
1893 None if itm is None else round(itm, 1)
1894 for itm in cmp_dict[tst_name]
1897 tbl_lst.append(itm_lst)
1899 tbl_lst.sort(key=lambda rel: rel[0], reverse=False)
1900 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
1902 # Generate csv table:
1903 csv_file = f"{table[u'output-file']}.csv"
1904 with open(csv_file, u"wt", encoding='utf-8') as file_handler:
1906 file_handler.write(u",".join(hdr) + u"\n")
1907 for test in tbl_lst:
1908 file_handler.write(u",".join(
1910 str(item).replace(u"None", u"-").replace(u"nan", u"-").
1911 replace(u"null", u"-") for item in test
1915 txt_file = f"{table[u'output-file']}.txt"
1916 convert_csv_to_pretty_txt(csv_file, txt_file, delimiter=u",")
1918 # Reorganize header in txt table
1920 with open(txt_file, u"rt", encoding='utf-8') as file_handler:
1921 for line in file_handler:
1922 txt_table.append(line)
1924 txt_table.insert(5, txt_table.pop(2))
1925 with open(txt_file, u"wt", encoding='utf-8') as file_handler:
1926 file_handler.writelines(txt_table)
1930 # Generate html table:
1932 u"<br>".join(row) for row in zip(*header)
1934 _tpc_generate_html_table(
1937 table[u'output-file'],
1939 title=table.get(u"title", u""),