1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Algorithms to generate tables.
23 from collections import OrderedDict
24 from xml.etree import ElementTree as ET
25 from datetime import datetime as dt
26 from datetime import timedelta
27 from copy import deepcopy
29 import plotly.graph_objects as go
30 import plotly.offline as ploff
34 from numpy import nan, isnan
35 from yaml import load, FullLoader, YAMLError
37 from pal_utils import mean, stdev, classify_anomalies, \
38 convert_csv_to_pretty_txt, relative_change_stdev, relative_change
41 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
44 def generate_tables(spec, data):
45 """Generate all tables specified in the specification file.
47 :param spec: Specification read from the specification file.
48 :param data: Data to process.
49 :type spec: Specification
54 u"table_merged_details": table_merged_details,
55 u"table_soak_vs_ndr": table_soak_vs_ndr,
56 u"table_perf_trending_dash": table_perf_trending_dash,
57 u"table_perf_trending_dash_html": table_perf_trending_dash_html,
58 u"table_last_failed_tests": table_last_failed_tests,
59 u"table_failed_tests": table_failed_tests,
60 u"table_failed_tests_html": table_failed_tests_html,
61 u"table_oper_data_html": table_oper_data_html,
62 u"table_comparison": table_comparison,
63 u"table_weekly_comparison": table_weekly_comparison,
64 u"table_job_spec_duration": table_job_spec_duration
67 logging.info(u"Generating the tables ...")
68 for table in spec.tables:
70 if table[u"algorithm"] == u"table_weekly_comparison":
71 table[u"testbeds"] = spec.environment.get(u"testbeds", None)
72 generator[table[u"algorithm"]](table, data)
73 except NameError as err:
75 f"Probably algorithm {table[u'algorithm']} is not defined: "
78 logging.info(u"Done.")
81 def table_job_spec_duration(table, input_data):
82 """Generate the table(s) with algorithm: table_job_spec_duration
83 specified in the specification file.
85 :param table: Table to generate.
86 :param input_data: Data to process.
87 :type table: pandas.Series
88 :type input_data: InputData
93 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
95 jb_type = table.get(u"jb-type", None)
98 if jb_type == u"iterative":
99 for line in table.get(u"lines", tuple()):
101 u"name": line.get(u"job-spec", u""),
104 for job, builds in line.get(u"data-set", dict()).items():
105 for build_nr in builds:
107 minutes = input_data.metadata(
109 )[u"elapsedtime"] // 60000
110 except (KeyError, IndexError, ValueError, AttributeError):
112 tbl_itm[u"data"].append(minutes)
113 tbl_itm[u"mean"] = mean(tbl_itm[u"data"])
114 tbl_itm[u"stdev"] = stdev(tbl_itm[u"data"])
115 tbl_lst.append(tbl_itm)
116 elif jb_type == u"coverage":
117 job = table.get(u"data", None)
120 for line in table.get(u"lines", tuple()):
123 u"name": line.get(u"job-spec", u""),
124 u"mean": input_data.metadata(
125 list(job.keys())[0], str(line[u"build"])
126 )[u"elapsedtime"] // 60000,
127 u"stdev": float(u"nan")
129 tbl_itm[u"data"] = [tbl_itm[u"mean"], ]
130 except (KeyError, IndexError, ValueError, AttributeError):
132 tbl_lst.append(tbl_itm)
134 logging.warning(f"Wrong type of job-spec: {jb_type}. Skipping.")
139 f"{int(line[u'mean'] // 60):02d}:{int(line[u'mean'] % 60):02d}"
140 if math.isnan(line[u"stdev"]):
144 f"{int(line[u'stdev'] //60):02d}:{int(line[u'stdev'] % 60):02d}"
153 f"{len(itm[u'data'])}",
154 f"{itm[u'mean']} +- {itm[u'stdev']}"
155 if itm[u"stdev"] != u"" else f"{itm[u'mean']}"
158 txt_table = prettytable.PrettyTable(
159 [u"Job Specification", u"Nr of Runs", u"Duration [HH:MM]"]
162 txt_table.add_row(row)
163 txt_table.align = u"r"
164 txt_table.align[u"Job Specification"] = u"l"
166 file_name = f"{table.get(u'output-file', u'')}.txt"
167 with open(file_name, u"wt", encoding='utf-8') as txt_file:
168 txt_file.write(str(txt_table))
171 def table_oper_data_html(table, input_data):
172 """Generate the table(s) with algorithm: html_table_oper_data
173 specified in the specification file.
175 :param table: Table to generate.
176 :param input_data: Data to process.
177 :type table: pandas.Series
178 :type input_data: InputData
181 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
184 f" Creating the data set for the {table.get(u'type', u'')} "
185 f"{table.get(u'title', u'')}."
187 data = input_data.filter_data(
189 params=[u"name", u"parent", u"telemetry-show-run", u"type"],
190 continue_on_error=True
194 data = input_data.merge_data(data)
196 sort_tests = table.get(u"sort", None)
200 ascending=(sort_tests == u"ascending")
202 data.sort_index(**args)
204 suites = input_data.filter_data(
206 continue_on_error=True,
211 suites = input_data.merge_data(suites)
213 def _generate_html_table(tst_data):
214 """Generate an HTML table with operational data for the given test.
216 :param tst_data: Test data to be used to generate the table.
217 :type tst_data: pandas.Series
218 :returns: HTML table with operational data.
223 u"header": u"#7eade7",
224 u"empty": u"#ffffff",
225 u"body": (u"#e9f1fb", u"#d4e4f7")
228 tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
230 trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
231 thead = ET.SubElement(
232 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
234 thead.text = tst_data[u"name"]
236 trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
237 thead = ET.SubElement(
238 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
242 if tst_data.get(u"telemetry-show-run", None) is None or \
243 isinstance(tst_data[u"telemetry-show-run"], str):
244 trow = ET.SubElement(
245 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
247 tcol = ET.SubElement(
248 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
250 tcol.text = u"No Data"
252 trow = ET.SubElement(
253 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
255 thead = ET.SubElement(
256 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
258 font = ET.SubElement(
259 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
262 return str(ET.tostring(tbl, encoding=u"unicode"))
269 u"Cycles per Packet",
270 u"Average Vector Size"
273 for dut_data in tst_data[u"telemetry-show-run"].values():
274 trow = ET.SubElement(
275 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
277 tcol = ET.SubElement(
278 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
280 if dut_data.get(u"runtime", None) is None:
281 tcol.text = u"No Data"
285 for item in dut_data[u"runtime"].get(u"data", tuple()):
286 tid = int(item[u"labels"][u"thread_id"])
287 if runtime.get(tid, None) is None:
288 runtime[tid] = dict()
289 gnode = item[u"labels"][u"graph_node"]
290 if runtime[tid].get(gnode, None) is None:
291 runtime[tid][gnode] = dict()
293 runtime[tid][gnode][item[u"name"]] = float(item[u"value"])
295 runtime[tid][gnode][item[u"name"]] = item[u"value"]
297 threads = dict({idx: list() for idx in range(len(runtime))})
298 for idx, run_data in runtime.items():
299 for gnode, gdata in run_data.items():
300 if gdata[u"vectors"] > 0:
301 clocks = gdata[u"clocks"] / gdata[u"vectors"]
302 elif gdata[u"calls"] > 0:
303 clocks = gdata[u"clocks"] / gdata[u"calls"]
304 elif gdata[u"suspends"] > 0:
305 clocks = gdata[u"clocks"] / gdata[u"suspends"]
308 if gdata[u"calls"] > 0:
309 vectors_call = gdata[u"vectors"] / gdata[u"calls"]
312 if int(gdata[u"calls"]) + int(gdata[u"vectors"]) + \
313 int(gdata[u"suspends"]):
314 threads[idx].append([
316 int(gdata[u"calls"]),
317 int(gdata[u"vectors"]),
318 int(gdata[u"suspends"]),
323 bold = ET.SubElement(tcol, u"b")
325 f"Host IP: {dut_data.get(u'host', '')}, "
326 f"Socket: {dut_data.get(u'socket', '')}"
328 trow = ET.SubElement(
329 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
331 thead = ET.SubElement(
332 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
336 for thread_nr, thread in threads.items():
337 trow = ET.SubElement(
338 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
340 tcol = ET.SubElement(
341 trow, u"td", attrib=dict(align=u"left", colspan=u"6")
343 bold = ET.SubElement(tcol, u"b")
344 bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
345 trow = ET.SubElement(
346 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
348 for idx, col in enumerate(tbl_hdr):
349 tcol = ET.SubElement(
351 attrib=dict(align=u"right" if idx else u"left")
353 font = ET.SubElement(
354 tcol, u"font", attrib=dict(size=u"2")
356 bold = ET.SubElement(font, u"b")
358 for row_nr, row in enumerate(thread):
359 trow = ET.SubElement(
361 attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
363 for idx, col in enumerate(row):
364 tcol = ET.SubElement(
366 attrib=dict(align=u"right" if idx else u"left")
368 font = ET.SubElement(
369 tcol, u"font", attrib=dict(size=u"2")
371 if isinstance(col, float):
372 font.text = f"{col:.2f}"
375 trow = ET.SubElement(
376 tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
378 thead = ET.SubElement(
379 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
383 trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
384 thead = ET.SubElement(
385 trow, u"th", attrib=dict(align=u"left", colspan=u"6")
387 font = ET.SubElement(
388 thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
392 return str(ET.tostring(tbl, encoding=u"unicode"))
394 for suite in suites.values:
396 for test_data in data.values:
397 if test_data[u"parent"] not in suite[u"name"]:
399 html_table += _generate_html_table(test_data)
403 file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
404 with open(f"{file_name}", u'w') as html_file:
405 logging.info(f" Writing file: {file_name}")
406 html_file.write(u".. raw:: html\n\n\t")
407 html_file.write(html_table)
408 html_file.write(u"\n\t<p><br><br></p>\n")
410 logging.warning(u"The output file is not defined.")
412 logging.info(u" Done.")
415 def table_merged_details(table, input_data):
416 """Generate the table(s) with algorithm: table_merged_details
417 specified in the specification file.
419 :param table: Table to generate.
420 :param input_data: Data to process.
421 :type table: pandas.Series
422 :type input_data: InputData
425 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
429 f" Creating the data set for the {table.get(u'type', u'')} "
430 f"{table.get(u'title', u'')}."
432 data = input_data.filter_data(table, continue_on_error=True)
433 data = input_data.merge_data(data)
435 sort_tests = table.get(u"sort", None)
439 ascending=(sort_tests == u"ascending")
441 data.sort_index(**args)
443 suites = input_data.filter_data(
444 table, continue_on_error=True, data_set=u"suites")
445 suites = input_data.merge_data(suites)
447 # Prepare the header of the tables
449 for column in table[u"columns"]:
451 u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
454 for suite in suites.values:
456 suite_name = suite[u"name"]
458 for test in data.keys():
459 if data[test][u"status"] != u"PASS" or \
460 data[test][u"parent"] not in suite_name:
463 for column in table[u"columns"]:
465 col_data = str(data[test][column[
466 u"data"].split(u" ")[1]]).replace(u'"', u'""')
467 # Do not include tests with "Test Failed" in test message
468 if u"Test Failed" in col_data:
470 col_data = col_data.replace(
471 u"No Data", u"Not Captured "
473 if column[u"data"].split(u" ")[1] in (u"name", ):
474 if len(col_data) > 30:
475 col_data_lst = col_data.split(u"-")
476 half = int(len(col_data_lst) / 2)
477 col_data = f"{u'-'.join(col_data_lst[:half])}" \
479 f"{u'-'.join(col_data_lst[half:])}"
480 col_data = f" |prein| {col_data} |preout| "
481 elif column[u"data"].split(u" ")[1] in (u"msg", ):
482 # Temporary solution: remove NDR results from message:
483 if bool(table.get(u'remove-ndr', False)):
485 col_data = col_data.split(u"\n", 1)[1]
488 col_data = col_data.replace(u'\n', u' |br| ').\
489 replace(u'\r', u'').replace(u'"', u"'")
490 col_data = f" |prein| {col_data} |preout| "
491 elif column[u"data"].split(u" ")[1] in (u"conf-history", ):
492 col_data = col_data.replace(u'\n', u' |br| ')
493 col_data = f" |prein| {col_data[:-5]} |preout| "
494 row_lst.append(f'"{col_data}"')
496 row_lst.append(u'"Not captured"')
497 if len(row_lst) == len(table[u"columns"]):
498 table_lst.append(row_lst)
500 # Write the data to file
502 separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
503 file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
504 logging.info(f" Writing file: {file_name}")
505 with open(file_name, u"wt") as file_handler:
506 file_handler.write(u",".join(header) + u"\n")
507 for item in table_lst:
508 file_handler.write(u",".join(item) + u"\n")
510 logging.info(u" Done.")
513 def _tpc_modify_test_name(test_name, ignore_nic=False):
514 """Modify a test name by replacing its parts.
516 :param test_name: Test name to be modified.
517 :param ignore_nic: If True, NIC is removed from TC name.
519 :type ignore_nic: bool
520 :returns: Modified test name.
523 test_name_mod = test_name.\
524 replace(u"-ndrpdr", u"").\
525 replace(u"1t1c", u"1c").\
526 replace(u"2t1c", u"1c"). \
527 replace(u"2t2c", u"2c").\
528 replace(u"4t2c", u"2c"). \
529 replace(u"4t4c", u"4c").\
530 replace(u"8t4c", u"4c")
533 return re.sub(REGEX_NIC, u"", test_name_mod)
537 def _tpc_modify_displayed_test_name(test_name):
538 """Modify a test name which is displayed in a table by replacing its parts.
540 :param test_name: Test name to be modified.
542 :returns: Modified test name.
546 replace(u"1t1c", u"1c").\
547 replace(u"2t1c", u"1c"). \
548 replace(u"2t2c", u"2c").\
549 replace(u"4t2c", u"2c"). \
550 replace(u"4t4c", u"4c").\
551 replace(u"8t4c", u"4c")
554 def _tpc_insert_data(target, src, include_tests):
555 """Insert src data to the target structure.
557 :param target: Target structure where the data is placed.
558 :param src: Source data to be placed into the target structure.
559 :param include_tests: Which results will be included (MRR, NDR, PDR).
562 :type include_tests: str
565 if include_tests == u"MRR":
566 target[u"mean"] = src[u"result"][u"receive-rate"]
567 target[u"stdev"] = src[u"result"][u"receive-stdev"]
568 elif include_tests == u"PDR":
569 target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
570 elif include_tests == u"NDR":
571 target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
572 elif u"latency" in include_tests:
573 keys = include_tests.split(u"-")
575 lat = src[keys[0]][keys[1]][keys[2]][keys[3]]
576 target[u"data"].append(
577 float(u"nan") if lat == -1 else lat * 1e6
579 except (KeyError, TypeError):
583 def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
584 footnote=u"", sort_data=True, title=u"",
586 """Generate html table from input data with simple sorting possibility.
588 :param header: Table header.
589 :param data: Input data to be included in the table. It is a list of lists.
590 Inner lists are rows in the table. All inner lists must be of the same
591 length. The length of these lists must be the same as the length of the
593 :param out_file_name: The name (relative or full path) where the
594 generated html table is written.
595 :param legend: The legend to display below the table.
596 :param footnote: The footnote to display below the table (and legend).
597 :param sort_data: If True the data sorting is enabled.
598 :param title: The table (and file) title.
599 :param generate_rst: If True, wrapping rst file is generated.
601 :type data: list of lists
602 :type out_file_name: str
605 :type sort_data: bool
607 :type generate_rst: bool
611 idx = header.index(u"Test Case")
617 [u"left", u"left", u"right"],
618 [u"left", u"left", u"left", u"right"]
622 [u"left", u"left", u"right"],
623 [u"left", u"left", u"left", u"right"]
625 u"width": ([15, 9], [4, 24, 10], [4, 4, 32, 10])
628 df_data = pd.DataFrame(data, columns=header)
631 df_sorted = [df_data.sort_values(
632 by=[key, header[idx]], ascending=[True, True]
633 if key != header[idx] else [False, True]) for key in header]
634 df_sorted_rev = [df_data.sort_values(
635 by=[key, header[idx]], ascending=[False, True]
636 if key != header[idx] else [True, True]) for key in header]
637 df_sorted.extend(df_sorted_rev)
641 fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
642 for idx in range(len(df_data))]]
644 values=[f"<b>{item.replace(u',', u',<br>')}</b>" for item in header],
645 fill_color=u"#7eade7",
646 align=params[u"align-hdr"][idx],
648 family=u"Courier New",
656 for table in df_sorted:
657 columns = [table.get(col) for col in header]
660 columnwidth=params[u"width"][idx],
664 fill_color=fill_color,
665 align=params[u"align-itm"][idx],
667 family=u"Courier New",
675 menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
676 menu_items.extend([f"<b>{itm}</b> (descending)" for itm in header])
677 for idx, hdr in enumerate(menu_items):
678 visible = [False, ] * len(menu_items)
682 label=hdr.replace(u" [Mpps]", u""),
684 args=[{u"visible": visible}],
690 go.layout.Updatemenu(
697 active=len(menu_items) - 1,
698 buttons=list(buttons)
705 columnwidth=params[u"width"][idx],
708 values=[df_sorted.get(col) for col in header],
709 fill_color=fill_color,
710 align=params[u"align-itm"][idx],
712 family=u"Courier New",
723 filename=f"{out_file_name}_in.html"
729 file_name = out_file_name.split(u"/")[-1]
730 if u"vpp" in out_file_name:
731 path = u"_tmp/src/vpp_performance_tests/comparisons/"
733 path = u"_tmp/src/dpdk_performance_tests/comparisons/"
734 logging.info(f" Writing the HTML file to {path}{file_name}.rst")
735 with open(f"{path}{file_name}.rst", u"wt") as rst_file:
738 u".. |br| raw:: html\n\n <br />\n\n\n"
739 u".. |prein| raw:: html\n\n <pre>\n\n\n"
740 u".. |preout| raw:: html\n\n </pre>\n\n"
743 rst_file.write(f"{title}\n")
744 rst_file.write(f"{u'`' * len(title)}\n\n")
747 f' <iframe frameborder="0" scrolling="no" '
748 f'width="1600" height="1200" '
749 f'src="../..{out_file_name.replace(u"_build", u"")}_in.html">'
755 itm_lst = legend[1:-2].split(u"\n")
757 f"{itm_lst[0]}\n\n- " + u'\n- '.join(itm_lst[1:]) + u"\n\n"
759 except IndexError as err:
760 logging.error(f"Legend cannot be written to html file\n{err}")
763 itm_lst = footnote[1:].split(u"\n")
765 f"{itm_lst[0]}\n\n- " + u'\n- '.join(itm_lst[1:]) + u"\n\n"
767 except IndexError as err:
768 logging.error(f"Footnote cannot be written to html file\n{err}")
771 def table_soak_vs_ndr(table, input_data):
772 """Generate the table(s) with algorithm: table_soak_vs_ndr
773 specified in the specification file.
775 :param table: Table to generate.
776 :param input_data: Data to process.
777 :type table: pandas.Series
778 :type input_data: InputData
781 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
785 f" Creating the data set for the {table.get(u'type', u'')} "
786 f"{table.get(u'title', u'')}."
788 data = input_data.filter_data(table, continue_on_error=True)
790 # Prepare the header of the table
794 f"Avg({table[u'reference'][u'title']})",
795 f"Stdev({table[u'reference'][u'title']})",
796 f"Avg({table[u'compare'][u'title']})",
797 f"Stdev{table[u'compare'][u'title']})",
801 header_str = u";".join(header) + u"\n"
804 f"Avg({table[u'reference'][u'title']}): "
805 f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
806 f"from a series of runs of the listed tests.\n"
807 f"Stdev({table[u'reference'][u'title']}): "
808 f"Standard deviation value of {table[u'reference'][u'title']} "
809 f"[Mpps] computed from a series of runs of the listed tests.\n"
810 f"Avg({table[u'compare'][u'title']}): "
811 f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
812 f"a series of runs of the listed tests.\n"
813 f"Stdev({table[u'compare'][u'title']}): "
814 f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
815 f"computed from a series of runs of the listed tests.\n"
816 f"Diff({table[u'reference'][u'title']},"
817 f"{table[u'compare'][u'title']}): "
818 f"Percentage change calculated for mean values.\n"
820 u"Standard deviation of percentage change calculated for mean "
823 except (AttributeError, KeyError) as err:
824 logging.error(f"The model is invalid, missing parameter: {repr(err)}")
827 # Create a list of available SOAK test results:
829 for job, builds in table[u"compare"][u"data"].items():
831 for tst_name, tst_data in data[job][str(build)].items():
832 if tst_data[u"type"] == u"SOAK":
833 tst_name_mod = tst_name.replace(u"-soak", u"")
834 if tbl_dict.get(tst_name_mod, None) is None:
835 groups = re.search(REGEX_NIC, tst_data[u"parent"])
836 nic = groups.group(0) if groups else u""
839 f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
841 tbl_dict[tst_name_mod] = {
847 tbl_dict[tst_name_mod][u"cmp-data"].append(
848 tst_data[u"throughput"][u"LOWER"])
849 except (KeyError, TypeError):
851 tests_lst = tbl_dict.keys()
853 # Add corresponding NDR test results:
854 for job, builds in table[u"reference"][u"data"].items():
856 for tst_name, tst_data in data[job][str(build)].items():
857 tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\
858 replace(u"-mrr", u"")
859 if tst_name_mod not in tests_lst:
862 if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
864 if table[u"include-tests"] == u"MRR":
865 result = (tst_data[u"result"][u"receive-rate"],
866 tst_data[u"result"][u"receive-stdev"])
867 elif table[u"include-tests"] == u"PDR":
869 tst_data[u"throughput"][u"PDR"][u"LOWER"]
870 elif table[u"include-tests"] == u"NDR":
872 tst_data[u"throughput"][u"NDR"][u"LOWER"]
875 if result is not None:
876 tbl_dict[tst_name_mod][u"ref-data"].append(
878 except (KeyError, TypeError):
882 for tst_name in tbl_dict:
883 item = [tbl_dict[tst_name][u"name"], ]
884 data_r = tbl_dict[tst_name][u"ref-data"]
886 if table[u"include-tests"] == u"MRR":
887 data_r_mean = data_r[0][0]
888 data_r_stdev = data_r[0][1]
890 data_r_mean = mean(data_r)
891 data_r_stdev = stdev(data_r)
892 item.append(round(data_r_mean / 1e6, 1))
893 item.append(round(data_r_stdev / 1e6, 1))
897 item.extend([None, None])
898 data_c = tbl_dict[tst_name][u"cmp-data"]
900 if table[u"include-tests"] == u"MRR":
901 data_c_mean = data_c[0][0]
902 data_c_stdev = data_c[0][1]
904 data_c_mean = mean(data_c)
905 data_c_stdev = stdev(data_c)
906 item.append(round(data_c_mean / 1e6, 1))
907 item.append(round(data_c_stdev / 1e6, 1))
911 item.extend([None, None])
912 if data_r_mean is not None and data_c_mean is not None:
913 delta, d_stdev = relative_change_stdev(
914 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
916 item.append(round(delta))
920 item.append(round(d_stdev))
925 # Sort the table according to the relative change
926 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
928 # Generate csv tables:
929 csv_file_name = f"{table[u'output-file']}.csv"
930 with open(csv_file_name, u"wt") as file_handler:
931 file_handler.write(header_str)
933 file_handler.write(u";".join([str(item) for item in test]) + u"\n")
935 convert_csv_to_pretty_txt(
936 csv_file_name, f"{table[u'output-file']}.txt", delimiter=u";"
938 with open(f"{table[u'output-file']}.txt", u'a') as file_handler:
939 file_handler.write(legend)
941 # Generate html table:
942 _tpc_generate_html_table(
945 table[u'output-file'],
947 title=table.get(u"title", u"")
951 def table_perf_trending_dash(table, input_data):
952 """Generate the table(s) with algorithm:
953 table_perf_trending_dash
954 specified in the specification file.
956 :param table: Table to generate.
957 :param input_data: Data to process.
958 :type table: pandas.Series
959 :type input_data: InputData
962 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
966 f" Creating the data set for the {table.get(u'type', u'')} "
967 f"{table.get(u'title', u'')}."
969 data = input_data.filter_data(table, continue_on_error=True)
971 # Prepare the header of the tables
975 u"Short-Term Change [%]",
976 u"Long-Term Change [%]",
980 header_str = u",".join(header) + u"\n"
982 incl_tests = table.get(u"include-tests", u"MRR")
984 # Prepare data to the table:
986 for job, builds in table[u"data"].items():
988 for tst_name, tst_data in data[job][str(build)].items():
989 if tst_name.lower() in table.get(u"ignore-list", list()):
991 if tbl_dict.get(tst_name, None) is None:
992 groups = re.search(REGEX_NIC, tst_data[u"parent"])
995 nic = groups.group(0)
996 tbl_dict[tst_name] = {
997 u"name": f"{nic}-{tst_data[u'name']}",
998 u"data": OrderedDict()
1001 if incl_tests == u"MRR":
1002 tbl_dict[tst_name][u"data"][str(build)] = \
1003 tst_data[u"result"][u"receive-rate"]
1004 elif incl_tests == u"NDR":
1005 tbl_dict[tst_name][u"data"][str(build)] = \
1006 tst_data[u"throughput"][u"NDR"][u"LOWER"]
1007 elif incl_tests == u"PDR":
1008 tbl_dict[tst_name][u"data"][str(build)] = \
1009 tst_data[u"throughput"][u"PDR"][u"LOWER"]
1010 except (TypeError, KeyError):
1011 pass # No data in output.xml for this test
1014 for tst_name in tbl_dict:
1015 data_t = tbl_dict[tst_name][u"data"]
1020 classification_lst, avgs, _ = classify_anomalies(data_t)
1021 except ValueError as err:
1022 logging.info(f"{err} Skipping")
1025 win_size = min(len(data_t), table[u"window"])
1026 long_win_size = min(len(data_t), table[u"long-trend-window"])
1030 [x for x in avgs[-long_win_size:-win_size]
1035 avg_week_ago = avgs[max(-win_size, -len(avgs))]
1037 if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
1038 rel_change_last = nan
1040 rel_change_last = round(
1041 ((last_avg - avg_week_ago) / avg_week_ago) * 1e2, 2)
1043 if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
1044 rel_change_long = nan
1046 rel_change_long = round(
1047 ((last_avg - max_long_avg) / max_long_avg) * 1e2, 2)
1049 if classification_lst:
1050 if isnan(rel_change_last) and isnan(rel_change_long):
1052 if isnan(last_avg) or isnan(rel_change_last) or \
1053 isnan(rel_change_long):
1056 [tbl_dict[tst_name][u"name"],
1057 round(last_avg / 1e6, 2),
1060 classification_lst[-win_size+1:].count(u"regression"),
1061 classification_lst[-win_size+1:].count(u"progression")])
1063 tbl_lst.sort(key=lambda rel: rel[0])
1064 tbl_lst.sort(key=lambda rel: rel[3])
1065 tbl_lst.sort(key=lambda rel: rel[2])
1068 for nrr in range(table[u"window"], -1, -1):
1069 tbl_reg = [item for item in tbl_lst if item[4] == nrr]
1070 for nrp in range(table[u"window"], -1, -1):
1071 tbl_out = [item for item in tbl_reg if item[5] == nrp]
1072 tbl_sorted.extend(tbl_out)
1074 file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1076 logging.info(f" Writing file: {file_name}")
1077 with open(file_name, u"wt") as file_handler:
1078 file_handler.write(header_str)
1079 for test in tbl_sorted:
1080 file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1082 logging.info(f" Writing file: {table[u'output-file']}.txt")
1083 convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1086 def _generate_url(testbed, test_name):
1087 """Generate URL to a trending plot from the name of the test case.
1089 :param testbed: The testbed used for testing.
1090 :param test_name: The name of the test case.
1092 :type test_name: str
1093 :returns: The URL to the plot with the trending data for the given test
1098 if u"x520" in test_name:
1100 elif u"x710" in test_name:
1102 elif u"xl710" in test_name:
1104 elif u"xxv710" in test_name:
1106 elif u"vic1227" in test_name:
1108 elif u"vic1385" in test_name:
1110 elif u"x553" in test_name:
1112 elif u"cx556" in test_name or u"cx556a" in test_name:
1114 elif u"ena" in test_name:
1119 if u"64b" in test_name:
1121 elif u"78b" in test_name:
1123 elif u"imix" in test_name:
1124 frame_size = u"imix"
1125 elif u"9000b" in test_name:
1126 frame_size = u"9000b"
1127 elif u"1518b" in test_name:
1128 frame_size = u"1518b"
1129 elif u"114b" in test_name:
1130 frame_size = u"114b"
1134 if u"1t1c" in test_name or \
1135 (u"-1c-" in test_name and
1136 testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv", u"2n-tx2")):
1138 elif u"2t2c" in test_name or \
1139 (u"-2c-" in test_name and
1140 testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv", u"2n-tx2")):
1142 elif u"4t4c" in test_name or \
1143 (u"-4c-" in test_name and
1144 testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv", u"2n-tx2")):
1146 elif u"2t1c" in test_name or \
1147 (u"-1c-" in test_name and
1149 (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2", u"2n-aws", u"3n-aws")):
1151 elif u"4t2c" in test_name or \
1152 (u"-2c-" in test_name and
1154 (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2", u"2n-aws", u"3n-aws")):
1156 elif u"8t4c" in test_name or \
1157 (u"-4c-" in test_name and
1159 (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2", u"2n-aws", u"3n-aws")):
1164 if u"testpmd" in test_name:
1166 elif u"l3fwd" in test_name:
1168 elif u"avf" in test_name:
1170 elif u"af-xdp" in test_name or u"af_xdp" in test_name:
1172 elif u"rdma" in test_name:
1174 elif u"dnv" in testbed or u"tsh" in testbed:
1176 elif u"ena" in test_name:
1181 if u"macip-iacl1s" in test_name:
1182 bsf = u"features-macip-iacl1"
1183 elif u"macip-iacl10s" in test_name:
1184 bsf = u"features-macip-iacl10"
1185 elif u"macip-iacl50s" in test_name:
1186 bsf = u"features-macip-iacl50"
1187 elif u"iacl1s" in test_name:
1188 bsf = u"features-iacl1"
1189 elif u"iacl10s" in test_name:
1190 bsf = u"features-iacl10"
1191 elif u"iacl50s" in test_name:
1192 bsf = u"features-iacl50"
1193 elif u"oacl1s" in test_name:
1194 bsf = u"features-oacl1"
1195 elif u"oacl10s" in test_name:
1196 bsf = u"features-oacl10"
1197 elif u"oacl50s" in test_name:
1198 bsf = u"features-oacl50"
1199 elif u"nat44det" in test_name:
1200 bsf = u"nat44det-bidir"
1201 elif u"nat44ed" in test_name and u"udir" in test_name:
1202 bsf = u"nat44ed-udir"
1203 elif u"-cps" in test_name and u"ethip4udp" in test_name:
1205 elif u"-cps" in test_name and u"ethip4tcp" in test_name:
1207 elif u"-pps" in test_name and u"ethip4udp" in test_name:
1209 elif u"-pps" in test_name and u"ethip4tcp" in test_name:
1211 elif u"-tput" in test_name and u"ethip4udp" in test_name:
1213 elif u"-tput" in test_name and u"ethip4tcp" in test_name:
1215 elif u"udpsrcscale" in test_name:
1216 bsf = u"features-udp"
1217 elif u"iacl" in test_name:
1219 elif u"policer" in test_name:
1221 elif u"adl" in test_name:
1223 elif u"cop" in test_name:
1225 elif u"nat" in test_name:
1227 elif u"macip" in test_name:
1229 elif u"scale" in test_name:
1231 elif u"base" in test_name:
1236 if u"114b" in test_name and u"vhost" in test_name:
1238 elif u"nat44" in test_name or u"-pps" in test_name or u"-cps" in test_name:
1240 if u"nat44det" in test_name:
1241 domain += u"-det-bidir"
1244 if u"udir" in test_name:
1245 domain += u"-unidir"
1246 elif u"-ethip4udp-" in test_name:
1248 elif u"-ethip4tcp-" in test_name:
1250 if u"-cps" in test_name:
1252 elif u"-pps" in test_name:
1254 elif u"-tput" in test_name:
1256 elif u"testpmd" in test_name or u"l3fwd" in test_name:
1258 elif u"memif" in test_name:
1259 domain = u"container_memif"
1260 elif u"srv6" in test_name:
1262 elif u"vhost" in test_name:
1264 if u"vppl2xc" in test_name:
1267 driver += u"-testpmd"
1268 if u"lbvpplacp" in test_name:
1269 bsf += u"-link-bonding"
1270 elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
1271 domain = u"nf_service_density_vnfc"
1272 elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
1273 domain = u"nf_service_density_cnfc"
1274 elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
1275 domain = u"nf_service_density_cnfp"
1276 elif u"ipsec" in test_name:
1278 if u"sw" in test_name:
1280 elif u"hw" in test_name:
1282 elif u"spe" in test_name:
1284 elif u"ethip4vxlan" in test_name:
1285 domain = u"ip4_tunnels"
1286 elif u"ethip4udpgeneve" in test_name:
1287 domain = u"ip4_tunnels"
1288 elif u"ip4base" in test_name or u"ip4scale" in test_name:
1290 elif u"ip6base" in test_name or u"ip6scale" in test_name:
1292 elif u"l2xcbase" in test_name or \
1293 u"l2xcscale" in test_name or \
1294 u"l2bdbasemaclrn" in test_name or \
1295 u"l2bdscale" in test_name or \
1296 u"l2patch" in test_name:
1301 file_name = u"-".join((domain, testbed, nic)) + u".html#"
1302 anchor_name = u"-".join((frame_size, cores, bsf, driver))
1304 return file_name + anchor_name
1307 def table_perf_trending_dash_html(table, input_data):
1308 """Generate the table(s) with algorithm:
1309 table_perf_trending_dash_html specified in the specification
1312 :param table: Table to generate.
1313 :param input_data: Data to process.
1315 :type input_data: InputData
1320 if not table.get(u"testbed", None):
1322 f"The testbed is not defined for the table "
1323 f"{table.get(u'title', u'')}. Skipping."
1327 test_type = table.get(u"test-type", u"MRR")
1328 if test_type not in (u"MRR", u"NDR", u"PDR"):
1330 f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
1335 if test_type in (u"NDR", u"PDR"):
1336 lnk_dir = u"../ndrpdr_trending/"
1337 lnk_sufix = f"-{test_type.lower()}"
1339 lnk_dir = u"../trending/"
1342 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1345 with open(table[u"input-file"], u'rt') as csv_file:
1346 csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1347 except FileNotFoundError as err:
1348 logging.warning(f"{err}")
1351 logging.warning(u"The input file is not defined.")
1353 except csv.Error as err:
1355 f"Not possible to process the file {table[u'input-file']}.\n"
1361 dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1364 trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1365 for idx, item in enumerate(csv_lst[0]):
1366 alignment = u"left" if idx == 0 else u"center"
1367 thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1385 for r_idx, row in enumerate(csv_lst[1:]):
1387 color = u"regression"
1389 color = u"progression"
1392 trow = ET.SubElement(
1393 dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2])
1397 for c_idx, item in enumerate(row):
1398 tdata = ET.SubElement(
1401 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1404 if c_idx == 0 and table.get(u"add-links", True):
1405 ref = ET.SubElement(
1410 f"{_generate_url(table.get(u'testbed', ''), item)}"
1418 with open(table[u"output-file"], u'w') as html_file:
1419 logging.info(f" Writing file: {table[u'output-file']}")
1420 html_file.write(u".. raw:: html\n\n\t")
1421 html_file.write(str(ET.tostring(dashboard, encoding=u"unicode")))
1422 html_file.write(u"\n\t<p><br><br></p>\n")
1424 logging.warning(u"The output file is not defined.")
1428 def table_last_failed_tests(table, input_data):
1429 """Generate the table(s) with algorithm: table_last_failed_tests
1430 specified in the specification file.
1432 :param table: Table to generate.
1433 :param input_data: Data to process.
1434 :type table: pandas.Series
1435 :type input_data: InputData
1438 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1440 # Transform the data
1442 f" Creating the data set for the {table.get(u'type', u'')} "
1443 f"{table.get(u'title', u'')}."
1446 data = input_data.filter_data(table, continue_on_error=True)
1448 if data is None or data.empty:
1450 f" No data for the {table.get(u'type', u'')} "
1451 f"{table.get(u'title', u'')}."
1456 for job, builds in table[u"data"].items():
1457 for build in builds:
1460 version = input_data.metadata(job, build).get(u"version", u"")
1462 input_data.metadata(job, build).get(u"elapsedtime", u"")
1464 logging.error(f"Data for {job}: {build} is not present.")
1466 tbl_list.append(build)
1467 tbl_list.append(version)
1468 failed_tests = list()
1471 for tst_data in data[job][build].values:
1472 if tst_data[u"status"] != u"FAIL":
1476 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1479 nic = groups.group(0)
1480 msg = tst_data[u'msg'].replace(u"\n", u"")
1481 msg = re.sub(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})',
1482 'xxx.xxx.xxx.xxx', msg)
1483 msg = msg.split(u'Also teardown failed')[0]
1484 failed_tests.append(f"{nic}-{tst_data[u'name']}###{msg}")
1485 tbl_list.append(passed)
1486 tbl_list.append(failed)
1487 tbl_list.append(duration)
1488 tbl_list.extend(failed_tests)
1490 file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1491 logging.info(f" Writing file: {file_name}")
1492 with open(file_name, u"wt") as file_handler:
1493 for test in tbl_list:
1494 file_handler.write(f"{test}\n")
1497 def table_failed_tests(table, input_data):
1498 """Generate the table(s) with algorithm: table_failed_tests
1499 specified in the specification file.
1501 :param table: Table to generate.
1502 :param input_data: Data to process.
1503 :type table: pandas.Series
1504 :type input_data: InputData
1507 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1509 # Transform the data
1511 f" Creating the data set for the {table.get(u'type', u'')} "
1512 f"{table.get(u'title', u'')}."
1514 data = input_data.filter_data(table, continue_on_error=True)
1517 if u"NDRPDR" in table.get(u"filter", list()):
1518 test_type = u"NDRPDR"
1520 # Prepare the header of the tables
1524 u"Last Failure [Time]",
1525 u"Last Failure [VPP-Build-Id]",
1526 u"Last Failure [CSIT-Job-Build-Id]"
1529 # Generate the data for the table according to the model in the table
1533 timeperiod = timedelta(int(table.get(u"window", 7)))
1536 for job, builds in table[u"data"].items():
1537 for build in builds:
1539 for tst_name, tst_data in data[job][build].items():
1540 if tst_name.lower() in table.get(u"ignore-list", list()):
1542 if tbl_dict.get(tst_name, None) is None:
1543 groups = re.search(REGEX_NIC, tst_data[u"parent"])
1546 nic = groups.group(0)
1547 tbl_dict[tst_name] = {
1548 u"name": f"{nic}-{tst_data[u'name']}",
1549 u"data": OrderedDict()
1552 generated = input_data.metadata(job, build).\
1553 get(u"generated", u"")
1556 then = dt.strptime(generated, u"%Y%m%d %H:%M")
1557 if (now - then) <= timeperiod:
1558 tbl_dict[tst_name][u"data"][build] = (
1559 tst_data[u"status"],
1561 input_data.metadata(job, build).get(u"version",
1565 except (TypeError, KeyError) as err:
1566 logging.warning(f"tst_name: {tst_name} - err: {repr(err)}")
1570 for tst_data in tbl_dict.values():
1572 fails_last_date = u""
1573 fails_last_vpp = u""
1574 fails_last_csit = u""
1575 for val in tst_data[u"data"].values():
1576 if val[0] == u"FAIL":
1578 fails_last_date = val[1]
1579 fails_last_vpp = val[2]
1580 fails_last_csit = val[3]
1582 max_fails = fails_nr if fails_nr > max_fails else max_fails
1588 f"{u'mrr-daily' if test_type == u'MRR' else u'ndrpdr-weekly'}"
1589 f"-build-{fails_last_csit}"
1592 tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1594 for nrf in range(max_fails, -1, -1):
1595 tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1596 tbl_sorted.extend(tbl_fails)
1598 file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
1599 logging.info(f" Writing file: {file_name}")
1600 with open(file_name, u"wt") as file_handler:
1601 file_handler.write(u",".join(header) + u"\n")
1602 for test in tbl_sorted:
1603 file_handler.write(u",".join([str(item) for item in test]) + u'\n')
1605 logging.info(f" Writing file: {table[u'output-file']}.txt")
1606 convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
1609 def table_failed_tests_html(table, input_data):
1610 """Generate the table(s) with algorithm: table_failed_tests_html
1611 specified in the specification file.
1613 :param table: Table to generate.
1614 :param input_data: Data to process.
1615 :type table: pandas.Series
1616 :type input_data: InputData
1621 if not table.get(u"testbed", None):
1623 f"The testbed is not defined for the table "
1624 f"{table.get(u'title', u'')}. Skipping."
1628 test_type = table.get(u"test-type", u"MRR")
1629 if test_type not in (u"MRR", u"NDR", u"PDR", u"NDRPDR"):
1631 f"Test type {table.get(u'test-type', u'MRR')} is not defined. "
1636 if test_type in (u"NDRPDR", u"NDR", u"PDR"):
1637 lnk_dir = u"../ndrpdr_trending/"
1640 lnk_dir = u"../trending/"
1643 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1646 with open(table[u"input-file"], u'rt') as csv_file:
1647 csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"'))
1649 logging.warning(u"The input file is not defined.")
1651 except csv.Error as err:
1653 f"Not possible to process the file {table[u'input-file']}.\n"
1659 failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0'))
1662 trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7"))
1663 for idx, item in enumerate(csv_lst[0]):
1664 alignment = u"left" if idx == 0 else u"center"
1665 thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment))
1669 colors = (u"#e9f1fb", u"#d4e4f7")
1670 for r_idx, row in enumerate(csv_lst[1:]):
1671 background = colors[r_idx % 2]
1672 trow = ET.SubElement(
1673 failed_tests, u"tr", attrib=dict(bgcolor=background)
1677 for c_idx, item in enumerate(row):
1678 tdata = ET.SubElement(
1681 attrib=dict(align=u"left" if c_idx == 0 else u"center")
1684 if c_idx == 0 and table.get(u"add-links", True):
1685 ref = ET.SubElement(
1690 f"{_generate_url(table.get(u'testbed', ''), item)}"
1698 with open(table[u"output-file"], u'w') as html_file:
1699 logging.info(f" Writing file: {table[u'output-file']}")
1700 html_file.write(u".. raw:: html\n\n\t")
1701 html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode")))
1702 html_file.write(u"\n\t<p><br><br></p>\n")
1704 logging.warning(u"The output file is not defined.")
1708 def table_comparison(table, input_data):
1709 """Generate the table(s) with algorithm: table_comparison
1710 specified in the specification file.
1712 :param table: Table to generate.
1713 :param input_data: Data to process.
1714 :type table: pandas.Series
1715 :type input_data: InputData
1717 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
1719 # Transform the data
1721 f" Creating the data set for the {table.get(u'type', u'')} "
1722 f"{table.get(u'title', u'')}."
1725 columns = table.get(u"columns", None)
1728 f"No columns specified for {table.get(u'title', u'')}. Skipping."
1733 for idx, col in enumerate(columns):
1734 if col.get(u"data-set", None) is None:
1735 logging.warning(f"No data for column {col.get(u'title', u'')}")
1737 tag = col.get(u"tag", None)
1738 data = input_data.filter_data(
1748 data=col[u"data-set"],
1749 continue_on_error=True
1752 u"title": col.get(u"title", f"Column{idx}"),
1755 for builds in data.values:
1756 for build in builds:
1757 for tst_name, tst_data in build.items():
1758 if tag and tag not in tst_data[u"tags"]:
1761 _tpc_modify_test_name(tst_name, ignore_nic=True).\
1762 replace(u"2n1l-", u"")
1763 if col_data[u"data"].get(tst_name_mod, None) is None:
1764 name = tst_data[u'name'].rsplit(u'-', 1)[0]
1765 if u"across testbeds" in table[u"title"].lower() or \
1766 u"across topologies" in table[u"title"].lower():
1767 name = _tpc_modify_displayed_test_name(name)
1768 col_data[u"data"][tst_name_mod] = {
1776 target=col_data[u"data"][tst_name_mod],
1778 include_tests=table[u"include-tests"]
1781 replacement = col.get(u"data-replacement", None)
1783 rpl_data = input_data.filter_data(
1794 continue_on_error=True
1796 for builds in rpl_data.values:
1797 for build in builds:
1798 for tst_name, tst_data in build.items():
1799 if tag and tag not in tst_data[u"tags"]:
1802 _tpc_modify_test_name(tst_name, ignore_nic=True).\
1803 replace(u"2n1l-", u"")
1804 if col_data[u"data"].get(tst_name_mod, None) is None:
1805 name = tst_data[u'name'].rsplit(u'-', 1)[0]
1806 if u"across testbeds" in table[u"title"].lower() \
1807 or u"across topologies" in \
1808 table[u"title"].lower():
1809 name = _tpc_modify_displayed_test_name(name)
1810 col_data[u"data"][tst_name_mod] = {
1817 if col_data[u"data"][tst_name_mod][u"replace"]:
1818 col_data[u"data"][tst_name_mod][u"replace"] = False
1819 col_data[u"data"][tst_name_mod][u"data"] = list()
1821 target=col_data[u"data"][tst_name_mod],
1823 include_tests=table[u"include-tests"]
1826 if table[u"include-tests"] in (u"NDR", u"PDR") or \
1827 u"latency" in table[u"include-tests"]:
1828 for tst_name, tst_data in col_data[u"data"].items():
1829 if tst_data[u"data"]:
1830 tst_data[u"mean"] = mean(tst_data[u"data"])
1831 tst_data[u"stdev"] = stdev(tst_data[u"data"])
1833 cols.append(col_data)
1837 for tst_name, tst_data in col[u"data"].items():
1838 if tbl_dict.get(tst_name, None) is None:
1839 tbl_dict[tst_name] = {
1840 "name": tst_data[u"name"]
1842 tbl_dict[tst_name][col[u"title"]] = {
1843 u"mean": tst_data[u"mean"],
1844 u"stdev": tst_data[u"stdev"]
1848 logging.warning(f"No data for table {table.get(u'title', u'')}!")
1852 for tst_data in tbl_dict.values():
1853 row = [tst_data[u"name"], ]
1855 row.append(tst_data.get(col[u"title"], None))
1858 comparisons = table.get(u"comparisons", None)
1860 if comparisons and isinstance(comparisons, list):
1861 for idx, comp in enumerate(comparisons):
1863 col_ref = int(comp[u"reference"])
1864 col_cmp = int(comp[u"compare"])
1866 logging.warning(u"Comparison: No references defined! Skipping.")
1867 comparisons.pop(idx)
1869 if not (0 < col_ref <= len(cols) and 0 < col_cmp <= len(cols) or
1870 col_ref == col_cmp):
1871 logging.warning(f"Wrong values of reference={col_ref} "
1872 f"and/or compare={col_cmp}. Skipping.")
1873 comparisons.pop(idx)
1875 rca_file_name = comp.get(u"rca-file", None)
1878 with open(rca_file_name, u"r") as file_handler:
1881 u"title": f"RCA{idx + 1}",
1882 u"data": load(file_handler, Loader=FullLoader)
1885 except (YAMLError, IOError) as err:
1887 f"The RCA file {rca_file_name} does not exist or "
1890 logging.debug(repr(err))
1897 tbl_cmp_lst = list()
1900 new_row = deepcopy(row)
1901 for comp in comparisons:
1902 ref_itm = row[int(comp[u"reference"])]
1903 if ref_itm is None and \
1904 comp.get(u"reference-alt", None) is not None:
1905 ref_itm = row[int(comp[u"reference-alt"])]
1906 cmp_itm = row[int(comp[u"compare"])]
1907 if ref_itm is not None and cmp_itm is not None and \
1908 ref_itm[u"mean"] is not None and \
1909 cmp_itm[u"mean"] is not None and \
1910 ref_itm[u"stdev"] is not None and \
1911 cmp_itm[u"stdev"] is not None:
1913 delta, d_stdev = relative_change_stdev(
1914 ref_itm[u"mean"], cmp_itm[u"mean"],
1915 ref_itm[u"stdev"], cmp_itm[u"stdev"]
1917 except ZeroDivisionError:
1919 if delta is None or math.isnan(delta):
1922 u"mean": delta * 1e6,
1923 u"stdev": d_stdev * 1e6
1928 tbl_cmp_lst.append(new_row)
1931 tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
1932 tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
1933 except TypeError as err:
1934 logging.warning(f"Empty data element in table\n{tbl_cmp_lst}\n{err}")
1936 tbl_for_csv = list()
1937 for line in tbl_cmp_lst:
1939 for idx, itm in enumerate(line[1:]):
1940 if itm is None or not isinstance(itm, dict) or\
1941 itm.get(u'mean', None) is None or \
1942 itm.get(u'stdev', None) is None:
1946 row.append(round(float(itm[u'mean']) / 1e6, 3))
1947 row.append(round(float(itm[u'stdev']) / 1e6, 3))
1951 rca_nr = rca[u"data"].get(row[0], u"-")
1952 row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-")
1953 tbl_for_csv.append(row)
1955 header_csv = [u"Test Case", ]
1957 header_csv.append(f"Avg({col[u'title']})")
1958 header_csv.append(f"Stdev({col[u'title']})")
1959 for comp in comparisons:
1961 f"Avg({comp.get(u'title', u'')})"
1964 f"Stdev({comp.get(u'title', u'')})"
1968 header_csv.append(rca[u"title"])
1970 legend_lst = table.get(u"legend", None)
1971 if legend_lst is None:
1974 legend = u"\n" + u"\n".join(legend_lst) + u"\n"
1977 if rcas and any(rcas):
1978 footnote += u"\nRoot Cause Analysis:\n"
1981 footnote += f"{rca[u'data'].get(u'footnote', u'')}\n"
1983 csv_file_name = f"{table[u'output-file']}-csv.csv"
1984 with open(csv_file_name, u"wt", encoding='utf-8') as file_handler:
1986 u",".join([f'"{itm}"' for itm in header_csv]) + u"\n"
1988 for test in tbl_for_csv:
1990 u",".join([f'"{item}"' for item in test]) + u"\n"
1993 for item in legend_lst:
1994 file_handler.write(f'"{item}"\n')
1996 for itm in footnote.split(u"\n"):
1997 file_handler.write(f'"{itm}"\n')
2000 max_lens = [0, ] * len(tbl_cmp_lst[0])
2001 for line in tbl_cmp_lst:
2003 for idx, itm in enumerate(line[1:]):
2004 if itm is None or not isinstance(itm, dict) or \
2005 itm.get(u'mean', None) is None or \
2006 itm.get(u'stdev', None) is None:
2011 f"{round(float(itm[u'mean']) / 1e6, 2)} "
2012 f"\u00B1{round(float(itm[u'stdev']) / 1e6, 2)}".
2013 replace(u"nan", u"NaN")
2017 f"{round(float(itm[u'mean']) / 1e6, 2):+} "
2018 f"\u00B1{round(float(itm[u'stdev']) / 1e6, 2)}".
2019 replace(u"nan", u"NaN")
2021 if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]:
2022 max_lens[idx] = len(new_itm.rsplit(u" ", 1)[-1])
2027 header = [u"Test Case", ]
2028 header.extend([col[u"title"] for col in cols])
2029 header.extend([comp.get(u"title", u"") for comp in comparisons])
2032 for line in tbl_tmp:
2034 for idx, itm in enumerate(line[1:]):
2035 if itm in (u"NT", u"NaN"):
2038 itm_lst = itm.rsplit(u"\u00B1", 1)
2040 f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}"
2041 itm_str = u"\u00B1".join(itm_lst)
2043 if idx >= len(cols):
2045 rca = rcas[idx - len(cols)]
2048 rca_nr = rca[u"data"].get(row[0], None)
2050 hdr_len = len(header[idx + 1]) - 1
2053 rca_nr = f"[{rca_nr}]"
2055 f"{u' ' * (4 - len(rca_nr))}{rca_nr}"
2056 f"{u' ' * (hdr_len - 4 - len(itm_str))}"
2060 tbl_final.append(row)
2062 # Generate csv tables:
2063 csv_file_name = f"{table[u'output-file']}.csv"
2064 logging.info(f" Writing the file {csv_file_name}")
2065 with open(csv_file_name, u"wt", encoding='utf-8') as file_handler:
2066 file_handler.write(u";".join(header) + u"\n")
2067 for test in tbl_final:
2068 file_handler.write(u";".join([str(item) for item in test]) + u"\n")
2070 # Generate txt table:
2071 txt_file_name = f"{table[u'output-file']}.txt"
2072 logging.info(f" Writing the file {txt_file_name}")
2073 convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u";")
2075 with open(txt_file_name, u'a', encoding='utf-8') as file_handler:
2076 file_handler.write(legend)
2077 file_handler.write(footnote)
2079 # Generate html table:
2080 _tpc_generate_html_table(
2083 table[u'output-file'],
2087 title=table.get(u"title", u"")
2091 def table_weekly_comparison(table, in_data):
2092 """Generate the table(s) with algorithm: table_weekly_comparison
2093 specified in the specification file.
2095 :param table: Table to generate.
2096 :param in_data: Data to process.
2097 :type table: pandas.Series
2098 :type in_data: InputData
2100 logging.info(f" Generating the table {table.get(u'title', u'')} ...")
2102 # Transform the data
2104 f" Creating the data set for the {table.get(u'type', u'')} "
2105 f"{table.get(u'title', u'')}."
2108 incl_tests = table.get(u"include-tests", None)
2109 if incl_tests not in (u"NDR", u"PDR"):
2110 logging.error(f"Wrong tests to include specified ({incl_tests}).")
2113 nr_cols = table.get(u"nr-of-data-columns", None)
2114 if not nr_cols or nr_cols < 2:
2116 f"No columns specified for {table.get(u'title', u'')}. Skipping."
2120 data = in_data.filter_data(
2122 params=[u"throughput", u"result", u"name", u"parent", u"tags"],
2123 continue_on_error=True
2128 [u"Start Timestamp", ],
2134 tb_tbl = table.get(u"testbeds", None)
2135 for job_name, job_data in data.items():
2136 for build_nr, build in job_data.items():
2142 tb_ip = in_data.metadata(job_name, build_nr).get(u"testbed", u"")
2143 if tb_ip and tb_tbl:
2144 testbed = tb_tbl.get(tb_ip, u"")
2147 header[2].insert(1, build_nr)
2148 header[3].insert(1, testbed)
2150 1, in_data.metadata(job_name, build_nr).get(u"generated", u"")
2153 1, in_data.metadata(job_name, build_nr).get(u"version", u"")
2156 for tst_name, tst_data in build.items():
2158 _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
2159 if not tbl_dict.get(tst_name_mod, None):
2160 tbl_dict[tst_name_mod] = dict(
2161 name=tst_data[u'name'].rsplit(u'-', 1)[0],
2164 tbl_dict[tst_name_mod][-idx - 1] = \
2165 tst_data[u"throughput"][incl_tests][u"LOWER"]
2166 except (TypeError, IndexError, KeyError, ValueError):
2171 logging.error(u"Not enough data to build the table! Skipping")
2175 for idx, cmp in enumerate(table.get(u"comparisons", list())):
2176 idx_ref = cmp.get(u"reference", None)
2177 idx_cmp = cmp.get(u"compare", None)
2178 if idx_ref is None or idx_cmp is None:
2181 f"Diff({header[0][idx_ref - idx].split(u'~')[-1]} vs "
2182 f"{header[0][idx_cmp - idx].split(u'~')[-1]})"
2184 header[1].append(u"")
2185 header[2].append(u"")
2186 header[3].append(u"")
2187 for tst_name, tst_data in tbl_dict.items():
2188 if not cmp_dict.get(tst_name, None):
2189 cmp_dict[tst_name] = list()
2190 ref_data = tst_data.get(idx_ref, None)
2191 cmp_data = tst_data.get(idx_cmp, None)
2192 if ref_data is None or cmp_data is None:
2193 cmp_dict[tst_name].append(float(u'nan'))
2195 cmp_dict[tst_name].append(
2196 relative_change(ref_data, cmp_data)
2199 tbl_lst_none = list()
2201 for tst_name, tst_data in tbl_dict.items():
2202 itm_lst = [tst_data[u"name"], ]
2203 for idx in range(nr_cols):
2204 item = tst_data.get(-idx - 1, None)
2206 itm_lst.insert(1, None)
2208 itm_lst.insert(1, round(item / 1e6, 1))
2211 None if itm is None else round(itm, 1)
2212 for itm in cmp_dict[tst_name]
2215 if str(itm_lst[-1]) == u"nan" or itm_lst[-1] is None:
2216 tbl_lst_none.append(itm_lst)
2218 tbl_lst.append(itm_lst)
2220 tbl_lst_none.sort(key=lambda rel: rel[0], reverse=False)
2221 tbl_lst.sort(key=lambda rel: rel[0], reverse=False)
2222 tbl_lst.sort(key=lambda rel: rel[-1], reverse=False)
2223 tbl_lst.extend(tbl_lst_none)
2225 # Generate csv table:
2226 csv_file_name = f"{table[u'output-file']}.csv"
2227 logging.info(f" Writing the file {csv_file_name}")
2228 with open(csv_file_name, u"wt", encoding='utf-8') as file_handler:
2230 file_handler.write(u",".join(hdr) + u"\n")
2231 for test in tbl_lst:
2232 file_handler.write(u",".join(
2234 str(item).replace(u"None", u"-").replace(u"nan", u"-").
2235 replace(u"null", u"-") for item in test
2239 txt_file_name = f"{table[u'output-file']}.txt"
2240 logging.info(f" Writing the file {txt_file_name}")
2241 convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u",")
2243 # Reorganize header in txt table
2245 with open(txt_file_name, u"rt", encoding='utf-8') as file_handler:
2246 for line in list(file_handler):
2247 txt_table.append(line)
2249 txt_table.insert(5, txt_table.pop(2))
2250 with open(txt_file_name, u"wt", encoding='utf-8') as file_handler:
2251 file_handler.writelines(txt_table)
2255 # Generate html table:
2257 u"<br>".join(row) for row in zip(*header)
2259 _tpc_generate_html_table(
2262 table[u'output-file'],
2264 title=table.get(u"title", u""),