+
+
+def table_comparison(table, input_data):
+ """Generate the table(s) with algorithm: table_comparison
+ specified in the specification file.
+
+ :param table: Table to generate.
+ :param input_data: Data to process.
+ :type table: pandas.Series
+ :type input_data: InputData
+ """
+ logging.info(f" Generating the table {table.get(u'title', u'')} ...")
+
+ # Transform the data
+ logging.info(
+ f" Creating the data set for the {table.get(u'type', u'')} "
+ f"{table.get(u'title', u'')}."
+ )
+
+ columns = table.get(u"columns", None)
+ if not columns:
+ logging.error(
+ f"No columns specified for {table.get(u'title', u'')}. Skipping."
+ )
+ return
+
+ cols = list()
+ for idx, col in enumerate(columns):
+ if col.get(u"data", None) is None:
+ logging.warning(f"No data for column {col.get(u'title', u'')}")
+ continue
+ data = input_data.filter_data(
+ table,
+ params=[u"throughput", u"result", u"name", u"parent", u"tags"],
+ data=col[u"data"],
+ continue_on_error=True
+ )
+ col_data = {
+ u"title": col.get(u"title", f"Column{idx}"),
+ u"data": dict()
+ }
+ for builds in data.values:
+ for build in builds:
+ for tst_name, tst_data in build.items():
+ tst_name_mod = \
+ _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"")
+ if col_data[u"data"].get(tst_name_mod, None) is None:
+ name = tst_data[u'name'].rsplit(u'-', 1)[0]
+ if u"across testbeds" in table[u"title"].lower() or \
+ u"across topologies" in table[u"title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ col_data[u"data"][tst_name_mod] = {
+ u"name": name,
+ u"replace": True,
+ u"data": list(),
+ u"mean": None,
+ u"stdev": None
+ }
+ _tpc_insert_data(
+ target=col_data[u"data"][tst_name_mod][u"data"],
+ src=tst_data,
+ include_tests=table[u"include-tests"]
+ )
+
+ replacement = col.get(u"data-replacement", None)
+ if replacement:
+ rpl_data = input_data.filter_data(
+ table,
+ params=[u"throughput", u"result", u"name", u"parent", u"tags"],
+ data=replacement,
+ continue_on_error=True
+ )
+ for builds in rpl_data.values:
+ for build in builds:
+ for tst_name, tst_data in build.items():
+ tst_name_mod = \
+ _tpc_modify_test_name(tst_name).\
+ replace(u"2n1l-", u"")
+ if col_data[u"data"].get(tst_name_mod, None) is None:
+ name = tst_data[u'name'].rsplit(u'-', 1)[0]
+ if u"across testbeds" in table[u"title"].lower() \
+ or u"across topologies" in \
+ table[u"title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ col_data[u"data"][tst_name_mod] = {
+ u"name": name,
+ u"replace": False,
+ u"data": list(),
+ u"mean": None,
+ u"stdev": None
+ }
+ if col_data[u"data"][tst_name_mod][u"replace"]:
+ col_data[u"data"][tst_name_mod][u"replace"] = False
+ col_data[u"data"][tst_name_mod][u"data"] = list()
+ _tpc_insert_data(
+ target=col_data[u"data"][tst_name_mod][u"data"],
+ src=tst_data,
+ include_tests=table[u"include-tests"]
+ )
+
+ if table[u"include-tests"] in (u"NDR", u"PDR"):
+ for tst_name, tst_data in col_data[u"data"].items():
+ if tst_data[u"data"]:
+ tst_data[u"mean"] = mean(tst_data[u"data"])
+ tst_data[u"stdev"] = stdev(tst_data[u"data"])
+ elif table[u"include-tests"] in (u"MRR", ):
+ for tst_name, tst_data in col_data[u"data"].items():
+ if tst_data[u"data"]:
+ tst_data[u"mean"] = tst_data[u"data"][0]
+ tst_data[u"stdev"] = tst_data[u"data"][0]
+
+ cols.append(col_data)
+
+ tbl_dict = dict()
+ for col in cols:
+ for tst_name, tst_data in col[u"data"].items():
+ if tbl_dict.get(tst_name, None) is None:
+ tbl_dict[tst_name] = {
+ "name": tst_data[u"name"]
+ }
+ tbl_dict[tst_name][col[u"title"]] = {
+ u"mean": tst_data[u"mean"],
+ u"stdev": tst_data[u"stdev"]
+ }
+
+ tbl_lst = list()
+ for tst_data in tbl_dict.values():
+ row = [tst_data[u"name"], ]
+ for col in cols:
+ row.append(tst_data.get(col[u"title"], None))
+ tbl_lst.append(row)
+
+ comparisons = table.get(u"comparisons", None)
+ if comparisons and isinstance(comparisons, list):
+ for idx, comp in enumerate(comparisons):
+ try:
+ col_ref = int(comp[u"reference"])
+ col_cmp = int(comp[u"compare"])
+ except KeyError:
+ logging.warning(u"Comparison: No references defined! Skipping.")
+ comparisons.pop(idx)
+ continue
+ if not (0 < col_ref <= len(cols) and
+ 0 < col_cmp <= len(cols)) or \
+ col_ref == col_cmp:
+ logging.warning(f"Wrong values of reference={col_ref} "
+ f"and/or compare={col_cmp}. Skipping.")
+ comparisons.pop(idx)
+ continue
+
+ tbl_cmp_lst = list()
+ if comparisons:
+ for row in tbl_lst:
+ new_row = deepcopy(row)
+ add_to_tbl = False
+ for comp in comparisons:
+ ref_itm = row[int(comp[u"reference"])]
+ if ref_itm is None and \
+ comp.get(u"reference-alt", None) is not None:
+ ref_itm = row[int(comp[u"reference-alt"])]
+ cmp_itm = row[int(comp[u"compare"])]
+ if ref_itm is not None and cmp_itm is not None and \
+ ref_itm[u"mean"] is not None and \
+ cmp_itm[u"mean"] is not None and \
+ ref_itm[u"stdev"] is not None and \
+ cmp_itm[u"stdev"] is not None:
+ delta, d_stdev = relative_change_stdev(
+ ref_itm[u"mean"], cmp_itm[u"mean"],
+ ref_itm[u"stdev"], cmp_itm[u"stdev"]
+ )
+ new_row.append(
+ {
+ u"mean": delta * 1e6,
+ u"stdev": d_stdev * 1e6
+ }
+ )
+ add_to_tbl = True
+ else:
+ new_row.append(None)
+ if add_to_tbl:
+ tbl_cmp_lst.append(new_row)
+
+ tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False)
+ tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True)
+
+ rcas = list()
+ rca_in = table.get(u"rca", None)
+ if rca_in and isinstance(rca_in, list):
+ for idx, itm in enumerate(rca_in):
+ try:
+ with open(itm.get(u"data", u""), u"r") as rca_file:
+ rcas.append(
+ {
+ u"title": itm.get(u"title", f"RCA{idx}"),
+ u"data": load(rca_file, Loader=FullLoader)
+ }
+ )
+ except (YAMLError, IOError) as err:
+ logging.warning(
+ f"The RCA file {itm.get(u'data', u'')} does not exist or "
+ f"it is corrupted!"
+ )
+ logging.debug(repr(err))
+
+ tbl_for_csv = list()
+ for line in tbl_cmp_lst:
+
+ row = [line[0], ]
+
+ for idx, rca in enumerate(rcas):
+ rca_nr = rca[u"data"].get(row[0 + idx], u"-")
+ row.insert(idx, f"[{rca_nr}]" if rca_nr != u"-" else u"-")
+
+ for idx, itm in enumerate(line[1:]):
+ if itm is None:
+ row.append(u"NT")
+ row.append(u"NT")
+ else:
+ row.append(round(float(itm[u'mean']) / 1e6, 3))
+ row.append(round(float(itm[u'stdev']) / 1e6, 3))
+ tbl_for_csv.append(row)
+
+ header_csv = [rca[u"title"] for rca in rcas]
+ header_csv.append(u"Test Case")
+ for col in cols:
+ header_csv.append(f"Avg({col[u'title']})")
+ header_csv.append(f"Stdev({col[u'title']})")
+ for comp in comparisons:
+ header_csv.append(
+ f"Avg({cols[comp[u'reference'] - 1][u'title']},"
+ f"{cols[comp[u'compare'] - 1][u'title']})"
+ )
+ header_csv.append(
+ f"Stdev({cols[comp[u'reference'] - 1][u'title']},"
+ f"{cols[comp[u'compare'] - 1][u'title']})"
+ )
+
+ csv_file = f"{table[u'output-file']}-csv.csv"
+ with open(csv_file, u"wt", encoding='utf-8') as file_handler:
+ file_handler.write(u";".join(header_csv) + u"\n")
+ for test in tbl_for_csv:
+ file_handler.write(u";".join([str(item) for item in test]) + u"\n")
+
+ tbl_final = list()
+ for line in tbl_cmp_lst:
+ row = [line[0], ]
+ for idx, rca in enumerate(rcas):
+ rca_nr = rca[u"data"].get(row[0 + idx], u"-")
+ row.insert(idx, f"[{rca_nr}]" if rca_nr != u"-" else u"-")
+ for idx, itm in enumerate(line[1:]):
+ if itm is None:
+ row.append(u"NT")
+ else:
+ if idx < len(cols):
+ row.append(
+ f"{round(float(itm[u'mean']) / 1e6, 1)} "
+ f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
+ replace(u"nan", u"NaN")
+ )
+ else:
+ row.append(
+ f"{round(float(itm[u'mean']) / 1e6, 1):+} "
+ f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}".
+ replace(u"nan", u"NaN")
+ )
+ tbl_final.append(row)
+
+ header = [rca[u"title"] for rca in rcas]
+ header.append(u"Test Case")
+ header.extend([col[u"title"] for col in cols])
+ header.extend(
+ [f"Diff({cols[comp[u'reference'] - 1][u'title']},"
+ f"{cols[comp[u'compare'] - 1][u'title']})"
+ for comp in comparisons]
+ )
+
+ # Generate csv tables:
+ csv_file = f"{table[u'output-file']}.csv"
+ with open(csv_file, u"wt", encoding='utf-8') as file_handler:
+ file_handler.write(u";".join(header) + u"\n")
+ for test in tbl_final:
+ file_handler.write(u";".join([str(item) for item in test]) + u"\n")
+
+ # Generate txt table:
+ txt_file_name = f"{table[u'output-file']}.txt"
+ convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
+
+ # Generate rst table:
+ file_name = table[u'output-file'].split(u"/")[-1]
+ if u"vpp" in table[u'output-file']:
+ path = u"_tmp/src/vpp_performance_tests/comparisons/"
+ else:
+ path = u"_tmp/src/dpdk_performance_tests/comparisons/"
+ rst_file_name = f"{path}{file_name}-txt.rst"
+ csv_file_name = f"{path}{file_name}.csv"
+ with open(csv_file_name, u"wt", encoding='utf-8') as file_handler:
+ file_handler.write(
+ u",".join(
+ [f'"{itm}"' for itm in header]
+ ) + u"\n"
+ )
+ for test in tbl_final:
+ file_handler.write(
+ u",".join(
+ [f'"{itm}"' for itm in test]
+ ) + u"\n"
+ )
+
+ convert_csv_to_pretty_txt(csv_file_name, rst_file_name, delimiter=u",")
+
+ legend = u"\nLegend:\n"
+ for idx, rca in enumerate(rcas):
+ try:
+ desc = (
+ f"Diff({cols[comparisons[idx][u'reference'] - 1][u'title']},"
+ f"{cols[comparisons[idx][u'compare'] - 1][u'title']})\n"
+ )
+ except (KeyError, IndexError):
+ desc = u"\n"
+ legend += f"{rca[u'title']}: Root Cause Analysis for {desc}"
+ legend += (
+ u"First part of the result is a mean value [Mpps].\n"
+ f"Second part of the result following '\u00B1' is a standard "
+ u"deviation [Mpps].\n"
+ u"First part of Diff is a relative change of mean values [%].\n"
+ f"Second part of Diff following '\u00B1' is a standard deviation "
+ u"of the Diff [percentual points].\n"
+ u"NT: Not tested.\n"
+ )
+
+ footnote = u""
+ for rca in rcas:
+ footnote += f"\n{rca[u'title']}:\n"
+ footnote += rca[u"data"].get(u"footnote", u"")
+
+ with open(txt_file_name, u'a', encoding='utf-8') as txt_file:
+ txt_file.write(legend)
+ if footnote:
+ txt_file.write(footnote)
+ txt_file.write(u":END")
+
+ with open(rst_file_name, u'a', encoding='utf-8') as txt_file:
+ txt_file.write(legend.replace(u"\n", u" |br| "))
+ if footnote:
+ txt_file.write(footnote.replace(u"\n", u" |br| "))
+ txt_file.write(u":END")
+
+ # Generate html table:
+ _tpc_generate_html_table(
+ header,
+ tbl_final,
+ table[u'output-file'],
+ legend=legend,
+ footnote=footnote,
+ sort_data=False
+ )
Code Review / csit.git / commitdiff