X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=4a1ac0ef713875faf3f2fcd907c3ad65011b644e;hb=refs%2Fchanges%2F37%2F23737%2F4;hp=36c6e645579a29c918fd6d2559fee465f4c27380;hpb=259620ea6ca1c5864d4864f0f04d260e89ecb72a;p=csit.git
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 36c6e64557..4a1ac0ef71 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -19,6 +19,10 @@ import logging
import csv
import re
+import plotly.graph_objects as go
+import plotly.offline as ploff
+import pandas as pd
+
from string import replace
from collections import OrderedDict
from numpy import nan, isnan
@@ -136,13 +140,14 @@ def table_merged_details(table, input_data):
# Transform the data
logging.info(" Creating the data set for the {0} '{1}'.".
format(table.get("type", ""), table.get("title", "")))
- data = input_data.filter_data(table)
+ data = input_data.filter_data(table, continue_on_error=True)
data = input_data.merge_data(data)
data.sort_index(inplace=True)
logging.info(" Creating the data set for the {0} '{1}'.".
format(table.get("type", ""), table.get("title", "")))
- suites = input_data.filter_data(table, data_set="suites")
+ suites = input_data.filter_data(
+ table, continue_on_error=True, data_set="suites")
suites = input_data.merge_data(suites)
# Prepare the header of the tables
@@ -187,6 +192,161 @@ def table_merged_details(table, input_data):
logging.info(" Done.")
+def _tpc_modify_test_name(test_name):
+ test_name_mod = test_name.replace("-ndrpdrdisc", ""). \
+ replace("-ndrpdr", "").replace("-pdrdisc", ""). \
+ replace("-ndrdisc", "").replace("-pdr", ""). \
+ replace("-ndr", ""). \
+ replace("1t1c", "1c").replace("2t1c", "1c"). \
+ replace("2t2c", "2c").replace("4t2c", "2c"). \
+ replace("4t4c", "4c").replace("8t4c", "4c")
+ test_name_mod = re.sub(REGEX_NIC, "", test_name_mod)
+ return test_name_mod
+
+
+def _tpc_modify_displayed_test_name(test_name):
+ return test_name.replace("1t1c", "1c").replace("2t1c", "1c"). \
+ replace("2t2c", "2c").replace("4t2c", "2c"). \
+ replace("4t4c", "4c").replace("8t4c", "4c")
+
+
+def _tpc_insert_data(target, src, include_tests):
+ try:
+ if include_tests == "MRR":
+ target.append(src["result"]["receive-rate"]) # .avg)
+ elif include_tests == "PDR":
+ target.append(src["throughput"]["PDR"]["LOWER"])
+ elif include_tests == "NDR":
+ target.append(src["throughput"]["NDR"]["LOWER"])
+ except (KeyError, TypeError):
+ pass
+
+
+def _tpc_sort_table(table):
+ # Sort the table:
+ # 1. New in CSIT-XXXX
+ # 2. See footnote
+ # 3. Delta
+ tbl_new = list()
+ tbl_see = list()
+ tbl_delta = list()
+ for item in table:
+ if isinstance(item[-1], str):
+ if "New in CSIT" in item[-1]:
+ tbl_new.append(item)
+ elif "See footnote" in item[-1]:
+ tbl_see.append(item)
+ else:
+ tbl_delta.append(item)
+
+ # Sort the tables:
+ tbl_new.sort(key=lambda rel: rel[0], reverse=False)
+ tbl_see.sort(key=lambda rel: rel[0], reverse=False)
+ tbl_see.sort(key=lambda rel: rel[-1], reverse=False)
+ tbl_delta.sort(key=lambda rel: rel[-1], reverse=True)
+
+ # Put the tables together:
+ table = list()
+ table.extend(tbl_new)
+ table.extend(tbl_see)
+ table.extend(tbl_delta)
+
+ return table
+
+
+def _tpc_generate_html_table(header, data, output_file_name):
+ """Generate html table from input data with simple sorting possibility.
+
+ :param header: Table header.
+ :param data: Input data to be included in the table. It is a list of lists.
+ Inner lists are rows in the table. All inner lists must be of the same
+ length. The length of these lists must be the same as the length of the
+ header.
+ :param output_file_name: The name (relative or full path) where the
+ generated html table is written.
+ :type header: list
+ :type data: list of lists
+ :type output_file_name: str
+ """
+
+ df = pd.DataFrame(data, columns=header)
+
+ df_sorted = [df.sort_values(
+ by=[key, header[0]], ascending=[True, True]
+ if key != header[0] else [False, True]) for key in header]
+ df_sorted_rev = [df.sort_values(
+ by=[key, header[0]], ascending=[False, True]
+ if key != header[0] else [True, True]) for key in header]
+ df_sorted.extend(df_sorted_rev)
+
+ fill_color = [["#d4e4f7" if idx % 2 else "#e9f1fb"
+ for idx in range(len(df))]]
+ table_header = dict(
+ values=["{item}".format(item=item) for item in header],
+ fill_color="#7eade7",
+ align=["left", "center"]
+ )
+
+ fig = go.Figure()
+
+ for table in df_sorted:
+ columns = [table.get(col) for col in header]
+ fig.add_trace(
+ go.Table(
+ columnwidth=[30, 10],
+ header=table_header,
+ cells=dict(
+ values=columns,
+ fill_color=fill_color,
+ align=["left", "right"]
+ )
+ )
+ )
+
+ buttons = list()
+ menu_items = ["{0} (ascending)".format(itm) for itm in header]
+ menu_items_rev = ["{0} (descending)".format(itm) for itm in header]
+ menu_items.extend(menu_items_rev)
+ for idx, hdr in enumerate(menu_items):
+ visible = [False, ] * len(menu_items)
+ visible[idx] = True
+ buttons.append(
+ dict(
+ label=hdr.replace(" [Mpps]", ""),
+ method="update",
+ args=[{"visible": visible}],
+ )
+ )
+
+ fig.update_layout(
+ updatemenus=[
+ go.layout.Updatemenu(
+ type="dropdown",
+ direction="down",
+ x=0.03,
+ xanchor="left",
+ y=1.045,
+ yanchor="top",
+ active=len(menu_items) - 1,
+ buttons=list(buttons)
+ )
+ ],
+ annotations=[
+ go.layout.Annotation(
+ text="Sort by:",
+ x=0,
+ xref="paper",
+ y=1.035,
+ yref="paper",
+ align="left",
+ showarrow=False
+ )
+ ]
+ )
+
+ ploff.plot(fig, show_link=False, auto_open=False, filename=output_file_name)
+
+
def table_performance_comparison(table, input_data):
"""Generate the table(s) with algorithm: table_performance_comparison
specified in the specification file.
@@ -210,9 +370,9 @@ def table_performance_comparison(table, input_data):
header = ["Test case", ]
if table["include-tests"] == "MRR":
- hdr_param = "Receive Rate"
+ hdr_param = "Rec Rate"
else:
- hdr_param = "Throughput"
+ hdr_param = "Thput"
history = table.get("history", None)
if history:
@@ -234,16 +394,12 @@ def table_performance_comparison(table, input_data):
# Prepare data to the table:
tbl_dict = dict()
+ topo = ""
for job, builds in table["reference"]["data"].items():
+ topo = "2n-skx" if "2n-skx" in job else ""
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
- tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
- replace("-ndrpdr", "").replace("-pdrdisc", "").\
- replace("-ndrdisc", "").replace("-pdr", "").\
- replace("-ndr", "").\
- replace("1t1c", "1c").replace("2t1c", "1c").\
- replace("2t2c", "2c").replace("4t2c", "2c").\
- replace("4t4c", "4c").replace("8t4c", "4c")
+ tst_name_mod = _tpc_modify_test_name(tst_name)
if "across topologies" in table["title"].lower():
tst_name_mod = tst_name_mod.replace("2n1l-", "")
if tbl_dict.get(tst_name_mod, None) is None:
@@ -253,47 +409,18 @@ def table_performance_comparison(table, input_data):
split("-")[:-1]))
if "across testbeds" in table["title"].lower() or \
"across topologies" in table["title"].lower():
- name = name.\
- replace("1t1c", "1c").replace("2t1c", "1c").\
- replace("2t2c", "2c").replace("4t2c", "2c").\
- replace("4t4c", "4c").replace("8t4c", "4c")
+ name = _tpc_modify_displayed_test_name(name)
tbl_dict[tst_name_mod] = {"name": name,
"ref-data": list(),
"cmp-data": list()}
- try:
- # TODO: Re-work when NDRPDRDISC tests are not used
- if table["include-tests"] == "MRR":
- tbl_dict[tst_name_mod]["ref-data"]. \
- append(tst_data["result"]["receive-rate"].avg)
- elif table["include-tests"] == "PDR":
- if tst_data["type"] == "PDR":
- tbl_dict[tst_name_mod]["ref-data"]. \
- append(tst_data["throughput"]["value"])
- elif tst_data["type"] == "NDRPDR":
- tbl_dict[tst_name_mod]["ref-data"].append(
- tst_data["throughput"]["PDR"]["LOWER"])
- elif table["include-tests"] == "NDR":
- if tst_data["type"] == "NDR":
- tbl_dict[tst_name_mod]["ref-data"]. \
- append(tst_data["throughput"]["value"])
- elif tst_data["type"] == "NDRPDR":
- tbl_dict[tst_name_mod]["ref-data"].append(
- tst_data["throughput"]["NDR"]["LOWER"])
- else:
- continue
- except TypeError:
- pass # No data in output.xml for this test
+ _tpc_insert_data(target=tbl_dict[tst_name_mod]["ref-data"],
+ src=tst_data,
+ include_tests=table["include-tests"])
for job, builds in table["compare"]["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
- tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
- replace("-ndrpdr", "").replace("-pdrdisc", ""). \
- replace("-ndrdisc", "").replace("-pdr", ""). \
- replace("-ndr", "").\
- replace("1t1c", "1c").replace("2t1c", "1c").\
- replace("2t2c", "2c").replace("4t2c", "2c").\
- replace("4t4c", "4c").replace("8t4c", "4c")
+ tst_name_mod = _tpc_modify_test_name(tst_name)
if "across topologies" in table["title"].lower():
tst_name_mod = tst_name_mod.replace("2n1l-", "")
if tbl_dict.get(tst_name_mod, None) is None:
@@ -303,64 +430,64 @@ def table_performance_comparison(table, input_data):
split("-")[:-1]))
if "across testbeds" in table["title"].lower() or \
"across topologies" in table["title"].lower():
- name = name.\
- replace("1t1c", "1c").replace("2t1c", "1c").\
- replace("2t2c", "2c").replace("4t2c", "2c").\
- replace("4t4c", "4c").replace("8t4c", "4c")
+ name = _tpc_modify_displayed_test_name(name)
tbl_dict[tst_name_mod] = {"name": name,
"ref-data": list(),
"cmp-data": list()}
- try:
- # TODO: Re-work when NDRPDRDISC tests are not used
- if table["include-tests"] == "MRR":
- tbl_dict[tst_name_mod]["cmp-data"]. \
- append(tst_data["result"]["receive-rate"].avg)
- elif table["include-tests"] == "PDR":
- if tst_data["type"] == "PDR":
- tbl_dict[tst_name_mod]["cmp-data"]. \
- append(tst_data["throughput"]["value"])
- elif tst_data["type"] == "NDRPDR":
- tbl_dict[tst_name_mod]["cmp-data"].append(
- tst_data["throughput"]["PDR"]["LOWER"])
- elif table["include-tests"] == "NDR":
- if tst_data["type"] == "NDR":
- tbl_dict[tst_name_mod]["cmp-data"]. \
- append(tst_data["throughput"]["value"])
- elif tst_data["type"] == "NDRPDR":
- tbl_dict[tst_name_mod]["cmp-data"].append(
- tst_data["throughput"]["NDR"]["LOWER"])
- else:
- continue
- except (KeyError, TypeError):
- pass
+ _tpc_insert_data(target=tbl_dict[tst_name_mod]["cmp-data"],
+ src=tst_data,
+ include_tests=table["include-tests"])
+
+ replacement = table["compare"].get("data-replacement", None)
+ if replacement:
+ create_new_list = True
+ rpl_data = input_data.filter_data(
+ table, data=replacement, continue_on_error=True)
+ for job, builds in replacement.items():
+ for build in builds:
+ for tst_name, tst_data in rpl_data[job][str(build)].iteritems():
+ tst_name_mod = _tpc_modify_test_name(tst_name)
+ if "across topologies" in table["title"].lower():
+ tst_name_mod = tst_name_mod.replace("2n1l-", "")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ name = "{0}".format("-".join(tst_data["name"].
+ split("-")[:-1]))
+ if "across testbeds" in table["title"].lower() or \
+ "across topologies" in table["title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ tbl_dict[tst_name_mod] = {"name": name,
+ "ref-data": list(),
+ "cmp-data": list()}
+ if create_new_list:
+ create_new_list = False
+ tbl_dict[tst_name_mod]["cmp-data"] = list()
+
+ _tpc_insert_data(target=tbl_dict[tst_name_mod]["cmp-data"],
+ src=tst_data,
+ include_tests=table["include-tests"])
+
if history:
for item in history:
for job, builds in item["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
- tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
- replace("-ndrpdr", "").replace("-pdrdisc", ""). \
- replace("-ndrdisc", "").replace("-pdr", ""). \
- replace("-ndr", "").\
- replace("1t1c", "1c").replace("2t1c", "1c").\
- replace("2t2c", "2c").replace("4t2c", "2c").\
- replace("4t4c", "4c").replace("8t4c", "4c")
+ tst_name_mod = _tpc_modify_test_name(tst_name)
if "across topologies" in table["title"].lower():
tst_name_mod = tst_name_mod.replace("2n1l-", "")
if tbl_dict.get(tst_name_mod, None) is None:
continue
if tbl_dict[tst_name_mod].get("history", None) is None:
tbl_dict[tst_name_mod]["history"] = OrderedDict()
- if tbl_dict[tst_name_mod]["history"].get(item["title"],
- None) is None:
+ if tbl_dict[tst_name_mod]["history"].\
+ get(item["title"], None) is None:
tbl_dict[tst_name_mod]["history"][item["title"]] = \
list()
try:
# TODO: Re-work when NDRPDRDISC tests are not used
if table["include-tests"] == "MRR":
- tbl_dict[tst_name_mod]["history"][item["title"
- ]].append(tst_data["result"]["receive-rate"].
- avg)
+ tbl_dict[tst_name_mod]["history"][item[
+ "title"]].append(tst_data["result"][
+ "receive-rate"].avg)
elif table["include-tests"] == "PDR":
if tst_data["type"] == "PDR":
tbl_dict[tst_name_mod]["history"][
@@ -369,7 +496,7 @@ def table_performance_comparison(table, input_data):
elif tst_data["type"] == "NDRPDR":
tbl_dict[tst_name_mod]["history"][item[
"title"]].append(tst_data["throughput"][
- "PDR"]["LOWER"])
+ "PDR"]["LOWER"])
elif table["include-tests"] == "NDR":
if tst_data["type"] == "NDR":
tbl_dict[tst_name_mod]["history"][
@@ -378,13 +505,14 @@ def table_performance_comparison(table, input_data):
elif tst_data["type"] == "NDRPDR":
tbl_dict[tst_name_mod]["history"][item[
"title"]].append(tst_data["throughput"][
- "NDR"]["LOWER"])
+ "NDR"]["LOWER"])
else:
continue
except (TypeError, KeyError):
pass
tbl_lst = list()
+ footnote = False
for tst_name in tbl_dict.keys():
item = [tbl_dict[tst_name]["name"], ]
if history:
@@ -394,32 +522,34 @@ def table_performance_comparison(table, input_data):
item.append(round(mean(hist_data) / 1000000, 2))
item.append(round(stdev(hist_data) / 1000000, 2))
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
data_t = tbl_dict[tst_name]["ref-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
data_t = tbl_dict[tst_name]["cmp-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
- item.extend([None, None])
- if "dot1q" in tbl_dict[tst_name]["name"]:
- item.append("Changed methodology")
- elif item[-4] is not None and item[-2] is not None and item[-4] != 0:
+ item.extend(["Not tested", "Not tested"])
+ if item[-2] == "Not tested":
+ pass
+ elif item[-4] == "Not tested":
+ item.append("New in CSIT-1908")
+ elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]:
+ item.append("See footnote [1]")
+ footnote = True
+ elif item[-4] != 0:
item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- else:
- item.append("n/a")
- if (len(item) == len(header)) and (item[-5] is not None):
+ if (len(item) == len(header)) and (item[-3] != "Not tested"):
tbl_lst.append(item)
- # Sort the table according to the relative change
- tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
+ tbl_lst = _tpc_sort_table(tbl_lst)
# Generate csv tables:
csv_file = "{0}.csv".format(table["output-file"])
@@ -428,7 +558,26 @@ def table_performance_comparison(table, input_data):
for test in tbl_lst:
file_handler.write(",".join([str(item) for item in test]) + "\n")
- convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
+ txt_file_name = "{0}.txt".format(table["output-file"])
+ convert_csv_to_pretty_txt(csv_file, txt_file_name)
+
+ if footnote:
+ with open(txt_file_name, 'a') as txt_file:
+ txt_file.writelines([
+ "\nFootnotes:\n",
+ "[1] CSIT-1908 changed test methodology of dot1q tests in "
+ "2-node testbeds, dot1q encapsulation is now used on both "
+ "links of SUT.\n",
+ " Previously dot1q was used only on a single link with the "
+ "other link carrying untagged Ethernet frames. This changes "
+ "results\n",
+ " in slightly lower throughput in CSIT-1908 for these "
+ "tests. See release notes."
+ ])
+
+ # Generate html table:
+ _tpc_generate_html_table(header, tbl_lst,
+ "{0}.html".format(table["output-file"]))
def table_performance_comparison_nic(table, input_data):
@@ -454,9 +603,9 @@ def table_performance_comparison_nic(table, input_data):
header = ["Test case", ]
if table["include-tests"] == "MRR":
- hdr_param = "Receive Rate"
+ hdr_param = "Rec Rate"
else:
- hdr_param = "Throughput"
+ hdr_param = "Thput"
history = table.get("history", None)
if history:
@@ -478,19 +627,14 @@ def table_performance_comparison_nic(table, input_data):
# Prepare data to the table:
tbl_dict = dict()
+ topo = ""
for job, builds in table["reference"]["data"].items():
+ topo = "2n-skx" if "2n-skx" in job else ""
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
if table["reference"]["nic"] not in tst_data["tags"]:
continue
- tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
- replace("-ndrpdr", "").replace("-pdrdisc", "").\
- replace("-ndrdisc", "").replace("-pdr", "").\
- replace("-ndr", "").\
- replace("1t1c", "1c").replace("2t1c", "1c").\
- replace("2t2c", "2c").replace("4t2c", "2c").\
- replace("4t4c", "4c").replace("8t4c", "4c")
- tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
+ tst_name_mod = _tpc_modify_test_name(tst_name)
if "across topologies" in table["title"].lower():
tst_name_mod = tst_name_mod.replace("2n1l-", "")
if tbl_dict.get(tst_name_mod, None) is None:
@@ -498,50 +642,20 @@ def table_performance_comparison_nic(table, input_data):
split("-")[:-1]))
if "across testbeds" in table["title"].lower() or \
"across topologies" in table["title"].lower():
- name = name.\
- replace("1t1c", "1c").replace("2t1c", "1c").\
- replace("2t2c", "2c").replace("4t2c", "2c").\
- replace("4t4c", "4c").replace("8t4c", "4c")
+ name = _tpc_modify_displayed_test_name(name)
tbl_dict[tst_name_mod] = {"name": name,
"ref-data": list(),
"cmp-data": list()}
- try:
- # TODO: Re-work when NDRPDRDISC tests are not used
- if table["include-tests"] == "MRR":
- tbl_dict[tst_name_mod]["ref-data"]. \
- append(tst_data["result"]["receive-rate"].avg)
- elif table["include-tests"] == "PDR":
- if tst_data["type"] == "PDR":
- tbl_dict[tst_name_mod]["ref-data"]. \
- append(tst_data["throughput"]["value"])
- elif tst_data["type"] == "NDRPDR":
- tbl_dict[tst_name_mod]["ref-data"].append(
- tst_data["throughput"]["PDR"]["LOWER"])
- elif table["include-tests"] == "NDR":
- if tst_data["type"] == "NDR":
- tbl_dict[tst_name_mod]["ref-data"]. \
- append(tst_data["throughput"]["value"])
- elif tst_data["type"] == "NDRPDR":
- tbl_dict[tst_name_mod]["ref-data"].append(
- tst_data["throughput"]["NDR"]["LOWER"])
- else:
- continue
- except TypeError:
- pass # No data in output.xml for this test
+ _tpc_insert_data(target=tbl_dict[tst_name_mod]["ref-data"],
+ src=tst_data,
+ include_tests=table["include-tests"])
for job, builds in table["compare"]["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
if table["compare"]["nic"] not in tst_data["tags"]:
continue
- tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
- replace("-ndrpdr", "").replace("-pdrdisc", ""). \
- replace("-ndrdisc", "").replace("-pdr", ""). \
- replace("-ndr", "").\
- replace("1t1c", "1c").replace("2t1c", "1c").\
- replace("2t2c", "2c").replace("4t2c", "2c").\
- replace("4t4c", "4c").replace("8t4c", "4c")
- tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
+ tst_name_mod = _tpc_modify_test_name(tst_name)
if "across topologies" in table["title"].lower():
tst_name_mod = tst_name_mod.replace("2n1l-", "")
if tbl_dict.get(tst_name_mod, None) is None:
@@ -549,36 +663,43 @@ def table_performance_comparison_nic(table, input_data):
split("-")[:-1]))
if "across testbeds" in table["title"].lower() or \
"across topologies" in table["title"].lower():
- name = name.\
- replace("1t1c", "1c").replace("2t1c", "1c").\
- replace("2t2c", "2c").replace("4t2c", "2c").\
- replace("4t4c", "4c").replace("8t4c", "4c")
+ name = _tpc_modify_displayed_test_name(name)
tbl_dict[tst_name_mod] = {"name": name,
"ref-data": list(),
"cmp-data": list()}
- try:
- # TODO: Re-work when NDRPDRDISC tests are not used
- if table["include-tests"] == "MRR":
- tbl_dict[tst_name_mod]["cmp-data"]. \
- append(tst_data["result"]["receive-rate"].avg)
- elif table["include-tests"] == "PDR":
- if tst_data["type"] == "PDR":
- tbl_dict[tst_name_mod]["cmp-data"]. \
- append(tst_data["throughput"]["value"])
- elif tst_data["type"] == "NDRPDR":
- tbl_dict[tst_name_mod]["cmp-data"].append(
- tst_data["throughput"]["PDR"]["LOWER"])
- elif table["include-tests"] == "NDR":
- if tst_data["type"] == "NDR":
- tbl_dict[tst_name_mod]["cmp-data"]. \
- append(tst_data["throughput"]["value"])
- elif tst_data["type"] == "NDRPDR":
- tbl_dict[tst_name_mod]["cmp-data"].append(
- tst_data["throughput"]["NDR"]["LOWER"])
- else:
+ _tpc_insert_data(target=tbl_dict[tst_name_mod]["cmp-data"],
+ src=tst_data,
+ include_tests=table["include-tests"])
+
+ replacement = table["compare"].get("data-replacement", None)
+ if replacement:
+ create_new_list = True
+ rpl_data = input_data.filter_data(
+ table, data=replacement, continue_on_error=True)
+ for job, builds in replacement.items():
+ for build in builds:
+ for tst_name, tst_data in rpl_data[job][str(build)].iteritems():
+ if table["compare"]["nic"] not in tst_data["tags"]:
continue
- except (KeyError, TypeError):
- pass
+ tst_name_mod = _tpc_modify_test_name(tst_name)
+ if "across topologies" in table["title"].lower():
+ tst_name_mod = tst_name_mod.replace("2n1l-", "")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ name = "{0}".format("-".join(tst_data["name"].
+ split("-")[:-1]))
+ if "across testbeds" in table["title"].lower() or \
+ "across topologies" in table["title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ tbl_dict[tst_name_mod] = {"name": name,
+ "ref-data": list(),
+ "cmp-data": list()}
+ if create_new_list:
+ create_new_list = False
+ tbl_dict[tst_name_mod]["cmp-data"] = list()
+
+ _tpc_insert_data(target=tbl_dict[tst_name_mod]["cmp-data"],
+ src=tst_data,
+ include_tests=table["include-tests"])
if history:
for item in history:
@@ -587,30 +708,23 @@ def table_performance_comparison_nic(table, input_data):
for tst_name, tst_data in data[job][str(build)].iteritems():
if item["nic"] not in tst_data["tags"]:
continue
- tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
- replace("-ndrpdr", "").replace("-pdrdisc", ""). \
- replace("-ndrdisc", "").replace("-pdr", ""). \
- replace("-ndr", "").\
- replace("1t1c", "1c").replace("2t1c", "1c").\
- replace("2t2c", "2c").replace("4t2c", "2c").\
- replace("4t4c", "4c").replace("8t4c", "4c")
- tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
+ tst_name_mod = _tpc_modify_test_name(tst_name)
if "across topologies" in table["title"].lower():
tst_name_mod = tst_name_mod.replace("2n1l-", "")
if tbl_dict.get(tst_name_mod, None) is None:
continue
if tbl_dict[tst_name_mod].get("history", None) is None:
tbl_dict[tst_name_mod]["history"] = OrderedDict()
- if tbl_dict[tst_name_mod]["history"].get(item["title"],
- None) is None:
+ if tbl_dict[tst_name_mod]["history"].\
+ get(item["title"], None) is None:
tbl_dict[tst_name_mod]["history"][item["title"]] = \
list()
try:
# TODO: Re-work when NDRPDRDISC tests are not used
if table["include-tests"] == "MRR":
- tbl_dict[tst_name_mod]["history"][item["title"
- ]].append(tst_data["result"]["receive-rate"].
- avg)
+ tbl_dict[tst_name_mod]["history"][item[
+ "title"]].append(tst_data["result"][
+ "receive-rate"].avg)
elif table["include-tests"] == "PDR":
if tst_data["type"] == "PDR":
tbl_dict[tst_name_mod]["history"][
@@ -619,7 +733,7 @@ def table_performance_comparison_nic(table, input_data):
elif tst_data["type"] == "NDRPDR":
tbl_dict[tst_name_mod]["history"][item[
"title"]].append(tst_data["throughput"][
- "PDR"]["LOWER"])
+ "PDR"]["LOWER"])
elif table["include-tests"] == "NDR":
if tst_data["type"] == "NDR":
tbl_dict[tst_name_mod]["history"][
@@ -628,13 +742,14 @@ def table_performance_comparison_nic(table, input_data):
elif tst_data["type"] == "NDRPDR":
tbl_dict[tst_name_mod]["history"][item[
"title"]].append(tst_data["throughput"][
- "NDR"]["LOWER"])
+ "NDR"]["LOWER"])
else:
continue
except (TypeError, KeyError):
pass
tbl_lst = list()
+ footnote = False
for tst_name in tbl_dict.keys():
item = [tbl_dict[tst_name]["name"], ]
if history:
@@ -644,32 +759,34 @@ def table_performance_comparison_nic(table, input_data):
item.append(round(mean(hist_data) / 1000000, 2))
item.append(round(stdev(hist_data) / 1000000, 2))
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
data_t = tbl_dict[tst_name]["ref-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
data_t = tbl_dict[tst_name]["cmp-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
- item.extend([None, None])
- if "dot1q" in tbl_dict[tst_name]["name"]:
- item.append("Changed methodology")
- elif item[-4] is not None and item[-2] is not None and item[-4] != 0:
+ item.extend(["Not tested", "Not tested"])
+ if item[-2] == "Not tested":
+ pass
+ elif item[-4] == "Not tested":
+ item.append("New in CSIT-1908")
+ elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]:
+ item.append("See footnote [1]")
+ footnote = True
+ elif item[-4] != 0:
item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- else:
- item.append("n/a")
- if (len(item) == len(header)) and (item[-5] is not None):
+ if (len(item) == len(header)) and (item[-3] != "Not tested"):
tbl_lst.append(item)
- # Sort the table according to the relative change
- tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
+ tbl_lst = _tpc_sort_table(tbl_lst)
# Generate csv tables:
csv_file = "{0}.csv".format(table["output-file"])
@@ -678,7 +795,26 @@ def table_performance_comparison_nic(table, input_data):
for test in tbl_lst:
file_handler.write(",".join([str(item) for item in test]) + "\n")
- convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
+ txt_file_name = "{0}.txt".format(table["output-file"])
+ convert_csv_to_pretty_txt(csv_file, txt_file_name)
+
+ if footnote:
+ with open(txt_file_name, 'a') as txt_file:
+ txt_file.writelines([
+ "\nFootnotes:\n",
+ "[1] CSIT-1908 changed test methodology of dot1q tests in "
+ "2-node testbeds, dot1q encapsulation is now used on both "
+ "links of SUT.\n",
+ " Previously dot1q was used only on a single link with the "
+ "other link carrying untagged Ethernet frames. This changes "
+ "results\n",
+ " in slightly lower throughput in CSIT-1908 for these "
+ "tests. See release notes."
+ ])
+
+ # Generate html table:
+ _tpc_generate_html_table(header, tbl_lst,
+ "{0}.html".format(table["output-file"]))
def table_nics_comparison(table, input_data):
@@ -704,9 +840,9 @@ def table_nics_comparison(table, input_data):
header = ["Test case", ]
if table["include-tests"] == "MRR":
- hdr_param = "Receive Rate"
+ hdr_param = "Rec Rate"
else:
- hdr_param = "Throughput"
+ hdr_param = "Thput"
header.extend(
["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
@@ -740,7 +876,7 @@ def table_nics_comparison(table, input_data):
"cmp-data": list()}
try:
if table["include-tests"] == "MRR":
- result = tst_data["result"]["receive-rate"].avg
+ result = tst_data["result"]["receive-rate"] # .avg
elif table["include-tests"] == "PDR":
result = tst_data["throughput"]["PDR"]["LOWER"]
elif table["include-tests"] == "NDR":
@@ -790,6 +926,10 @@ def table_nics_comparison(table, input_data):
convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
+ # Generate html table:
+ _tpc_generate_html_table(header, tbl_lst,
+ "{0}.html".format(table["output-file"]))
+
def table_soak_vs_ndr(table, input_data):
"""Generate the table(s) with algorithm: table_soak_vs_ndr
@@ -813,9 +953,9 @@ def table_soak_vs_ndr(table, input_data):
try:
header = [
"Test case",
- "{0} Throughput [Mpps]".format(table["reference"]["title"]),
+ "{0} Thput [Mpps]".format(table["reference"]["title"]),
"{0} Stdev [Mpps]".format(table["reference"]["title"]),
- "{0} Throughput [Mpps]".format(table["compare"]["title"]),
+ "{0} Thput [Mpps]".format(table["compare"]["title"]),
"{0} Stdev [Mpps]".format(table["compare"]["title"]),
"Delta [%]", "Stdev of delta [%]"]
header_str = ",".join(header) + "\n"
@@ -858,7 +998,7 @@ def table_soak_vs_ndr(table, input_data):
try:
if tst_data["type"] in ("NDRPDR", "MRR", "BMRR"):
if table["include-tests"] == "MRR":
- result = tst_data["result"]["receive-rate"].avg
+ result = tst_data["result"]["receive-rate"]
elif table["include-tests"] == "PDR":
result = tst_data["throughput"]["PDR"]["LOWER"]
elif table["include-tests"] == "NDR":
@@ -913,6 +1053,10 @@ def table_soak_vs_ndr(table, input_data):
convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
+ # Generate html table:
+ _tpc_generate_html_table(header, tbl_lst,
+ "{0}.html".format(table["output-file"]))
+
def table_performance_trending_dashboard(table, input_data):
"""Generate the table(s) with algorithm:
@@ -999,9 +1143,8 @@ def table_performance_trending_dashboard(table, input_data):
if classification_lst:
if isnan(rel_change_last) and isnan(rel_change_long):
continue
- if (isnan(last_avg) or
- isnan(rel_change_last) or
- isnan(rel_change_long)):
+ if isnan(last_avg) or isnan(rel_change_last) or \
+ isnan(rel_change_long):
continue
tbl_lst.append(
[tbl_dict[tst_name]["name"],
@@ -1298,14 +1441,22 @@ def table_last_failed_tests(table, input_data):
return
tbl_list.append(build)
tbl_list.append(version)
+ failed_tests = list()
+ passed = 0
+ failed = 0
for tst_name, tst_data in data[job][build].iteritems():
if tst_data["status"] != "FAIL":
+ passed += 1
continue
+ failed += 1
groups = re.search(REGEX_NIC, tst_data["parent"])
if not groups:
continue
nic = groups.group(0)
- tbl_list.append("{0}-{1}".format(nic, tst_data["name"]))
+ failed_tests.append("{0}-{1}".format(nic, tst_data["name"]))
+ tbl_list.append(str(passed))
+ tbl_list.append(str(failed))
+ tbl_list.extend(failed_tests)
file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
logging.info(" Writing file: '{0}'".format(file_name))
@@ -1380,6 +1531,9 @@ def table_failed_tests(table, input_data):
tbl_lst = list()
for tst_data in tbl_dict.values():
fails_nr = 0
+ fails_last_date = ""
+ fails_last_vpp = ""
+ fails_last_csit = ""
for val in tst_data["data"].values():
if val[0] == "FAIL":
fails_nr += 1