header = ["Test case", ]
if table["include-tests"] == "MRR":
- hdr_param = "Receive Rate"
+ hdr_param = "Rec Rate"
else:
- hdr_param = "Throughput"
+ hdr_param = "Thput"
history = table.get("history", None)
if history:
# Prepare data to the table:
tbl_dict = dict()
for job, builds in table["reference"]["data"].items():
+ topo = "2n-skx" if "2n-skx" in job else ""
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
replace("4t4c", "4c").replace("8t4c", "4c")
if "across topologies" in table["title"].lower():
tst_name_mod = tst_name_mod.replace("2n1l-", "")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ groups = re.search(REGEX_NIC, tst_data["parent"])
+ nic = groups.group(0) if groups else ""
+ name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
+ split("-")[:-1]))
+ if "across testbeds" in table["title"].lower() or \
+ "across topologies" in table["title"].lower():
+ name = name.\
+ replace("1t1c", "1c").replace("2t1c", "1c").\
+ replace("2t2c", "2c").replace("4t2c", "2c").\
+ replace("4t4c", "4c").replace("8t4c", "4c")
+ tbl_dict[tst_name_mod] = {"name": name,
+ "ref-data": list(),
+ "cmp-data": list()}
try:
# TODO: Re-work when NDRPDRDISC tests are not used
if table["include-tests"] == "MRR":
tst_data["throughput"]["NDR"]["LOWER"])
else:
continue
- except KeyError:
+ except (KeyError, TypeError):
pass
- except TypeError:
- tbl_dict.pop(tst_name_mod, None)
if history:
for item in history:
for job, builds in item["data"].items():
pass
tbl_lst = list()
+ footnote = False
for tst_name in tbl_dict.keys():
item = [tbl_dict[tst_name]["name"], ]
if history:
item.append(round(mean(hist_data) / 1000000, 2))
item.append(round(stdev(hist_data) / 1000000, 2))
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
data_t = tbl_dict[tst_name]["ref-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
data_t = tbl_dict[tst_name]["cmp-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
- item.extend([None, None])
- if item[-4] is not None and item[-2] is not None and item[-4] != 0:
+ item.extend(["Not tested", "Not tested"])
+ if item[-2] == "Not tested":
+ pass
+ elif item[-4] == "Not tested":
+ item.append("New in CSIT-1908")
+ elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]:
+ item.append("See footnote [1]")
+ footnote = True
+ elif item[-4] != 0:
item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- if len(item) == len(header):
+ if (len(item) == len(header)) and (item[-3] != "Not tested"):
tbl_lst.append(item)
# Sort the table according to the relative change
for test in tbl_lst:
file_handler.write(",".join([str(item) for item in test]) + "\n")
- convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
+ txt_file_name = "{0}.txt".format(table["output-file"])
+ convert_csv_to_pretty_txt(csv_file, txt_file_name)
+
+ if footnote:
+ with open(txt_file_name, 'a') as txt_file:
+ txt_file.writelines([
+ "\nFootnotes:\n",
+ "[1] CSIT-1908 changed test methodology of dot1q tests in "
+ "2n-skx testbeds, dot1q encapsulation is now used on both "
+ "links of SUT.\n",
+ " Previously dot1q was used only on a single link with the "
+ "other link carrying untagged Ethernet frames. This change "
+ "results\n",
+ " in slightly lower throughput in CSIT-1908 for these "
+ "tests. See release notes."
+ ])
def table_performance_comparison_nic(table, input_data):
header = ["Test case", ]
if table["include-tests"] == "MRR":
- hdr_param = "Receive Rate"
+ hdr_param = "Rec Rate"
else:
- hdr_param = "Throughput"
+ hdr_param = "Thput"
history = table.get("history", None)
if history:
# Prepare data to the table:
tbl_dict = dict()
for job, builds in table["reference"]["data"].items():
+ topo = "2n-skx" if "2n-skx" in job else ""
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
if table["reference"]["nic"] not in tst_data["tags"]:
tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
if "across topologies" in table["title"].lower():
tst_name_mod = tst_name_mod.replace("2n1l-", "")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ name = "{0}".format("-".join(tst_data["name"].
+ split("-")[:-1]))
+ if "across testbeds" in table["title"].lower() or \
+ "across topologies" in table["title"].lower():
+ name = name.\
+ replace("1t1c", "1c").replace("2t1c", "1c").\
+ replace("2t2c", "2c").replace("4t2c", "2c").\
+ replace("4t4c", "4c").replace("8t4c", "4c")
+ tbl_dict[tst_name_mod] = {"name": name,
+ "ref-data": list(),
+ "cmp-data": list()}
try:
# TODO: Re-work when NDRPDRDISC tests are not used
if table["include-tests"] == "MRR":
tst_data["throughput"]["NDR"]["LOWER"])
else:
continue
- except KeyError:
+ except (KeyError, TypeError):
pass
- except TypeError:
- tbl_dict.pop(tst_name_mod, None)
+
if history:
for item in history:
for job, builds in item["data"].items():
pass
tbl_lst = list()
+ footnote = False
for tst_name in tbl_dict.keys():
item = [tbl_dict[tst_name]["name"], ]
if history:
item.append(round(mean(hist_data) / 1000000, 2))
item.append(round(stdev(hist_data) / 1000000, 2))
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
data_t = tbl_dict[tst_name]["ref-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
- item.extend([None, None])
+ item.extend(["Not tested", "Not tested"])
data_t = tbl_dict[tst_name]["cmp-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
- item.extend([None, None])
- if item[-4] is not None and item[-2] is not None and item[-4] != 0:
+ item.extend(["Not tested", "Not tested"])
+ if item[-2] == "Not tested":
+ pass
+ elif item[-4] == "Not tested":
+ item.append("New in CSIT-1908")
+ elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]:
+ item.append("See footnote [1]")
+ footnote = True
+ elif item[-4] != 0:
item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- if len(item) == len(header):
+ if (len(item) == len(header)) and (item[-3] != "Not tested"):
tbl_lst.append(item)
# Sort the table according to the relative change
for test in tbl_lst:
file_handler.write(",".join([str(item) for item in test]) + "\n")
- convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
+ txt_file_name = "{0}.txt".format(table["output-file"])
+ convert_csv_to_pretty_txt(csv_file, txt_file_name)
+
+ if footnote:
+ with open(txt_file_name, 'a') as txt_file:
+ txt_file.writelines([
+ "\nFootnotes:\n",
+ "[1] CSIT-1908 changed test methodology of dot1q tests in "
+ "2n-skx testbeds, dot1q encapsulation is now used on both "
+ "links of SUT.\n",
+ " Previously dot1q was used only on a single link with the "
+ "other link carrying untagged Ethernet frames. This change "
+ "results\n",
+ " in slightly lower throughput in CSIT-1908 for these "
+ "tests. See release notes."
+ ])
def table_nics_comparison(table, input_data):
header = ["Test case", ]
if table["include-tests"] == "MRR":
- hdr_param = "Receive Rate"
+ hdr_param = "Rec Rate"
else:
- hdr_param = "Throughput"
+ hdr_param = "Thput"
header.extend(
["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
try:
header = [
"Test case",
- "{0} Throughput [Mpps]".format(table["reference"]["title"]),
+ "{0} Thput [Mpps]".format(table["reference"]["title"]),
"{0} Stdev [Mpps]".format(table["reference"]["title"]),
- "{0} Throughput [Mpps]".format(table["compare"]["title"]),
+ "{0} Thput [Mpps]".format(table["compare"]["title"]),
"{0} Stdev [Mpps]".format(table["compare"]["title"]),
"Delta [%]", "Stdev of delta [%]"]
header_str = ",".join(header) + "\n"
Code Review / csit.git / blobdiff