- # Prepare data to the table:
- tbl_dict = dict()
- topo = ""
- for job, builds in table["reference"]["data"].items():
- topo = "2n-skx" if "2n-skx" in job else ""
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].iteritems():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if "across topologies" in table["title"].lower():
- tst_name_mod = tst_name_mod.replace("2n1l-", "")
- if tbl_dict.get(tst_name_mod, None) is None:
- groups = re.search(REGEX_NIC, tst_data["parent"])
- nic = groups.group(0) if groups else ""
- name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
- split("-")[:-1]))
- if "across testbeds" in table["title"].lower() or \
- "across topologies" in table["title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {"name": name,
- "ref-data": list(),
- "cmp-data": list()}
- _tpc_insert_data(target=tbl_dict[tst_name_mod]["ref-data"],
- src=tst_data,
- include_tests=table["include-tests"])
-
- for job, builds in table["compare"]["data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].iteritems():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if "across topologies" in table["title"].lower():
- tst_name_mod = tst_name_mod.replace("2n1l-", "")
- if tbl_dict.get(tst_name_mod, None) is None:
- groups = re.search(REGEX_NIC, tst_data["parent"])
- nic = groups.group(0) if groups else ""
- name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
- split("-")[:-1]))
- if "across testbeds" in table["title"].lower() or \
- "across topologies" in table["title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {"name": name,
- "ref-data": list(),
- "cmp-data": list()}
- _tpc_insert_data(target=tbl_dict[tst_name_mod]["cmp-data"],
- src=tst_data,
- include_tests=table["include-tests"])
-
- replacement = table["compare"].get("data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].iteritems():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if "across topologies" in table["title"].lower():
- tst_name_mod = tst_name_mod.replace("2n1l-", "")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = "{0}".format("-".join(tst_data["name"].
- split("-")[:-1]))
- if "across testbeds" in table["title"].lower() or \
- "across topologies" in table["title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {"name": name,
- "ref-data": list(),
- "cmp-data": list()}
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod]["cmp-data"] = list()
-
- _tpc_insert_data(target=tbl_dict[tst_name_mod]["cmp-data"],
- src=tst_data,
- include_tests=table["include-tests"])
-
- if history:
- for item in history:
- for job, builds in item["data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].iteritems():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if "across topologies" in table["title"].lower():
- tst_name_mod = tst_name_mod.replace("2n1l-", "")
- if tbl_dict.get(tst_name_mod, None) is None:
- continue
- if tbl_dict[tst_name_mod].get("history", None) is None:
- tbl_dict[tst_name_mod]["history"] = OrderedDict()
- if tbl_dict[tst_name_mod]["history"].\
- get(item["title"], None) is None:
- tbl_dict[tst_name_mod]["history"][item["title"]] = \
- list()
- try:
- # TODO: Re-work when NDRPDRDISC tests are not used
- if table["include-tests"] == "MRR":
- tbl_dict[tst_name_mod]["history"][item[
- "title"]].append(tst_data["result"][
- "receive-rate"].avg)
- elif table["include-tests"] == "PDR":
- if tst_data["type"] == "PDR":
- tbl_dict[tst_name_mod]["history"][
- item["title"]].\
- append(tst_data["throughput"]["value"])
- elif tst_data["type"] == "NDRPDR":
- tbl_dict[tst_name_mod]["history"][item[
- "title"]].append(tst_data["throughput"][
- "PDR"]["LOWER"])
- elif table["include-tests"] == "NDR":
- if tst_data["type"] == "NDR":
- tbl_dict[tst_name_mod]["history"][
- item["title"]].\
- append(tst_data["throughput"]["value"])
- elif tst_data["type"] == "NDRPDR":
- tbl_dict[tst_name_mod]["history"][item[
- "title"]].append(tst_data["throughput"][
- "NDR"]["LOWER"])
- else:
- continue
- except (TypeError, KeyError):
- pass
-
- tbl_lst = list()
- footnote = False
- for tst_name in tbl_dict.keys():
- item = [tbl_dict[tst_name]["name"], ]
- if history:
- if tbl_dict[tst_name].get("history", None) is not None:
- for hist_data in tbl_dict[tst_name]["history"].values():
- if hist_data:
- item.append(round(mean(hist_data) / 1000000, 2))
- item.append(round(stdev(hist_data) / 1000000, 2))
- else:
- item.extend(["Not tested", "Not tested"])
- else:
- item.extend(["Not tested", "Not tested"])
- data_t = tbl_dict[tst_name]["ref-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend(["Not tested", "Not tested"])
- data_t = tbl_dict[tst_name]["cmp-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend(["Not tested", "Not tested"])
- if item[-2] == "Not tested":
- pass
- elif item[-4] == "Not tested":
- item.append("New in CSIT-1908")
- elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]:
- item.append("See footnote [1]")
- footnote = True
- elif item[-4] != 0:
- item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- if (len(item) == len(header)) and (item[-3] != "Not tested"):
- tbl_lst.append(item)
-
- tbl_lst = _tpc_sort_table(tbl_lst)
-
- # Generate csv tables:
- csv_file = "{0}.csv".format(table["output-file"])
- with open(csv_file, "w") as file_handler:
- file_handler.write(header_str)
- for test in tbl_lst:
- file_handler.write(",".join([str(item) for item in test]) + "\n")
-
- txt_file_name = "{0}.txt".format(table["output-file"])
- convert_csv_to_pretty_txt(csv_file, txt_file_name)
-
- if footnote:
- with open(txt_file_name, 'a') as txt_file:
- txt_file.writelines([
- "\nFootnotes:\n",
- "[1] CSIT-1908 changed test methodology of dot1q tests in "
- "2-node testbeds, dot1q encapsulation is now used on both "
- "links of SUT.\n",
- " Previously dot1q was used only on a single link with the "
- "other link carrying untagged Ethernet frames. This changes "
- "results\n",
- " in slightly lower throughput in CSIT-1908 for these "
- "tests. See release notes."
- ])
-
- # Generate html table:
- _tpc_generate_html_table(header, tbl_lst,
- "{0}.html".format(table["output-file"]))
-
-
-def table_performance_comparison_nic(table, input_data):
- """Generate the table(s) with algorithm: table_performance_comparison
- specified in the specification file.
-
- :param table: Table to generate.
- :param input_data: Data to process.
- :type table: pandas.Series
- :type input_data: InputData
- """
-
- logging.info(" Generating the table {0} ...".
- format(table.get("title", "")))
-
- # Transform the data
- logging.info(" Creating the data set for the {0} '{1}'.".
- format(table.get("type", ""), table.get("title", "")))
- data = input_data.filter_data(table, continue_on_error=True)
-
- # Prepare the header of the tables
- try:
- header = ["Test case", ]
-
- if table["include-tests"] == "MRR":
- hdr_param = "Rec Rate"
- else:
- hdr_param = "Thput"
-
- history = table.get("history", None)
- if history:
- for item in history:
- header.extend(
- ["{0} {1} [Mpps]".format(item["title"], hdr_param),
- "{0} Stdev [Mpps]".format(item["title"])])
- header.extend(
- ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
- "{0} Stdev [Mpps]".format(table["reference"]["title"]),
- "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
- "{0} Stdev [Mpps]".format(table["compare"]["title"]),
- "Delta [%]"])
- header_str = ",".join(header) + "\n"
- except (AttributeError, KeyError) as err:
- logging.error("The model is invalid, missing parameter: {0}".
- format(err))
- return
-
- # Prepare data to the table:
- tbl_dict = dict()
- topo = ""
- for job, builds in table["reference"]["data"].items():
- topo = "2n-skx" if "2n-skx" in job else ""
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].iteritems():
- if table["reference"]["nic"] not in tst_data["tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if "across topologies" in table["title"].lower():
- tst_name_mod = tst_name_mod.replace("2n1l-", "")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = "{0}".format("-".join(tst_data["name"].
- split("-")[:-1]))
- if "across testbeds" in table["title"].lower() or \
- "across topologies" in table["title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {"name": name,
- "ref-data": list(),
- "cmp-data": list()}
- _tpc_insert_data(target=tbl_dict[tst_name_mod]["ref-data"],
- src=tst_data,
- include_tests=table["include-tests"])
-
- for job, builds in table["compare"]["data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].iteritems():
- if table["compare"]["nic"] not in tst_data["tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if "across topologies" in table["title"].lower():
- tst_name_mod = tst_name_mod.replace("2n1l-", "")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = "{0}".format("-".join(tst_data["name"].
- split("-")[:-1]))
- if "across testbeds" in table["title"].lower() or \
- "across topologies" in table["title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {"name": name,
- "ref-data": list(),
- "cmp-data": list()}
- _tpc_insert_data(target=tbl_dict[tst_name_mod]["cmp-data"],
- src=tst_data,
- include_tests=table["include-tests"])
-
- replacement = table["compare"].get("data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].iteritems():
- if table["compare"]["nic"] not in tst_data["tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if "across topologies" in table["title"].lower():
- tst_name_mod = tst_name_mod.replace("2n1l-", "")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = "{0}".format("-".join(tst_data["name"].
- split("-")[:-1]))
- if "across testbeds" in table["title"].lower() or \
- "across topologies" in table["title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {"name": name,
- "ref-data": list(),
- "cmp-data": list()}
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod]["cmp-data"] = list()
-
- _tpc_insert_data(target=tbl_dict[tst_name_mod]["cmp-data"],
- src=tst_data,
- include_tests=table["include-tests"])
-
- if history:
- for item in history:
- for job, builds in item["data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].iteritems():
- if item["nic"] not in tst_data["tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if "across topologies" in table["title"].lower():
- tst_name_mod = tst_name_mod.replace("2n1l-", "")
- if tbl_dict.get(tst_name_mod, None) is None:
- continue
- if tbl_dict[tst_name_mod].get("history", None) is None:
- tbl_dict[tst_name_mod]["history"] = OrderedDict()
- if tbl_dict[tst_name_mod]["history"].\
- get(item["title"], None) is None:
- tbl_dict[tst_name_mod]["history"][item["title"]] = \
- list()
- try:
- # TODO: Re-work when NDRPDRDISC tests are not used
- if table["include-tests"] == "MRR":
- tbl_dict[tst_name_mod]["history"][item[
- "title"]].append(tst_data["result"][
- "receive-rate"].avg)
- elif table["include-tests"] == "PDR":
- if tst_data["type"] == "PDR":
- tbl_dict[tst_name_mod]["history"][
- item["title"]].\
- append(tst_data["throughput"]["value"])
- elif tst_data["type"] == "NDRPDR":
- tbl_dict[tst_name_mod]["history"][item[
- "title"]].append(tst_data["throughput"][
- "PDR"]["LOWER"])
- elif table["include-tests"] == "NDR":
- if tst_data["type"] == "NDR":
- tbl_dict[tst_name_mod]["history"][
- item["title"]].\
- append(tst_data["throughput"]["value"])
- elif tst_data["type"] == "NDRPDR":
- tbl_dict[tst_name_mod]["history"][item[
- "title"]].append(tst_data["throughput"][
- "NDR"]["LOWER"])
- else:
- continue
- except (TypeError, KeyError):
- pass
-
- tbl_lst = list()
- footnote = False
- for tst_name in tbl_dict.keys():
- item = [tbl_dict[tst_name]["name"], ]
- if history:
- if tbl_dict[tst_name].get("history", None) is not None:
- for hist_data in tbl_dict[tst_name]["history"].values():
- if hist_data:
- item.append(round(mean(hist_data) / 1000000, 2))
- item.append(round(stdev(hist_data) / 1000000, 2))
- else:
- item.extend(["Not tested", "Not tested"])
- else:
- item.extend(["Not tested", "Not tested"])
- data_t = tbl_dict[tst_name]["ref-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend(["Not tested", "Not tested"])
- data_t = tbl_dict[tst_name]["cmp-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend(["Not tested", "Not tested"])
- if item[-2] == "Not tested":
- pass
- elif item[-4] == "Not tested":
- item.append("New in CSIT-1908")
- elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]:
- item.append("See footnote [1]")
- footnote = True
- elif item[-4] != 0:
- item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- if (len(item) == len(header)) and (item[-3] != "Not tested"):
- tbl_lst.append(item)
-
- tbl_lst = _tpc_sort_table(tbl_lst)
-
- # Generate csv tables:
- csv_file = "{0}.csv".format(table["output-file"])
- with open(csv_file, "w") as file_handler:
- file_handler.write(header_str)
- for test in tbl_lst:
- file_handler.write(",".join([str(item) for item in test]) + "\n")
-
- txt_file_name = "{0}.txt".format(table["output-file"])
- convert_csv_to_pretty_txt(csv_file, txt_file_name)
-
- if footnote:
- with open(txt_file_name, 'a') as txt_file:
- txt_file.writelines([
- "\nFootnotes:\n",
- "[1] CSIT-1908 changed test methodology of dot1q tests in "
- "2-node testbeds, dot1q encapsulation is now used on both "
- "links of SUT.\n",
- " Previously dot1q was used only on a single link with the "
- "other link carrying untagged Ethernet frames. This changes "
- "results\n",
- " in slightly lower throughput in CSIT-1908 for these "
- "tests. See release notes."
- ])
-
- # Generate html table:
- _tpc_generate_html_table(header, tbl_lst,
- "{0}.html".format(table["output-file"]))
-
-
-def table_nics_comparison(table, input_data):
- """Generate the table(s) with algorithm: table_nics_comparison
- specified in the specification file.
-
- :param table: Table to generate.
- :param input_data: Data to process.
- :type table: pandas.Series
- :type input_data: InputData
- """
-
- logging.info(" Generating the table {0} ...".
- format(table.get("title", "")))
-
- # Transform the data
- logging.info(" Creating the data set for the {0} '{1}'.".
- format(table.get("type", ""), table.get("title", "")))
- data = input_data.filter_data(table, continue_on_error=True)
-
- # Prepare the header of the tables
- try:
- header = ["Test case", ]
-
- if table["include-tests"] == "MRR":
- hdr_param = "Rec Rate"
- else:
- hdr_param = "Thput"
-
- header.extend(
- ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
- "{0} Stdev [Mpps]".format(table["reference"]["title"]),
- "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
- "{0} Stdev [Mpps]".format(table["compare"]["title"]),
- "Delta [%]"])
- header_str = ",".join(header) + "\n"
- except (AttributeError, KeyError) as err:
- logging.error("The model is invalid, missing parameter: {0}".
- format(err))
- return
-
- # Prepare data to the table:
- tbl_dict = dict()
- for job, builds in table["data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].iteritems():
- tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
- replace("-ndrpdr", "").replace("-pdrdisc", "").\
- replace("-ndrdisc", "").replace("-pdr", "").\
- replace("-ndr", "").\
- replace("1t1c", "1c").replace("2t1c", "1c").\
- replace("2t2c", "2c").replace("4t2c", "2c").\
- replace("4t4c", "4c").replace("8t4c", "4c")
- tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
- if tbl_dict.get(tst_name_mod, None) is None:
- name = "-".join(tst_data["name"].split("-")[:-1])
- tbl_dict[tst_name_mod] = {"name": name,
- "ref-data": list(),
- "cmp-data": list()}
- try:
- if table["include-tests"] == "MRR":
- result = tst_data["result"]["receive-rate"] # .avg
- elif table["include-tests"] == "PDR":
- result = tst_data["throughput"]["PDR"]["LOWER"]
- elif table["include-tests"] == "NDR":
- result = tst_data["throughput"]["NDR"]["LOWER"]
- else:
- result = None
-
- if result:
- if table["reference"]["nic"] in tst_data["tags"]:
- tbl_dict[tst_name_mod]["ref-data"].append(result)
- elif table["compare"]["nic"] in tst_data["tags"]:
- tbl_dict[tst_name_mod]["cmp-data"].append(result)
- except (TypeError, KeyError) as err:
- logging.debug("No data for {0}".format(tst_name))
- logging.debug(repr(err))
- # No data in output.xml for this test
-
- tbl_lst = list()
- for tst_name in tbl_dict.keys():
- item = [tbl_dict[tst_name]["name"], ]
- data_t = tbl_dict[tst_name]["ref-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend([None, None])
- data_t = tbl_dict[tst_name]["cmp-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend([None, None])
- if item[-4] is not None and item[-2] is not None and item[-4] != 0:
- item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- if len(item) == len(header):
- tbl_lst.append(item)
-
- # Sort the table according to the relative change
- tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
-
- # Generate csv tables:
- csv_file = "{0}.csv".format(table["output-file"])
- with open(csv_file, "w") as file_handler:
- file_handler.write(header_str)
- for test in tbl_lst:
- file_handler.write(",".join([str(item) for item in test]) + "\n")
-
- convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))