X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=20779dde25f5e2bbde724482e320e07adcef6150;hp=887ee727974475bf77a8370cffc99f84fad930b4;hb=d75974b805248dd484876b7e196fdc7475e5999b;hpb=019a796394d6b8484225fe133c24c0e502b219f6 diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 887ee72797..20779dde25 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -27,7 +27,7 @@ from datetime import datetime as dt from datetime import timedelta from utils import mean, stdev, relative_change, classify_anomalies, \ - convert_csv_to_pretty_txt + convert_csv_to_pretty_txt, relative_change_stdev REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*') @@ -210,9 +210,9 @@ def table_performance_comparison(table, input_data): header = ["Test case", ] if table["include-tests"] == "MRR": - hdr_param = "Receive Rate" + hdr_param = "Rec Rate" else: - hdr_param = "Throughput" + hdr_param = "Thput" history = table.get("history", None) if history: @@ -235,6 +235,7 @@ def table_performance_comparison(table, input_data): # Prepare data to the table: tbl_dict = dict() for job, builds in table["reference"]["data"].items(): + topo = "2n-skx" if "2n-skx" in job else "" for build in builds: for tst_name, tst_data in data[job][str(build)].iteritems(): tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\ @@ -296,6 +297,20 @@ def table_performance_comparison(table, input_data): replace("4t4c", "4c").replace("8t4c", "4c") if "across topologies" in table["title"].lower(): tst_name_mod = tst_name_mod.replace("2n1l-", "") + if tbl_dict.get(tst_name_mod, None) is None: + groups = re.search(REGEX_NIC, tst_data["parent"]) + nic = groups.group(0) if groups else "" + name = "{0}-{1}".format(nic, "-".join(tst_data["name"]. + split("-")[:-1])) + if "across testbeds" in table["title"].lower() or \ + "across topologies" in table["title"].lower(): + name = name.\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + tbl_dict[tst_name_mod] = {"name": name, + "ref-data": list(), + "cmp-data": list()} try: # TODO: Re-work when NDRPDRDISC tests are not used if table["include-tests"] == "MRR": @@ -317,15 +332,282 @@ def table_performance_comparison(table, input_data): tst_data["throughput"]["NDR"]["LOWER"]) else: continue - except KeyError: + except (KeyError, TypeError): pass + if history: + for item in history: + for job, builds in item["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \ + replace("-ndrpdr", "").replace("-pdrdisc", ""). \ + replace("-ndrdisc", "").replace("-pdr", ""). \ + replace("-ndr", "").\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + if "across topologies" in table["title"].lower(): + tst_name_mod = tst_name_mod.replace("2n1l-", "") + if tbl_dict.get(tst_name_mod, None) is None: + continue + if tbl_dict[tst_name_mod].get("history", None) is None: + tbl_dict[tst_name_mod]["history"] = OrderedDict() + if tbl_dict[tst_name_mod]["history"].get(item["title"], + None) is None: + tbl_dict[tst_name_mod]["history"][item["title"]] = \ + list() + try: + # TODO: Re-work when NDRPDRDISC tests are not used + if table["include-tests"] == "MRR": + tbl_dict[tst_name_mod]["history"][item["title" + ]].append(tst_data["result"]["receive-rate"]. + avg) + elif table["include-tests"] == "PDR": + if tst_data["type"] == "PDR": + tbl_dict[tst_name_mod]["history"][ + item["title"]].\ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["history"][item[ + "title"]].append(tst_data["throughput"][ + "PDR"]["LOWER"]) + elif table["include-tests"] == "NDR": + if tst_data["type"] == "NDR": + tbl_dict[tst_name_mod]["history"][ + item["title"]].\ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["history"][item[ + "title"]].append(tst_data["throughput"][ + "NDR"]["LOWER"]) + else: + continue + except (TypeError, KeyError): + pass + + tbl_lst = list() + footnote = False + for tst_name in tbl_dict.keys(): + item = [tbl_dict[tst_name]["name"], ] + if history: + if tbl_dict[tst_name].get("history", None) is not None: + for hist_data in tbl_dict[tst_name]["history"].values(): + if hist_data: + item.append(round(mean(hist_data) / 1000000, 2)) + item.append(round(stdev(hist_data) / 1000000, 2)) + else: + item.extend(["Not tested", "Not tested"]) + else: + item.extend(["Not tested", "Not tested"]) + data_t = tbl_dict[tst_name]["ref-data"] + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend(["Not tested", "Not tested"]) + data_t = tbl_dict[tst_name]["cmp-data"] + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend(["Not tested", "Not tested"]) + if item[-2] == "Not tested": + pass + elif item[-4] == "Not tested": + item.append("New in CSIT-1908") + elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]: + item.append("See footnote [1]") + footnote = True + elif item[-4] != 0: + item.append(int(relative_change(float(item[-4]), float(item[-2])))) + if (len(item) == len(header)) and (item[-3] != "Not tested"): + tbl_lst.append(item) + + # Sort the table according to the relative change + tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) + + # Generate csv tables: + csv_file = "{0}.csv".format(table["output-file"]) + with open(csv_file, "w") as file_handler: + file_handler.write(header_str) + for test in tbl_lst: + file_handler.write(",".join([str(item) for item in test]) + "\n") + + txt_file_name = "{0}.txt".format(table["output-file"]) + convert_csv_to_pretty_txt(csv_file, txt_file_name) + + if footnote: + with open(txt_file_name, 'a') as txt_file: + txt_file.writelines([ + "\nFootnotes:\n", + "[1] CSIT-1908 changed test methodology of dot1q tests in " + "2n-skx testbeds, dot1q encapsulation is now used on both " + "links of SUT.\n", + " Previously dot1q was used only on a single link with the " + "other link carrying untagged Ethernet frames. This change " + "results\n", + " in slightly lower throughput in CSIT-1908 for these " + "tests. See release notes." + ]) + + +def table_performance_comparison_nic(table, input_data): + """Generate the table(s) with algorithm: table_performance_comparison + specified in the specification file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData + """ + + logging.info(" Generating the table {0} ...". + format(table.get("title", ""))) + + # Transform the data + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) + data = input_data.filter_data(table, continue_on_error=True) + + # Prepare the header of the tables + try: + header = ["Test case", ] + + if table["include-tests"] == "MRR": + hdr_param = "Rec Rate" + else: + hdr_param = "Thput" + + history = table.get("history", None) + if history: + for item in history: + header.extend( + ["{0} {1} [Mpps]".format(item["title"], hdr_param), + "{0} Stdev [Mpps]".format(item["title"])]) + header.extend( + ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param), + "{0} Stdev [Mpps]".format(table["reference"]["title"]), + "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param), + "{0} Stdev [Mpps]".format(table["compare"]["title"]), + "Delta [%]"]) + header_str = ",".join(header) + "\n" + except (AttributeError, KeyError) as err: + logging.error("The model is invalid, missing parameter: {0}". + format(err)) + return + + # Prepare data to the table: + tbl_dict = dict() + for job, builds in table["reference"]["data"].items(): + topo = "2n-skx" if "2n-skx" in job else "" + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + if table["reference"]["nic"] not in tst_data["tags"]: + continue + tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\ + replace("-ndrpdr", "").replace("-pdrdisc", "").\ + replace("-ndrdisc", "").replace("-pdr", "").\ + replace("-ndr", "").\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod) + if "across topologies" in table["title"].lower(): + tst_name_mod = tst_name_mod.replace("2n1l-", "") + if tbl_dict.get(tst_name_mod, None) is None: + name = "{0}".format("-".join(tst_data["name"]. + split("-")[:-1])) + if "across testbeds" in table["title"].lower() or \ + "across topologies" in table["title"].lower(): + name = name.\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + tbl_dict[tst_name_mod] = {"name": name, + "ref-data": list(), + "cmp-data": list()} + try: + # TODO: Re-work when NDRPDRDISC tests are not used + if table["include-tests"] == "MRR": + tbl_dict[tst_name_mod]["ref-data"]. \ + append(tst_data["result"]["receive-rate"].avg) + elif table["include-tests"] == "PDR": + if tst_data["type"] == "PDR": + tbl_dict[tst_name_mod]["ref-data"]. \ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["ref-data"].append( + tst_data["throughput"]["PDR"]["LOWER"]) + elif table["include-tests"] == "NDR": + if tst_data["type"] == "NDR": + tbl_dict[tst_name_mod]["ref-data"]. \ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["ref-data"].append( + tst_data["throughput"]["NDR"]["LOWER"]) + else: + continue except TypeError: - tbl_dict.pop(tst_name_mod, None) + pass # No data in output.xml for this test + + for job, builds in table["compare"]["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + if table["compare"]["nic"] not in tst_data["tags"]: + continue + tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \ + replace("-ndrpdr", "").replace("-pdrdisc", ""). \ + replace("-ndrdisc", "").replace("-pdr", ""). \ + replace("-ndr", "").\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod) + if "across topologies" in table["title"].lower(): + tst_name_mod = tst_name_mod.replace("2n1l-", "") + if tbl_dict.get(tst_name_mod, None) is None: + name = "{0}".format("-".join(tst_data["name"]. + split("-")[:-1])) + if "across testbeds" in table["title"].lower() or \ + "across topologies" in table["title"].lower(): + name = name.\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + tbl_dict[tst_name_mod] = {"name": name, + "ref-data": list(), + "cmp-data": list()} + try: + # TODO: Re-work when NDRPDRDISC tests are not used + if table["include-tests"] == "MRR": + tbl_dict[tst_name_mod]["cmp-data"]. \ + append(tst_data["result"]["receive-rate"].avg) + elif table["include-tests"] == "PDR": + if tst_data["type"] == "PDR": + tbl_dict[tst_name_mod]["cmp-data"]. \ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["cmp-data"].append( + tst_data["throughput"]["PDR"]["LOWER"]) + elif table["include-tests"] == "NDR": + if tst_data["type"] == "NDR": + tbl_dict[tst_name_mod]["cmp-data"]. \ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["cmp-data"].append( + tst_data["throughput"]["NDR"]["LOWER"]) + else: + continue + except (KeyError, TypeError): + pass + if history: for item in history: for job, builds in item["data"].items(): for build in builds: for tst_name, tst_data in data[job][str(build)].iteritems(): + if item["nic"] not in tst_data["tags"]: + continue tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \ replace("-ndrpdr", "").replace("-pdrdisc", ""). \ replace("-ndrdisc", "").replace("-pdr", ""). \ @@ -333,6 +615,7 @@ def table_performance_comparison(table, input_data): replace("1t1c", "1c").replace("2t1c", "1c").\ replace("2t2c", "2c").replace("4t2c", "2c").\ replace("4t4c", "4c").replace("8t4c", "4c") + tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod) if "across topologies" in table["title"].lower(): tst_name_mod = tst_name_mod.replace("2n1l-", "") if tbl_dict.get(tst_name_mod, None) is None: @@ -373,6 +656,7 @@ def table_performance_comparison(table, input_data): pass tbl_lst = list() + footnote = False for tst_name in tbl_dict.keys(): item = [tbl_dict[tst_name]["name"], ] if history: @@ -382,24 +666,31 @@ def table_performance_comparison(table, input_data): item.append(round(mean(hist_data) / 1000000, 2)) item.append(round(stdev(hist_data) / 1000000, 2)) else: - item.extend([None, None]) + item.extend(["Not tested", "Not tested"]) else: - item.extend([None, None]) + item.extend(["Not tested", "Not tested"]) data_t = tbl_dict[tst_name]["ref-data"] if data_t: item.append(round(mean(data_t) / 1000000, 2)) item.append(round(stdev(data_t) / 1000000, 2)) else: - item.extend([None, None]) + item.extend(["Not tested", "Not tested"]) data_t = tbl_dict[tst_name]["cmp-data"] if data_t: item.append(round(mean(data_t) / 1000000, 2)) item.append(round(stdev(data_t) / 1000000, 2)) else: - item.extend([None, None]) - if item[-4] is not None and item[-2] is not None and item[-4] != 0: + item.extend(["Not tested", "Not tested"]) + if item[-2] == "Not tested": + pass + elif item[-4] == "Not tested": + item.append("New in CSIT-1908") + elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]: + item.append("See footnote [1]") + footnote = True + elif item[-4] != 0: item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if len(item) == len(header): + if (len(item) == len(header)) and (item[-3] != "Not tested"): tbl_lst.append(item) # Sort the table according to the relative change @@ -412,7 +703,22 @@ def table_performance_comparison(table, input_data): for test in tbl_lst: file_handler.write(",".join([str(item) for item in test]) + "\n") - convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"])) + txt_file_name = "{0}.txt".format(table["output-file"]) + convert_csv_to_pretty_txt(csv_file, txt_file_name) + + if footnote: + with open(txt_file_name, 'a') as txt_file: + txt_file.writelines([ + "\nFootnotes:\n", + "[1] CSIT-1908 changed test methodology of dot1q tests in " + "2n-skx testbeds, dot1q encapsulation is now used on both " + "links of SUT.\n", + " Previously dot1q was used only on a single link with the " + "other link carrying untagged Ethernet frames. This change " + "results\n", + " in slightly lower throughput in CSIT-1908 for these " + "tests. See release notes." + ]) def table_nics_comparison(table, input_data): @@ -438,9 +744,9 @@ def table_nics_comparison(table, input_data): header = ["Test case", ] if table["include-tests"] == "MRR": - hdr_param = "Receive Rate" + hdr_param = "Rec Rate" else: - hdr_param = "Throughput" + hdr_param = "Thput" header.extend( ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param), @@ -547,11 +853,11 @@ def table_soak_vs_ndr(table, input_data): try: header = [ "Test case", - "{0} Throughput [Mpps]".format(table["reference"]["title"]), + "{0} Thput [Mpps]".format(table["reference"]["title"]), "{0} Stdev [Mpps]".format(table["reference"]["title"]), - "{0} Throughput [Mpps]".format(table["compare"]["title"]), + "{0} Thput [Mpps]".format(table["compare"]["title"]), "{0} Stdev [Mpps]".format(table["compare"]["title"]), - "Delta [%]"] + "Delta [%]", "Stdev of delta [%]"] header_str = ",".join(header) + "\n" except (AttributeError, KeyError) as err: logging.error("The model is invalid, missing parameter: {0}". @@ -608,21 +914,31 @@ def table_soak_vs_ndr(table, input_data): tbl_lst = list() for tst_name in tbl_dict.keys(): item = [tbl_dict[tst_name]["name"], ] - data_t = tbl_dict[tst_name]["ref-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r = tbl_dict[tst_name]["ref-data"] + if data_r: + data_r_mean = mean(data_r) + item.append(round(data_r_mean / 1000000, 2)) + data_r_stdev = stdev(data_r) + item.append(round(data_r_stdev / 1000000, 2)) else: + data_r_mean = None + data_r_stdev = None item.extend([None, None]) - data_t = tbl_dict[tst_name]["cmp-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_c = tbl_dict[tst_name]["cmp-data"] + if data_c: + data_c_mean = mean(data_c) + item.append(round(data_c_mean / 1000000, 2)) + data_c_stdev = stdev(data_c) + item.append(round(data_c_stdev / 1000000, 2)) else: + data_c_mean = None + data_c_stdev = None item.extend([None, None]) - if item[-4] is not None and item[-2] is not None and item[-4] != 0: - item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if len(item) == len(header): + if data_r_mean and data_c_mean: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev) + item.append(round(delta, 2)) + item.append(round(d_stdev, 2)) tbl_lst.append(item) # Sort the table according to the relative change @@ -672,7 +988,7 @@ def table_performance_trending_dashboard(table, input_data): for job, builds in table["data"].items(): for build in builds: for tst_name, tst_data in data[job][str(build)].iteritems(): - if tst_name.lower() in table["ignore-list"]: + if tst_name.lower() in table.get("ignore-list", list()): continue if tbl_dict.get(tst_name, None) is None: groups = re.search(REGEX_NIC, tst_data["parent"]) @@ -808,9 +1124,17 @@ def _generate_url(base, testbed, test_name): file_name = "vm_vhost_ip4" feature = "-base" + elif "ipsecbasetnlsw" in test_name: + file_name = "ipsecsw" + feature = "-base-scale" + elif "ipsec" in test_name: file_name = "ipsec" feature = "-base-scale" + if "hw-" in test_name: + file_name = "ipsechw" + elif "sw-" in test_name: + file_name = "ipsecsw" elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name: file_name = "ip4_tunnels" @@ -891,8 +1215,8 @@ def _generate_url(base, testbed, test_name): elif "8t4c" in test_name: anchor += "8t4c" - return url + file_name + '-' + testbed + '-' + nic + framesize + feature + \ - anchor + feature + return url + file_name + '-' + testbed + '-' + nic + framesize + \ + feature.replace("-int", "").replace("-tnl", "") + anchor + feature def table_performance_trending_dashboard_html(table, input_data): @@ -1060,7 +1384,7 @@ def table_failed_tests(table, input_data): for build in builds: build = str(build) for tst_name, tst_data in data[job][build].iteritems(): - if tst_name.lower() in table["ignore-list"]: + if tst_name.lower() in table.get("ignore-list", list()): continue if tbl_dict.get(tst_name, None) is None: groups = re.search(REGEX_NIC, tst_data["parent"])