From b95d8570e5edb080d7cfc001daed66b47811068a Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Tue, 24 Mar 2020 14:02:54 +0100 Subject: [PATCH] Report: Add mrr stdev to comp tables Change-Id: I94336a7e624b82a5794ff09085c46028f4416a49 Signed-off-by: Tibor Frank --- resources/tools/presentation/generator_tables.py | 147 ++++++++++++++------- resources/tools/presentation/input_data_parser.py | 1 + .../tools/presentation/rca/rca-2n-skx-2t1c.yaml | 92 ++++++------- resources/tools/presentation/specification.yaml | 12 +- 4 files changed, 155 insertions(+), 97 deletions(-) diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 1366ea0824..4cbc7c0746 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -427,7 +427,12 @@ def _tpc_insert_data(target, src, include_tests): """ try: if include_tests == u"MRR": - target.append(src[u"result"][u"receive-rate"]) + target.append( + ( + src[u"result"][u"receive-rate"], + src[u"result"][u"receive-stdev"] + ) + ) elif include_tests == u"PDR": target.append(src[u"throughput"][u"PDR"][u"LOWER"]) elif include_tests == u"NDR": @@ -783,7 +788,8 @@ def table_perf_comparison(table, input_data): u"title"]] = list() try: if table[u"include-tests"] == u"MRR": - res = tst_data[u"result"][u"receive-rate"] + res = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) elif table[u"include-tests"] == u"PDR": res = tst_data[u"throughput"][u"PDR"][u"LOWER"] elif table[u"include-tests"] == u"NDR": @@ -802,28 +808,40 @@ def table_perf_comparison(table, input_data): if tbl_dict[tst_name].get(u"history", None) is not None: for hist_data in tbl_dict[tst_name][u"history"].values(): if hist_data: - item.append(round(mean(hist_data) / 1000000, 2)) - item.append(round(stdev(hist_data) / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + item.append(round(hist_data[0][0] / 1e6, 2)) + item.append(round(hist_data[0][1] / 1e6, 2)) + else: + item.append(round(mean(hist_data) / 1e6, 2)) + item.append(round(stdev(hist_data) / 1e6, 2)) else: item.extend([u"Not tested", u"Not tested"]) else: item.extend([u"Not tested", u"Not tested"]) data_r = tbl_dict[tst_name][u"ref-data"] if data_r: - data_r_mean = mean(data_r) - item.append(round(data_r_mean / 1000000, 2)) - data_r_stdev = stdev(data_r) - item.append(round(data_r_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 2)) + item.append(round(data_r_stdev / 1e6, 2)) else: data_r_mean = None data_r_stdev = None item.extend([u"Not tested", u"Not tested"]) data_c = tbl_dict[tst_name][u"cmp-data"] if data_c: - data_c_mean = mean(data_c) - item.append(round(data_c_mean / 1000000, 2)) - data_c_stdev = stdev(data_c) - item.append(round(data_c_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 2)) + item.append(round(data_c_stdev / 1e6, 2)) else: data_c_mean = None data_c_stdev = None @@ -833,7 +851,7 @@ def table_perf_comparison(table, input_data): elif item[-4] == u"Not tested": item.append(u"New in CSIT-2001") item.append(u"New in CSIT-2001") - elif data_r_mean and data_c_mean: + elif data_r_mean is not None and data_c_mean is not None: delta, d_stdev = relative_change_stdev( data_r_mean, data_c_mean, data_r_stdev, data_c_stdev ) @@ -846,7 +864,8 @@ def table_perf_comparison(table, input_data): except ValueError: item.append(d_stdev) if rca_data: - item.insert(0, rca_data.get(item[0], u" ")) + rca_nr = rca_data.get(item[0], u"-") + item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-") if (len(item) == len(header)) and (item[-4] != u"Not tested"): tbl_lst.append(item) @@ -860,7 +879,7 @@ def table_perf_comparison(table, input_data): file_handler.write(u";".join([str(item) for item in test]) + u"\n") txt_file_name = f"{table[u'output-file']}.txt" - convert_csv_to_pretty_txt(csv_file, txt_file_name) + convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";") if rca_data: footnote = rca_data.get(u"footnote", "") @@ -1080,7 +1099,8 @@ def table_perf_comparison_nic(table, input_data): u"title"]] = list() try: if table[u"include-tests"] == u"MRR": - res = tst_data[u"result"][u"receive-rate"] + res = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) elif table[u"include-tests"] == u"PDR": res = tst_data[u"throughput"][u"PDR"][u"LOWER"] elif table[u"include-tests"] == u"NDR": @@ -1099,28 +1119,40 @@ def table_perf_comparison_nic(table, input_data): if tbl_dict[tst_name].get(u"history", None) is not None: for hist_data in tbl_dict[tst_name][u"history"].values(): if hist_data: - item.append(round(mean(hist_data) / 1000000, 2)) - item.append(round(stdev(hist_data) / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + item.append(round(hist_data[0][0] / 1e6, 2)) + item.append(round(hist_data[0][1] / 1e6, 2)) + else: + item.append(round(mean(hist_data) / 1e6, 2)) + item.append(round(stdev(hist_data) / 1e6, 2)) else: item.extend([u"Not tested", u"Not tested"]) else: item.extend([u"Not tested", u"Not tested"]) data_r = tbl_dict[tst_name][u"ref-data"] if data_r: - data_r_mean = mean(data_r) - item.append(round(data_r_mean / 1000000, 2)) - data_r_stdev = stdev(data_r) - item.append(round(data_r_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 2)) + item.append(round(data_r_stdev / 1e6, 2)) else: data_r_mean = None data_r_stdev = None item.extend([u"Not tested", u"Not tested"]) data_c = tbl_dict[tst_name][u"cmp-data"] if data_c: - data_c_mean = mean(data_c) - item.append(round(data_c_mean / 1000000, 2)) - data_c_stdev = stdev(data_c) - item.append(round(data_c_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 2)) + item.append(round(data_c_stdev / 1e6, 2)) else: data_c_mean = None data_c_stdev = None @@ -1130,7 +1162,7 @@ def table_perf_comparison_nic(table, input_data): elif item[-4] == u"Not tested": item.append(u"New in CSIT-2001") item.append(u"New in CSIT-2001") - elif data_r_mean and data_c_mean: + elif data_r_mean is not None and data_c_mean is not None: delta, d_stdev = relative_change_stdev( data_r_mean, data_c_mean, data_r_stdev, data_c_stdev ) @@ -1143,7 +1175,8 @@ def table_perf_comparison_nic(table, input_data): except ValueError: item.append(d_stdev) if rca_data: - item.insert(0, rca_data.get(item[0], u" ")) + rca_nr = rca_data.get(item[0], u"-") + item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-") if (len(item) == len(header)) and (item[-4] != u"Not tested"): tbl_lst.append(item) @@ -1227,7 +1260,8 @@ def table_nics_comparison(table, input_data): } try: if table[u"include-tests"] == u"MRR": - result = tst_data[u"result"][u"receive-rate"] + result = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) elif table[u"include-tests"] == u"PDR": result = tst_data[u"throughput"][u"PDR"][u"LOWER"] elif table[u"include-tests"] == u"NDR": @@ -1250,25 +1284,33 @@ def table_nics_comparison(table, input_data): item = [tbl_dict[tst_name][u"name"], ] data_r = tbl_dict[tst_name][u"ref-data"] if data_r: - data_r_mean = mean(data_r) - item.append(round(data_r_mean / 1000000, 2)) - data_r_stdev = stdev(data_r) - item.append(round(data_r_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 2)) + item.append(round(data_r_stdev / 1e6, 2)) else: data_r_mean = None data_r_stdev = None item.extend([None, None]) data_c = tbl_dict[tst_name][u"cmp-data"] if data_c: - data_c_mean = mean(data_c) - item.append(round(data_c_mean / 1000000, 2)) - data_c_stdev = stdev(data_c) - item.append(round(data_c_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 2)) + item.append(round(data_c_stdev / 1e6, 2)) else: data_c_mean = None data_c_stdev = None item.extend([None, None]) - if data_r_mean and data_c_mean: + if data_r_mean is not None and data_c_mean is not None: delta, d_stdev = relative_change_stdev( data_r_mean, data_c_mean, data_r_stdev, data_c_stdev ) @@ -1371,7 +1413,8 @@ def table_soak_vs_ndr(table, input_data): if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"): continue if table[u"include-tests"] == u"MRR": - result = tst_data[u"result"][u"receive-rate"] + result = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) elif table[u"include-tests"] == u"PDR": result = \ tst_data[u"throughput"][u"PDR"][u"LOWER"] @@ -1391,25 +1434,33 @@ def table_soak_vs_ndr(table, input_data): item = [tbl_dict[tst_name][u"name"], ] data_r = tbl_dict[tst_name][u"ref-data"] if data_r: - data_r_mean = mean(data_r) - item.append(round(data_r_mean / 1000000, 2)) - data_r_stdev = stdev(data_r) - item.append(round(data_r_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 2)) + item.append(round(data_r_stdev / 1e6, 2)) else: data_r_mean = None data_r_stdev = None item.extend([None, None]) data_c = tbl_dict[tst_name][u"cmp-data"] if data_c: - data_c_mean = mean(data_c) - item.append(round(data_c_mean / 1000000, 2)) - data_c_stdev = stdev(data_c) - item.append(round(data_c_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 2)) + item.append(round(data_c_stdev / 1e6, 2)) else: data_c_mean = None data_c_stdev = None item.extend([None, None]) - if data_r_mean and data_c_mean: + if data_r_mean is not None and data_c_mean is not None: delta, d_stdev = relative_change_stdev( data_r_mean, data_c_mean, data_r_stdev, data_c_stdev) try: diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py index 9e460f934e..987b9964bc 100644 --- a/resources/tools/presentation/input_data_parser.py +++ b/resources/tools/presentation/input_data_parser.py @@ -1100,6 +1100,7 @@ class ExecutionChecker(ResultVisitor): # Use whole list in CSIT-1180. stats = jumpavg.AvgStdevStats.for_runs(items_float) test_result[u"result"][u"receive-rate"] = stats.avg + test_result[u"result"][u"receive-stdev"] = stats.stdev else: groups = re.search(self.REGEX_MRR, test.message) test_result[u"result"][u"receive-rate"] = \ diff --git a/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml b/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml index 564e15a219..267e4c90e3 100644 --- a/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml +++ b/resources/tools/presentation/rca/rca-2n-skx-2t1c.yaml @@ -1,46 +1,46 @@ -64b-2t1c-eth-l2patch: rc1 -64b-2t1c-eth-l2xcbase-eth-2memif-1dcr: rc4 -64b-2t1c-avf-eth-l2patch: rc1 -64b-2t1c-avf-eth-l2xcbase: rc1 -64b-2t1c-eth-l2bdbasemaclrn-eth-2memif-1dcr: rc1, rc5 -64b-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc: rc1 -64b-2t1c-dot1q-l2bdbasemaclrn-eth-2memif-1dcr: rc1 -64b-2t1c-dot1q-l2xcbase: rc1 -64b-2t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-vppip4: rc1 -64b-2t1c-eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc: rc3 -64b-2t1c-dot1q-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc: rc3 -64b-2t1c-ethip4-ip4base-eth-2memif-1dcr: rc1 -64b-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm: rc1 -64b-2t1c-eth-l2xcbase-eth-2vhostvr1024-1vm: rc2 -64b-2t1c-ethip4-ip4base-eth-2vhostvr1024-1vm: rc1 -64b-2t1c-eth-l2bdscale10kmaclrn: rc1 -64b-2t1c-eth-l2bdbasemaclrn: rc1 -64b-2t1c-eth-l2bdscale100kmaclrn: rc1 -64b-2t1c-eth-l2bdscale1mmaclrn: rc1 -78b-2t1c-ethip6-ip6base: rc1 -64b-2t1c-eth-l2xcbase: rc1 -78b-2t1c-dot1q-ip6base: rc1 -64b-2t1c-dot1q-l2bdbasemaclrn-eth-2vhostvr1024-1vm: rc2 -78b-2t1c-ethip6-ip6scale200k: rc1 -78b-2t1c-ethip6-ip6scale20k: rc1 -78b-2t1c-ethip6-ip6scale2m: rc1 -64b-2t1c-dot1q-l2bdbasemaclrn: rc1 -64b-2t1c-ethip4udp-ip4base-oacl50sf-10kflows: rc1 -64b-2t1c-ethip4udp-ip4base-oacl50sl-10kflows: rc1 -64b-2t1c-avf-eth-l2bdbasemaclrn: rc1 -64b-2t1c-ethip4-ip4scale20k: rc1 -64b-2t1c-avf-dot1q-l2bdbasemaclrn: rc1, rc3 -64b-2t1c-dot1q-ip4base: rc1 -64b-2t1c-ethip4-ip4base: rc1 -64b-2t1c-ethip4udp-ip4base-iacl50sf-10kflows: rc1 -64b-2t1c-ethip4udp-ip4base-iacl50sl-10kflows: rc1 -64b-2t1c-avf-dot1q-l2bdbasemaclrn-gbp: rc1 -64b-2t1c-ethip4-ip4scale200k: rc1 -64b-2t1c-ethip4-ip4scale2m: rc1 -64b-2t1c-avf-dot1q-ip4base: rc1 -64b-2t1c-avf-ethip4-ip4base: rc1 -64b-2t1c-avf-ethip4-ip4scale200k: rc1 -64b-2t1c-avf-ethip4-ip4scale20k: rc1 -64b-2t1c-avf-ethip4-ip4scale2m: rc1 -64b-2t1c-ethip4udp-ip4base-nat44: rc1 -footnote: "\nRoot causes:\nrc1: Description 1\nrc2: Description 2\nrc3: Description 3" +64b-2t1c-eth-l2patch: 0 +64b-2t1c-eth-l2xcbase-eth-2memif-1dcr: 9 +64b-2t1c-avf-eth-l2patch: 8 +64b-2t1c-avf-eth-l2xcbase: 7 +64b-2t1c-eth-l2bdbasemaclrn-eth-2memif-1dcr: 6 +64b-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc: 5 +64b-2t1c-dot1q-l2bdbasemaclrn-eth-2memif-1dcr: "-" +64b-2t1c-dot1q-l2xcbase: 1 +64b-2t1c-ethip4-ip4base-eth-2vhostvr1024-1vm-vppip4: 2 +64b-2t1c-eth-l2xcbase-eth-2vhostvr1024-1vm-vppl2xc: 3 +64b-2t1c-dot1q-l2bdbasemaclrn-eth-2vhostvr1024-1vm-vppl2xc: 4 +64b-2t1c-ethip4-ip4base-eth-2memif-1dcr: 0 +64b-2t1c-eth-l2bdbasemaclrn-eth-2vhostvr1024-1vm: 0 +64b-2t1c-eth-l2xcbase-eth-2vhostvr1024-1vm: 8 +64b-2t1c-ethip4-ip4base-eth-2vhostvr1024-1vm: "-" +64b-2t1c-eth-l2bdscale10kmaclrn: 2 +64b-2t1c-eth-l2bdbasemaclrn: 2 +64b-2t1c-eth-l2bdscale100kmaclrn: 56 +64b-2t1c-eth-l2bdscale1mmaclrn: 5 +78b-2t1c-ethip6-ip6base: 4 +64b-2t1c-eth-l2xcbase: 34 +78b-2t1c-dot1q-ip6base: 0 +64b-2t1c-dot1q-l2bdbasemaclrn-eth-2vhostvr1024-1vm: 2 +78b-2t1c-ethip6-ip6scale200k: 6 +78b-2t1c-ethip6-ip6scale20k: 5 +78b-2t1c-ethip6-ip6scale2m: 3 +64b-2t1c-dot1q-l2bdbasemaclrn: 2 +64b-2t1c-ethip4udp-ip4base-oacl50sf-10kflows: "-" +64b-2t1c-ethip4udp-ip4base-oacl50sl-10kflows: 4 +64b-2t1c-avf-eth-l2bdbasemaclrn: 3 +64b-2t1c-ethip4-ip4scale20k: 6 +64b-2t1c-avf-dot1q-l2bdbasemaclrn: 8 +64b-2t1c-dot1q-ip4base: 4 +64b-2t1c-ethip4-ip4base: 6 +64b-2t1c-ethip4udp-ip4base-iacl50sf-10kflows: 0 +64b-2t1c-ethip4udp-ip4base-iacl50sl-10kflows: 9 +64b-2t1c-avf-dot1q-l2bdbasemaclrn-gbp: 8 +64b-2t1c-ethip4-ip4scale200k: 9 +64b-2t1c-ethip4-ip4scale2m: 22 +64b-2t1c-avf-dot1q-ip4base: 11 +64b-2t1c-avf-ethip4-ip4base: 4 +64b-2t1c-avf-ethip4-ip4scale200k: 5 +64b-2t1c-avf-ethip4-ip4scale20k: 6 +64b-2t1c-avf-ethip4-ip4scale2m: 1 +64b-2t1c-ethip4udp-ip4base-nat44: 4 +footnote: "\nRoot causes:\n[0]: Description 0\n[1]: Description \n[2]: Description 2" diff --git a/resources/tools/presentation/specification.yaml b/resources/tools/presentation/specification.yaml index 45bef9efab..e5810a215f 100644 --- a/resources/tools/presentation/specification.yaml +++ b/resources/tools/presentation/specification.yaml @@ -286,6 +286,7 @@ - 60 # NDRPDR sel addition - 61 # NDRPDR sel addition - 62 # NDRPDR sel addition + - 63 # NDRPDR sel addition plot-vpp-hdrh-lat-3n-skx: csit-vpp-perf-verify-2001-3n-skx: @@ -624,6 +625,7 @@ - 60 # NDRPDR sel addition - 61 # NDRPDR sel addition - 62 # NDRPDR sel addition + - 63 # NDRPDR sel addition vpp-nic-comparison-2n-skx: csit-vpp-perf-verify-2001-2n-skx: @@ -909,6 +911,7 @@ - 60 # NDRPDR sel addition - 61 # NDRPDR sel addition - 62 # NDRPDR sel addition + - 63 # NDRPDR sel addition vpp-performance-changes-3n-skx: - "vpp-performance-changes-3n-skx-h1" - "vpp-performance-changes-3n-skx-ref" @@ -1418,6 +1421,7 @@ - 60 # NDRPDR sel addition - 61 # NDRPDR sel addition - 62 # NDRPDR sel addition + - 63 # NDRPDR sel addition vpp-compare-testbeds: - "vpp-compare-testbeds-ref" - "vpp-compare-testbeds-cmp" @@ -1637,6 +1641,7 @@ - 60 # NDRPDR sel addition - 61 # NDRPDR sel addition - 62 # NDRPDR sel addition + - 63 # NDRPDR sel addition vpp-compare-topologies-cmp: # TODO: Add the data csit-vpp-perf-verify-2001-2n-skx: @@ -2519,6 +2524,7 @@ - 60 # NDRPDR sel addition - 61 # NDRPDR sel addition - 62 # NDRPDR sel addition + - 63 # NDRPDR sel addition - 50 # MRR sel - 53 # MRR sel addition @@ -3462,9 +3468,9 @@ title: "VPP Performance Changes 2n-skx 2t1c pdr" algorithm: "table_perf_comparison_nic" output-file: "{DIR[STATIC,VPP]}/performance-changes-2n-skx-2t1c-pdr" -# rca: -# title: "RCA" -# data-file: "rca/rca-2n-skx-2t1c.yaml" + rca: + title: "RCA" + data-file: "rca/rca-2n-skx-2t1c.yaml" history: - title: "rls1904 x710" data: "vpp-performance-changes-2n-skx-h1" -- 2.16.6