X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=1366ea0824f30895aa99d3593d715a1867440b4f;hp=6f075d1f9f7f57fdcb34e1ac268f48e19fca0912;hb=6b353c8ae146ed5ce1c30addff6744954ed4d305;hpb=2a4a12ac1473d7d90e90a880867dfc21b986e386

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 6f075d1f9f..1366ea0824 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -29,8 +29,9 @@ import plotly.offline as ploff
 import pandas as pd
 
 from numpy import nan, isnan
+from yaml import load, FullLoader, YAMLError
 
-from pal_utils import mean, stdev, relative_change, classify_anomalies, \
+from pal_utils import mean, stdev, classify_anomalies, \
     convert_csv_to_pretty_txt, relative_change_stdev
 
 
@@ -264,7 +265,7 @@ def table_oper_data_html(table, input_data):
         if not html_table:
             continue
         try:
-            file_name = f"{table[u'output-file']}_{suite[u'name']}.rst"
+            file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
             with open(f"{file_name}", u'w') as html_file:
                 logging.info(f"    Writing file: {file_name}")
                 html_file.write(u".. raw:: html\n\n\t")
@@ -287,6 +288,7 @@ def table_merged_details(table, input_data):
     """
 
     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
+
     # Transform the data
     logging.info(
         f"    Creating the data set for the {table.get(u'type', u'')} "
@@ -341,9 +343,15 @@ def table_merged_details(table, input_data):
                                        f"{u'-'.join(col_data_lst[half:])}"
                         col_data = f" |prein| {col_data} |preout| "
                     elif column[u"data"].split(u" ")[1] in (u"msg", ):
+                        # Temporary solution: remove NDR results from message:
+                        if bool(table.get(u'remove-ndr', False)):
+                            try:
+                                col_data = col_data.split(u" |br| ", 1)[1]
+                            except IndexError:
+                                pass
                         col_data = f" |prein| {col_data} |preout| "
                     elif column[u"data"].split(u" ")[1] in \
-                        (u"conf-history", u"show-run"):
+                            (u"conf-history", u"show-run"):
                         col_data = col_data.replace(u" |br| ", u"", 1)
                         col_data = f" |prein| {col_data[:-5]} |preout| "
                     row_lst.append(f'"{col_data}"')
@@ -354,7 +362,8 @@ def table_merged_details(table, input_data):
 
         # Write the data to file
         if table_lst:
-            file_name = f"{table[u'output-file']}_{suite_name}.csv"
+            separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
+            file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
             logging.info(f"      Writing file: {file_name}")
             with open(file_name, u"wt") as file_handler:
                 file_handler.write(u",".join(header) + u"\n")
@@ -440,7 +449,6 @@ def _tpc_sort_table(table):
     :rtype: list
     """
 
-
     tbl_new = list()
     tbl_see = list()
     tbl_delta = list()
@@ -456,12 +464,14 @@ def _tpc_sort_table(table):
     # Sort the tables:
     tbl_new.sort(key=lambda rel: rel[0], reverse=False)
     tbl_see.sort(key=lambda rel: rel[0], reverse=False)
-    tbl_see.sort(key=lambda rel: rel[-1], reverse=False)
-    tbl_delta.sort(key=lambda rel: rel[-1], reverse=True)
+    tbl_see.sort(key=lambda rel: rel[-2], reverse=False)
+    tbl_delta.sort(key=lambda rel: rel[0], reverse=False)
+    tbl_delta.sort(key=lambda rel: rel[-2], reverse=True)
 
     # Put the tables together:
     table = list()
-    table.extend(tbl_new)
+    # We do not want "New in CSIT":
+    # table.extend(tbl_new)
     table.extend(tbl_see)
     table.extend(tbl_delta)
 
@@ -483,14 +493,24 @@ def _tpc_generate_html_table(header, data, output_file_name):
     :type output_file_name: str
     """
 
+    try:
+        idx = header.index(u"Test case")
+    except ValueError:
+        idx = 0
+    params = {
+        u"align-hdr": ([u"left", u"center"], [u"left", u"left", u"center"]),
+        u"align-itm": ([u"left", u"right"], [u"left", u"left", u"right"]),
+        u"width": ([28, 9], [4, 24, 10])
+    }
+
     df_data = pd.DataFrame(data, columns=header)
 
     df_sorted = [df_data.sort_values(
-        by=[key, header[0]], ascending=[True, True]
-        if key != header[0] else [False, True]) for key in header]
+        by=[key, header[idx]], ascending=[True, True]
+        if key != header[idx] else [False, True]) for key in header]
     df_sorted_rev = [df_data.sort_values(
-        by=[key, header[0]], ascending=[False, True]
-        if key != header[0] else [True, True]) for key in header]
+        by=[key, header[idx]], ascending=[False, True]
+        if key != header[idx] else [True, True]) for key in header]
     df_sorted.extend(df_sorted_rev)
 
     fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
@@ -498,7 +518,7 @@ def _tpc_generate_html_table(header, data, output_file_name):
     table_header = dict(
         values=[f"<b>{item}</b>" for item in header],
         fill_color=u"#7eade7",
-        align=[u"left", u"center"]
+        align=params[u"align-hdr"][idx]
     )
 
     fig = go.Figure()
@@ -507,12 +527,12 @@ def _tpc_generate_html_table(header, data, output_file_name):
         columns = [table.get(col) for col in header]
         fig.add_trace(
             go.Table(
-                columnwidth=[30, 10],
+                columnwidth=params[u"width"][idx],
                 header=table_header,
                 cells=dict(
                     values=columns,
                     fill_color=fill_color,
-                    align=[u"left", u"right"]
+                    align=params[u"align-itm"][idx]
                 )
             )
         )
@@ -584,6 +604,16 @@ def table_perf_comparison(table, input_data):
     try:
         header = [u"Test case", ]
 
+        rca_data = None
+        rca = table.get(u"rca", None)
+        if rca:
+            try:
+                with open(rca.get(u"data-file", ""), u"r") as rca_file:
+                    rca_data = load(rca_file, Loader=FullLoader)
+                header.insert(0, rca.get(u"title", "RCA"))
+            except (YAMLError, IOError) as err:
+                logging.warning(repr(err))
+
         if table[u"include-tests"] == u"MRR":
             hdr_param = u"Rec Rate"
         else:
@@ -603,19 +633,18 @@ def table_perf_comparison(table, input_data):
                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
-                u"Delta [%]"
+                u"Delta [%]",
+                u"Stdev of delta [%]"
             ]
         )
-        header_str = u",".join(header) + u"\n"
+        header_str = u";".join(header) + u"\n"
     except (AttributeError, KeyError) as err:
         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
         return
 
     # Prepare data to the table:
     tbl_dict = dict()
-    # topo = ""
     for job, builds in table[u"reference"][u"data"].items():
-        # topo = u"2n-skx" if u"2n-skx" in job else u""
         for build in builds:
             for tst_name, tst_data in data[job][str(build)].items():
                 tst_name_mod = _tpc_modify_test_name(tst_name)
@@ -767,7 +796,6 @@ def table_perf_comparison(table, input_data):
                         pass
 
     tbl_lst = list()
-    footnote = False
     for tst_name in tbl_dict:
         item = [tbl_dict[tst_name][u"name"], ]
         if history:
@@ -780,28 +808,46 @@ def table_perf_comparison(table, input_data):
                         item.extend([u"Not tested", u"Not tested"])
             else:
                 item.extend([u"Not tested", u"Not tested"])
-        data_t = tbl_dict[tst_name][u"ref-data"]
-        if data_t:
-            item.append(round(mean(data_t) / 1000000, 2))
-            item.append(round(stdev(data_t) / 1000000, 2))
+        data_r = tbl_dict[tst_name][u"ref-data"]
+        if data_r:
+            data_r_mean = mean(data_r)
+            item.append(round(data_r_mean / 1000000, 2))
+            data_r_stdev = stdev(data_r)
+            item.append(round(data_r_stdev / 1000000, 2))
         else:
+            data_r_mean = None
+            data_r_stdev = None
             item.extend([u"Not tested", u"Not tested"])
-        data_t = tbl_dict[tst_name][u"cmp-data"]
-        if data_t:
-            item.append(round(mean(data_t) / 1000000, 2))
-            item.append(round(stdev(data_t) / 1000000, 2))
+        data_c = tbl_dict[tst_name][u"cmp-data"]
+        if data_c:
+            data_c_mean = mean(data_c)
+            item.append(round(data_c_mean / 1000000, 2))
+            data_c_stdev = stdev(data_c)
+            item.append(round(data_c_stdev / 1000000, 2))
         else:
+            data_c_mean = None
+            data_c_stdev = None
             item.extend([u"Not tested", u"Not tested"])
         if item[-2] == u"Not tested":
             pass
         elif item[-4] == u"Not tested":
             item.append(u"New in CSIT-2001")
-        # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
-        #     item.append(u"See footnote [1]")
-        #     footnote = True
-        elif item[-4] != 0:
-            item.append(int(relative_change(float(item[-4]), float(item[-2]))))
-        if (len(item) == len(header)) and (item[-3] != u"Not tested"):
+            item.append(u"New in CSIT-2001")
+        elif data_r_mean and data_c_mean:
+            delta, d_stdev = relative_change_stdev(
+                data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
+            )
+            try:
+                item.append(round(delta))
+            except ValueError:
+                item.append(delta)
+            try:
+                item.append(round(d_stdev))
+            except ValueError:
+                item.append(d_stdev)
+        if rca_data:
+            item.insert(0, rca_data.get(item[0], u" "))
+        if (len(item) == len(header)) and (item[-4] != u"Not tested"):
             tbl_lst.append(item)
 
     tbl_lst = _tpc_sort_table(tbl_lst)
@@ -811,24 +857,16 @@ def table_perf_comparison(table, input_data):
     with open(csv_file, u"wt") as file_handler:
         file_handler.write(header_str)
         for test in tbl_lst:
-            file_handler.write(u",".join([str(item) for item in test]) + u"\n")
+            file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 
     txt_file_name = f"{table[u'output-file']}.txt"
     convert_csv_to_pretty_txt(csv_file, txt_file_name)
 
-    if footnote:
-        with open(txt_file_name, u'a') as txt_file:
-            txt_file.writelines([
-                u"\nFootnotes:\n",
-                u"[1] CSIT-1908 changed test methodology of dot1q tests in "
-                u"2-node testbeds, dot1q encapsulation is now used on both "
-                u"links of SUT.\n",
-                u"    Previously dot1q was used only on a single link with the "
-                u"other link carrying untagged Ethernet frames. This changes "
-                u"results\n",
-                u"    in slightly lower throughput in CSIT-1908 for these "
-                u"tests. See release notes."
-            ])
+    if rca_data:
+        footnote = rca_data.get(u"footnote", "")
+        if footnote:
+            with open(txt_file_name, u'a') as txt_file:
+                txt_file.writelines(footnote)
 
     # Generate html table:
     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
@@ -857,6 +895,16 @@ def table_perf_comparison_nic(table, input_data):
     try:
         header = [u"Test case", ]
 
+        rca_data = None
+        rca = table.get(u"rca", None)
+        if rca:
+            try:
+                with open(rca.get(u"data-file", ""), u"r") as rca_file:
+                    rca_data = load(rca_file, Loader=FullLoader)
+                header.insert(0, rca.get(u"title", "RCA"))
+            except (YAMLError, IOError) as err:
+                logging.warning(repr(err))
+
         if table[u"include-tests"] == u"MRR":
             hdr_param = u"Rec Rate"
         else:
@@ -876,19 +924,18 @@ def table_perf_comparison_nic(table, input_data):
                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
-                u"Delta [%]"
+                u"Delta [%]",
+                u"Stdev of delta [%]"
             ]
         )
-        header_str = u",".join(header) + u"\n"
+        header_str = u";".join(header) + u"\n"
     except (AttributeError, KeyError) as err:
         logging.error(f"The model is invalid, missing parameter: {repr(err)}")
         return
 
     # Prepare data to the table:
     tbl_dict = dict()
-    # topo = u""
     for job, builds in table[u"reference"][u"data"].items():
-        # topo = u"2n-skx" if u"2n-skx" in job else u""
         for build in builds:
             for tst_name, tst_data in data[job][str(build)].items():
                 if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
@@ -1046,7 +1093,6 @@ def table_perf_comparison_nic(table, input_data):
                         pass
 
     tbl_lst = list()
-    footnote = False
     for tst_name in tbl_dict:
         item = [tbl_dict[tst_name][u"name"], ]
         if history:
@@ -1059,28 +1105,46 @@ def table_perf_comparison_nic(table, input_data):
                         item.extend([u"Not tested", u"Not tested"])
             else:
                 item.extend([u"Not tested", u"Not tested"])
-        data_t = tbl_dict[tst_name][u"ref-data"]
-        if data_t:
-            item.append(round(mean(data_t) / 1000000, 2))
-            item.append(round(stdev(data_t) / 1000000, 2))
+        data_r = tbl_dict[tst_name][u"ref-data"]
+        if data_r:
+            data_r_mean = mean(data_r)
+            item.append(round(data_r_mean / 1000000, 2))
+            data_r_stdev = stdev(data_r)
+            item.append(round(data_r_stdev / 1000000, 2))
         else:
+            data_r_mean = None
+            data_r_stdev = None
             item.extend([u"Not tested", u"Not tested"])
-        data_t = tbl_dict[tst_name][u"cmp-data"]
-        if data_t:
-            item.append(round(mean(data_t) / 1000000, 2))
-            item.append(round(stdev(data_t) / 1000000, 2))
+        data_c = tbl_dict[tst_name][u"cmp-data"]
+        if data_c:
+            data_c_mean = mean(data_c)
+            item.append(round(data_c_mean / 1000000, 2))
+            data_c_stdev = stdev(data_c)
+            item.append(round(data_c_stdev / 1000000, 2))
         else:
+            data_c_mean = None
+            data_c_stdev = None
             item.extend([u"Not tested", u"Not tested"])
         if item[-2] == u"Not tested":
             pass
         elif item[-4] == u"Not tested":
             item.append(u"New in CSIT-2001")
-        # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
-        #     item.append(u"See footnote [1]")
-        #     footnote = True
-        elif item[-4] != 0:
-            item.append(int(relative_change(float(item[-4]), float(item[-2]))))
-        if (len(item) == len(header)) and (item[-3] != u"Not tested"):
+            item.append(u"New in CSIT-2001")
+        elif data_r_mean and data_c_mean:
+            delta, d_stdev = relative_change_stdev(
+                data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
+            )
+            try:
+                item.append(round(delta))
+            except ValueError:
+                item.append(delta)
+            try:
+                item.append(round(d_stdev))
+            except ValueError:
+                item.append(d_stdev)
+        if rca_data:
+            item.insert(0, rca_data.get(item[0], u" "))
+        if (len(item) == len(header)) and (item[-4] != u"Not tested"):
             tbl_lst.append(item)
 
     tbl_lst = _tpc_sort_table(tbl_lst)
@@ -1090,24 +1154,16 @@ def table_perf_comparison_nic(table, input_data):
     with open(csv_file, u"wt") as file_handler:
         file_handler.write(header_str)
         for test in tbl_lst:
-            file_handler.write(u",".join([str(item) for item in test]) + u"\n")
+            file_handler.write(u";".join([str(item) for item in test]) + u"\n")
 
     txt_file_name = f"{table[u'output-file']}.txt"
-    convert_csv_to_pretty_txt(csv_file, txt_file_name)
+    convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
 
-    if footnote:
-        with open(txt_file_name, u'a') as txt_file:
-            txt_file.writelines([
-                u"\nFootnotes:\n",
-                u"[1] CSIT-1908 changed test methodology of dot1q tests in "
-                u"2-node testbeds, dot1q encapsulation is now used on both "
-                u"links of SUT.\n",
-                u"    Previously dot1q was used only on a single link with the "
-                u"other link carrying untagged Ethernet frames. This changes "
-                u"results\n",
-                u"    in slightly lower throughput in CSIT-1908 for these "
-                u"tests. See release notes."
-            ])
+    if rca_data:
+        footnote = rca_data.get(u"footnote", "")
+        if footnote:
+            with open(txt_file_name, u'a') as txt_file:
+                txt_file.writelines(footnote)
 
     # Generate html table:
     _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
@@ -1147,7 +1203,8 @@ def table_nics_comparison(table, input_data):
                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
-                u"Delta [%]"
+                u"Delta [%]",
+                u"Stdev of delta [%]"
             ]
         )
 
@@ -1169,7 +1226,6 @@ def table_nics_comparison(table, input_data):
                         u"cmp-data": list()
                     }
                 try:
-                    result = None
                     if table[u"include-tests"] == u"MRR":
                         result = tst_data[u"result"][u"receive-rate"]
                     elif table[u"include-tests"] == u"PDR":
@@ -1192,21 +1248,38 @@ def table_nics_comparison(table, input_data):
     tbl_lst = list()
     for tst_name in tbl_dict:
         item = [tbl_dict[tst_name][u"name"], ]
-        data_t = tbl_dict[tst_name][u"ref-data"]
-        if data_t:
-            item.append(round(mean(data_t) / 1000000, 2))
-            item.append(round(stdev(data_t) / 1000000, 2))
+        data_r = tbl_dict[tst_name][u"ref-data"]
+        if data_r:
+            data_r_mean = mean(data_r)
+            item.append(round(data_r_mean / 1000000, 2))
+            data_r_stdev = stdev(data_r)
+            item.append(round(data_r_stdev / 1000000, 2))
         else:
+            data_r_mean = None
+            data_r_stdev = None
             item.extend([None, None])
-        data_t = tbl_dict[tst_name][u"cmp-data"]
-        if data_t:
-            item.append(round(mean(data_t) / 1000000, 2))
-            item.append(round(stdev(data_t) / 1000000, 2))
+        data_c = tbl_dict[tst_name][u"cmp-data"]
+        if data_c:
+            data_c_mean = mean(data_c)
+            item.append(round(data_c_mean / 1000000, 2))
+            data_c_stdev = stdev(data_c)
+            item.append(round(data_c_stdev / 1000000, 2))
         else:
+            data_c_mean = None
+            data_c_stdev = None
             item.extend([None, None])
-        if item[-4] is not None and item[-2] is not None and item[-4] != 0:
-            item.append(int(relative_change(float(item[-4]), float(item[-2]))))
-        if len(item) == len(header):
+        if data_r_mean and data_c_mean:
+            delta, d_stdev = relative_change_stdev(
+                data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
+            )
+            try:
+                item.append(round(delta))
+            except ValueError:
+                item.append(delta)
+            try:
+                item.append(round(d_stdev))
+            except ValueError:
+                item.append(d_stdev)
             tbl_lst.append(item)
 
     # Sort the table according to the relative change
@@ -1252,7 +1325,8 @@ def table_soak_vs_ndr(table, input_data):
             f"{table[u'reference'][u'title']} Stdev [Mpps]",
             f"{table[u'compare'][u'title']} Thput [Mpps]",
             f"{table[u'compare'][u'title']} Stdev [Mpps]",
-            u"Delta [%]", u"Stdev of delta [%]"
+            u"Delta [%]",
+            u"Stdev of delta [%]"
         ]
         header_str = u",".join(header) + u"\n"
     except (AttributeError, KeyError) as err:
@@ -1338,8 +1412,14 @@ def table_soak_vs_ndr(table, input_data):
         if data_r_mean and data_c_mean:
             delta, d_stdev = relative_change_stdev(
                 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
-            item.append(round(delta, 2))
-            item.append(round(d_stdev, 2))
+            try:
+                item.append(round(delta))
+            except ValueError:
+                item.append(delta)
+            try:
+                item.append(round(d_stdev))
+            except ValueError:
+                item.append(d_stdev)
             tbl_lst.append(item)
 
     # Sort the table according to the relative change