PAL: Add stdev of delta to tables

[csit.git] / resources / tools / presentation / generator_tables.py

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 6f075d1..2e2dfef 100644 (file)
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -30,7 +30,7 @@ import pandas as pd
 
 from numpy import nan, isnan
 
-from pal_utils import mean, stdev, relative_change, classify_anomalies, \
+from pal_utils import mean, stdev, classify_anomalies, \
     convert_csv_to_pretty_txt, relative_change_stdev
 
 
@@ -264,7 +264,7 @@ def table_oper_data_html(table, input_data):
         if not html_table:
             continue
         try:
-            file_name = f"{table[u'output-file']}_{suite[u'name']}.rst"
+            file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
             with open(f"{file_name}", u'w') as html_file:
                 logging.info(f"    Writing file: {file_name}")
                 html_file.write(u".. raw:: html\n\n\t")
@@ -287,6 +287,7 @@ def table_merged_details(table, input_data):
     """
 
     logging.info(f"  Generating the table {table.get(u'title', u'')} ...")
+
     # Transform the data
     logging.info(
         f"    Creating the data set for the {table.get(u'type', u'')} "
@@ -341,9 +342,15 @@ def table_merged_details(table, input_data):
                                        f"{u'-'.join(col_data_lst[half:])}"
                         col_data = f" |prein| {col_data} |preout| "
                     elif column[u"data"].split(u" ")[1] in (u"msg", ):
+                        # Temporary solution: remove NDR results from message:
+                        if bool(table.get(u'remove-ndr', False)):
+                            try:
+                                col_data = col_data.split(u" |br| ", 1)[1]
+                            except IndexError:
+                                pass
                         col_data = f" |prein| {col_data} |preout| "
                     elif column[u"data"].split(u" ")[1] in \
-                        (u"conf-history", u"show-run"):
+                            (u"conf-history", u"show-run"):
                         col_data = col_data.replace(u" |br| ", u"", 1)
                         col_data = f" |prein| {col_data[:-5]} |preout| "
                     row_lst.append(f'"{col_data}"')
@@ -354,7 +361,8 @@ def table_merged_details(table, input_data):
 
         # Write the data to file
         if table_lst:
-            file_name = f"{table[u'output-file']}_{suite_name}.csv"
+            separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
+            file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
             logging.info(f"      Writing file: {file_name}")
             with open(file_name, u"wt") as file_handler:
                 file_handler.write(u",".join(header) + u"\n")
@@ -440,7 +448,6 @@ def _tpc_sort_table(table):
     :rtype: list
     """
 
-
     tbl_new = list()
     tbl_see = list()
     tbl_delta = list()
@@ -461,7 +468,8 @@ def _tpc_sort_table(table):
 
     # Put the tables together:
     table = list()
-    table.extend(tbl_new)
+    # We do not want "New in CSIT":
+    # table.extend(tbl_new)
     table.extend(tbl_see)
     table.extend(tbl_delta)
 
@@ -603,7 +611,8 @@ def table_perf_comparison(table, input_data):
                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
-                u"Delta [%]"
+                u"Delta [%]",
+                u"Stdev of delta [%]"
             ]
         )
         header_str = u",".join(header) + u"\n"
@@ -780,17 +789,25 @@ def table_perf_comparison(table, input_data):
                         item.extend([u"Not tested", u"Not tested"])
             else:
                 item.extend([u"Not tested", u"Not tested"])
-        data_t = tbl_dict[tst_name][u"ref-data"]
-        if data_t:
-            item.append(round(mean(data_t) / 1000000, 2))
-            item.append(round(stdev(data_t) / 1000000, 2))
+        data_r = tbl_dict[tst_name][u"ref-data"]
+        if data_r:
+            data_r_mean = mean(data_r)
+            item.append(round(data_r_mean / 1000000, 2))
+            data_r_stdev = stdev(data_r)
+            item.append(round(data_r_stdev / 1000000, 2))
         else:
+            data_r_mean = None
+            data_r_stdev = None
             item.extend([u"Not tested", u"Not tested"])
-        data_t = tbl_dict[tst_name][u"cmp-data"]
-        if data_t:
-            item.append(round(mean(data_t) / 1000000, 2))
-            item.append(round(stdev(data_t) / 1000000, 2))
+        data_c = tbl_dict[tst_name][u"cmp-data"]
+        if data_c:
+            data_c_mean = mean(data_c)
+            item.append(round(data_c_mean / 1000000, 2))
+            data_c_stdev = stdev(data_c)
+            item.append(round(data_c_stdev / 1000000, 2))
         else:
+            data_c_mean = None
+            data_c_stdev = None
             item.extend([u"Not tested", u"Not tested"])
         if item[-2] == u"Not tested":
             pass
@@ -799,8 +816,12 @@ def table_perf_comparison(table, input_data):
         # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
         #     item.append(u"See footnote [1]")
         #     footnote = True
-        elif item[-4] != 0:
-            item.append(int(relative_change(float(item[-4]), float(item[-2]))))
+        elif data_r_mean and data_c_mean:
+            delta, d_stdev = relative_change_stdev(
+                data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
+            )
+            item.append(round(delta, 2))
+            item.append(round(d_stdev, 2))
         if (len(item) == len(header)) and (item[-3] != u"Not tested"):
             tbl_lst.append(item)
 
@@ -876,7 +897,8 @@ def table_perf_comparison_nic(table, input_data):
                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
-                u"Delta [%]"
+                u"Delta [%]",
+                u"Stdev of delta [%]"
             ]
         )
         header_str = u",".join(header) + u"\n"
@@ -1059,17 +1081,25 @@ def table_perf_comparison_nic(table, input_data):
                         item.extend([u"Not tested", u"Not tested"])
             else:
                 item.extend([u"Not tested", u"Not tested"])
-        data_t = tbl_dict[tst_name][u"ref-data"]
-        if data_t:
-            item.append(round(mean(data_t) / 1000000, 2))
-            item.append(round(stdev(data_t) / 1000000, 2))
+        data_r = tbl_dict[tst_name][u"ref-data"]
+        if data_r:
+            data_r_mean = mean(data_r)
+            item.append(round(data_r_mean / 1000000, 2))
+            data_r_stdev = stdev(data_r)
+            item.append(round(data_r_stdev / 1000000, 2))
         else:
+            data_r_mean = None
+            data_r_stdev = None
             item.extend([u"Not tested", u"Not tested"])
-        data_t = tbl_dict[tst_name][u"cmp-data"]
-        if data_t:
-            item.append(round(mean(data_t) / 1000000, 2))
-            item.append(round(stdev(data_t) / 1000000, 2))
+        data_c = tbl_dict[tst_name][u"cmp-data"]
+        if data_c:
+            data_c_mean = mean(data_c)
+            item.append(round(data_c_mean / 1000000, 2))
+            data_c_stdev = stdev(data_c)
+            item.append(round(data_c_stdev / 1000000, 2))
         else:
+            data_c_mean = None
+            data_c_stdev = None
             item.extend([u"Not tested", u"Not tested"])
         if item[-2] == u"Not tested":
             pass
@@ -1078,8 +1108,12 @@ def table_perf_comparison_nic(table, input_data):
         # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
         #     item.append(u"See footnote [1]")
         #     footnote = True
-        elif item[-4] != 0:
-            item.append(int(relative_change(float(item[-4]), float(item[-2]))))
+        elif data_r_mean and data_c_mean:
+            delta, d_stdev = relative_change_stdev(
+                data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
+            )
+            item.append(round(delta, 2))
+            item.append(round(d_stdev, 2))
         if (len(item) == len(header)) and (item[-3] != u"Not tested"):
             tbl_lst.append(item)
 
@@ -1147,7 +1181,8 @@ def table_nics_comparison(table, input_data):
                 f"{table[u'reference'][u'title']} Stdev [Mpps]",
                 f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
                 f"{table[u'compare'][u'title']} Stdev [Mpps]",
-                u"Delta [%]"
+                u"Delta [%]",
+                u"Stdev of delta [%]"
             ]
         )
 
@@ -1169,7 +1204,6 @@ def table_nics_comparison(table, input_data):
                         u"cmp-data": list()
                     }
                 try:
-                    result = None
                     if table[u"include-tests"] == u"MRR":
                         result = tst_data[u"result"][u"receive-rate"]
                     elif table[u"include-tests"] == u"PDR":
@@ -1192,21 +1226,32 @@ def table_nics_comparison(table, input_data):
     tbl_lst = list()
     for tst_name in tbl_dict:
         item = [tbl_dict[tst_name][u"name"], ]
-        data_t = tbl_dict[tst_name][u"ref-data"]
-        if data_t:
-            item.append(round(mean(data_t) / 1000000, 2))
-            item.append(round(stdev(data_t) / 1000000, 2))
+        data_r = tbl_dict[tst_name][u"ref-data"]
+        if data_r:
+            data_r_mean = mean(data_r)
+            item.append(round(data_r_mean / 1000000, 2))
+            data_r_stdev = stdev(data_r)
+            item.append(round(data_r_stdev / 1000000, 2))
         else:
+            data_r_mean = None
+            data_r_stdev = None
             item.extend([None, None])
-        data_t = tbl_dict[tst_name][u"cmp-data"]
-        if data_t:
-            item.append(round(mean(data_t) / 1000000, 2))
-            item.append(round(stdev(data_t) / 1000000, 2))
+        data_c = tbl_dict[tst_name][u"cmp-data"]
+        if data_c:
+            data_c_mean = mean(data_c)
+            item.append(round(data_c_mean / 1000000, 2))
+            data_c_stdev = stdev(data_c)
+            item.append(round(data_c_stdev / 1000000, 2))
         else:
+            data_c_mean = None
+            data_c_stdev = None
             item.extend([None, None])
-        if item[-4] is not None and item[-2] is not None and item[-4] != 0:
-            item.append(int(relative_change(float(item[-4]), float(item[-2]))))
-        if len(item) == len(header):
+        if data_r_mean and data_c_mean:
+            delta, d_stdev = relative_change_stdev(
+                data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
+            )
+            item.append(round(delta, 2))
+            item.append(round(d_stdev, 2))
             tbl_lst.append(item)
 
     # Sort the table according to the relative change
@@ -1252,7 +1297,8 @@ def table_soak_vs_ndr(table, input_data):
             f"{table[u'reference'][u'title']} Stdev [Mpps]",
             f"{table[u'compare'][u'title']} Thput [Mpps]",
             f"{table[u'compare'][u'title']} Stdev [Mpps]",
-            u"Delta [%]", u"Stdev of delta [%]"
+            u"Delta [%]",
+            u"Stdev of delta [%]"
         ]
         header_str = u",".join(header) + u"\n"
     except (AttributeError, KeyError) as err: