Trending: weekly comparison - workaround

[csit.git] / resources / tools / presentation / generator_tables.py

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index b8ceffa..b09b820 100644 (file)
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -39,7 +39,6 @@ from pal_utils import mean, stdev, classify_anomalies, \
 
 
 REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
-REGEX_TOPO_ARCH = re.compile(r'^(\dn-.{3})')
 
 NORM_FREQ = 2.0  # [GHz]
 
@@ -1740,8 +1739,6 @@ def table_comparison(table, input_data):
         f"{table.get('title', '')}."
     )
 
-    normalize = table.get('normalize', False)
-
     columns = table.get("columns", None)
     if not columns:
         logging.error(
@@ -1872,15 +1869,7 @@ def table_comparison(table, input_data):
     for tst_data in tbl_dict.values():
         row = [tst_data[u"name"], ]
         for col in cols:
-            row_data = tst_data.get(col["title"], None)
-            if normalize and row_data:
-                groups = re.search(REGEX_TOPO_ARCH, col["title"])
-                topo_arch = groups.group(0) if groups else ""
-                norm_factor = table["norm_factor"].get(topo_arch, 1.0)
-                row_data_norm = row_data * norm_factor
-            else:
-                row_data_norm = row_data
-            row.append(row_data_norm)
+            row.append(tst_data.get(col[u"title"], None))
         tbl_lst.append(row)
 
     comparisons = table.get("comparisons", None)
@@ -1937,10 +1926,20 @@ def table_comparison(table, input_data):
                         cmp_itm["mean"] is not None and \
                         ref_itm["stdev"] is not None and \
                         cmp_itm["stdev"] is not None:
+                    norm_factor_ref = table["norm_factor"].get(
+                        comp.get("norm-ref", ""),
+                        1.0
+                    )
+                    norm_factor_cmp = table["norm_factor"].get(
+                        comp.get("norm-cmp", ""),
+                        1.0
+                    )
                     try:
                         delta, d_stdev = relative_change_stdev(
-                            ref_itm["mean"], cmp_itm["mean"],
-                            ref_itm["stdev"], cmp_itm["stdev"]
+                            ref_itm["mean"] * norm_factor_ref,
+                            cmp_itm["mean"] * norm_factor_cmp,
+                            ref_itm["stdev"] * norm_factor_ref,
+                            cmp_itm["stdev"] * norm_factor_cmp
                         )
                     except ZeroDivisionError:
                         break
@@ -2177,8 +2176,10 @@ def table_weekly_comparison(table, in_data):
             header[1].insert(
                 1, in_data.metadata(job_name, build_nr).get(u"generated", u"")
             )
+            logging.info(
+                in_data.metadata(job_name, build_nr).get(u"version", u"ERROR"))
             header[0].insert(
-                1, in_data.metadata(job_name, build_nr).get(u"version", u"")
+                1, in_data.metadata(job_name, build_nr).get("version", build_nr)
             )
 
             for tst_name, tst_data in build.items():
@@ -2220,9 +2221,7 @@ def table_weekly_comparison(table, in_data):
             if ref_data is None or cmp_data is None:
                 cmp_dict[tst_name].append(float(u'nan'))
             else:
-                cmp_dict[tst_name].append(
-                    relative_change(ref_data, cmp_data)
-                )
+                cmp_dict[tst_name].append(relative_change(ref_data, cmp_data))
 
     tbl_lst_none = list()
     tbl_lst = list()
@@ -2266,17 +2265,31 @@ def table_weekly_comparison(table, in_data):
 
     txt_file_name = f"{table[u'output-file']}.txt"
     logging.info(f"    Writing the file {txt_file_name}")
-    convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u",")
+    try:
+        convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u",")
+    except Exception as err:
+        logging.error(repr(err))
+        for hdr in header:
+            logging.info(",".join(hdr))
+        for test in tbl_lst:
+            logging.info(",".join(
+                [
+                    str(item).replace(u"None", u"-").replace(u"nan", u"-").
+                    replace(u"null", u"-") for item in test
+                ]
+            ))
 
     # Reorganize header in txt table
     txt_table = list()
-    with open(txt_file_name, u"rt", encoding='utf-8') as file_handler:
-        for line in list(file_handler):
-            txt_table.append(line)
     try:
+        with open(txt_file_name, u"rt", encoding='utf-8') as file_handler:
+            for line in list(file_handler):
+                txt_table.append(line)
         txt_table.insert(5, txt_table.pop(2))
         with open(txt_file_name, u"wt", encoding='utf-8') as file_handler:
             file_handler.writelines(txt_table)
+    except FileNotFoundError as err:
+        logging.error(repr(err))
     except IndexError:
         pass