Trending: Add graph with statistics

[csit.git] / resources / tools / presentation / generator_tables.py

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 8218084..a715ecb 100644 (file)
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -94,7 +94,7 @@ def table_oper_data_html(table, input_data):
     )
     data = input_data.filter_data(
         table,
-        params=[u"name", u"parent", u"show-run", u"type"],
+        params=[u"name", u"parent", u"telemetry-show-run", u"type"],
         continue_on_error=True
     )
     if data.empty:
@@ -147,7 +147,8 @@ def table_oper_data_html(table, input_data):
         )
         thead.text = u"\t"
 
-        if tst_data.get(u"show-run", u"No Data") == u"No Data":
+        if tst_data.get(u"telemetry-show-run", None) is None or \
+                isinstance(tst_data[u"telemetry-show-run"], str):
             trow = ET.SubElement(
                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
             )
@@ -177,7 +178,7 @@ def table_oper_data_html(table, input_data):
             u"Average Vector Size"
         )
 
-        for dut_data in tst_data[u"show-run"].values():
+        for dut_data in tst_data[u"telemetry-show-run"].values():
             trow = ET.SubElement(
                 tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
             )
@@ -188,39 +189,41 @@ def table_oper_data_html(table, input_data):
                 tcol.text = u"No Data"
                 continue
 
-            runtime = loads(dut_data[u"runtime"])
-
-            try:
-                threads_nr = len(runtime[0][u"clocks"])
-            except (IndexError, KeyError):
-                tcol.text = u"No Data"
-                continue
-
-            threads = OrderedDict({idx: list() for idx in range(threads_nr)})
-            for item in runtime:
-                for idx in range(threads_nr):
-                    if item[u"vectors"][idx] > 0:
-                        clocks = item[u"clocks"][idx] / item[u"vectors"][idx]
-                    elif item[u"calls"][idx] > 0:
-                        clocks = item[u"clocks"][idx] / item[u"calls"][idx]
-                    elif item[u"suspends"][idx] > 0:
-                        clocks = item[u"clocks"][idx] / item[u"suspends"][idx]
+            runtime = dict()
+            for item in dut_data[u"runtime"].get(u"data", tuple()):
+                tid = int(item[u"labels"][u"thread_id"])
+                if runtime.get(tid, None) is None:
+                    runtime[tid] = dict()
+                gnode = item[u"labels"][u"graph_node"]
+                if runtime[tid].get(gnode, None) is None:
+                    runtime[tid][gnode] = dict()
+                try:
+                    runtime[tid][gnode][item[u"name"]] = float(item[u"value"])
+                except ValueError:
+                    runtime[tid][gnode][item[u"name"]] = item[u"value"]
+
+            threads = dict({idx: list() for idx in range(len(runtime))})
+            for idx, run_data in runtime.items():
+                for gnode, gdata in run_data.items():
+                    if gdata[u"vectors"] > 0:
+                        clocks = gdata[u"clocks"] / gdata[u"vectors"]
+                    elif gdata[u"calls"] > 0:
+                        clocks = gdata[u"clocks"] / gdata[u"calls"]
+                    elif gdata[u"suspends"] > 0:
+                        clocks = gdata[u"clocks"] / gdata[u"suspends"]
                     else:
                         clocks = 0.0
-
-                    if item[u"calls"][idx] > 0:
-                        vectors_call = item[u"vectors"][idx] / item[u"calls"][
-                            idx]
+                    if gdata[u"calls"] > 0:
+                        vectors_call = gdata[u"vectors"] / gdata[u"calls"]
                     else:
                         vectors_call = 0.0
-
-                    if int(item[u"calls"][idx]) + int(item[u"vectors"][idx]) + \
-                        int(item[u"suspends"][idx]):
+                    if int(gdata[u"calls"]) + int(gdata[u"vectors"]) + \
+                            int(gdata[u"suspends"]):
                         threads[idx].append([
-                            item[u"name"],
-                            item[u"calls"][idx],
-                            item[u"vectors"][idx],
-                            item[u"suspends"][idx],
+                            gnode,
+                            int(gdata[u"calls"]),
+                            int(gdata[u"vectors"]),
+                            int(gdata[u"suspends"]),
                             clocks,
                             vectors_call
                         ])
@@ -387,14 +390,13 @@ def table_merged_details(table, input_data):
                         # Temporary solution: remove NDR results from message:
                         if bool(table.get(u'remove-ndr', False)):
                             try:
-                                col_data = col_data.split(u" |br| ", 1)[1]
+                                col_data = col_data.split(u"\n", 1)[1]
                             except IndexError:
                                 pass
                         col_data = col_data.replace(u'\n', u' |br| ').\
                             replace(u'\r', u'').replace(u'"', u"'")
                         col_data = f" |prein| {col_data} |preout| "
-                    elif column[u"data"].split(u" ")[1] in \
-                            (u"conf-history", u"show-run"):
+                    elif column[u"data"].split(u" ")[1] in (u"conf-history", ):
                         col_data = col_data.replace(u'\n', u' |br| ')
                         col_data = f" |prein| {col_data[:-5]} |preout| "
                     row_lst.append(f'"{col_data}"')
@@ -915,7 +917,11 @@ def table_perf_trending_dash(table, input_data):
         if len(data_t) < 2:
             continue
 
-        classification_lst, avgs, _ = classify_anomalies(data_t)
+        try:
+            classification_lst, avgs, _ = classify_anomalies(data_t)
+        except ValueError as err:
+            logging.info(f"{err} Skipping")
+            return
 
         win_size = min(len(data_t), table[u"window"])
         long_win_size = min(len(data_t), table[u"long-trend-window"])
@@ -1339,6 +1345,8 @@ def table_last_failed_tests(table, input_data):
             build = str(build)
             try:
                 version = input_data.metadata(job, build).get(u"version", u"")
+                duration = \
+                    input_data.metadata(job, build).get(u"elapsedtime", u"")
             except KeyError:
                 logging.error(f"Data for {job}: {build} is not present.")
                 return
@@ -1357,15 +1365,16 @@ def table_last_failed_tests(table, input_data):
                     continue
                 nic = groups.group(0)
                 failed_tests.append(f"{nic}-{tst_data[u'name']}")
-            tbl_list.append(str(passed))
-            tbl_list.append(str(failed))
+            tbl_list.append(passed)
+            tbl_list.append(failed)
+            tbl_list.append(duration)
             tbl_list.extend(failed_tests)
 
     file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
     logging.info(f"    Writing file: {file_name}")
     with open(file_name, u"wt") as file_handler:
         for test in tbl_list:
-            file_handler.write(test + u'\n')
+            file_handler.write(f"{test}\n")
 
 
 def table_failed_tests(table, input_data):