CSIT-1041: Trending dashboard

[csit.git] / resources / tools / presentation / generator_CPTA.py

diff --git a/resources/tools/presentation/generator_CPTA.py b/resources/tools/presentation/generator_CPTA.py
index 69c52d4..54679a2 100644 (file)
--- a/resources/tools/presentation/generator_CPTA.py
+++ b/resources/tools/presentation/generator_CPTA.py
@@ -197,7 +197,7 @@ def _evaluate_results(in_data, trimmed_data, window=10):
     return results
 
 
-def _generate_trending_traces(in_data, period, moving_win_size=10,
+def _generate_trending_traces(in_data, build_info, period, moving_win_size=10,
                               fill_missing=True, use_first=False,
                               show_moving_median=True, name="", color=""):
     """Generate the trending traces:
@@ -206,6 +206,7 @@ def _generate_trending_traces(in_data, period, moving_win_size=10,
      - outliers, regress, progress
 
     :param in_data: Full data set.
+    :param build_info: Information about the builds.
     :param period: Sampling period.
     :param moving_win_size: Window size.
     :param fill_missing: If the chosen sample is missing in the full set, its
@@ -215,6 +216,7 @@ def _generate_trending_traces(in_data, period, moving_win_size=10,
     :param name: Name of the plot
     :param color: Name of the color for the plot.
     :type in_data: OrderedDict
+    :type build_info: dict
     :type period: int
     :type moving_win_size: int
     :type fill_missing: bool
@@ -230,8 +232,15 @@ def _generate_trending_traces(in_data, period, moving_win_size=10,
         in_data = _select_data(in_data, period,
                                fill_missing=fill_missing,
                                use_first=use_first)
+    try:
+        data_x = ["{0}/{1}".format(key, build_info[str(key)][1].split("~")[-1])
+                  for key in in_data.keys()]
+    except KeyError:
+        data_x = [key for key in in_data.keys()]
+    # hover_text = ["vpp-build: {0}".format(x[1].split("~")[-1])
+    #               for x in build_info.values()]
+    # data_x = [key for key in in_data.keys()]
 
-    data_x = [key for key in in_data.keys()]
     data_y = [val for val in in_data.values()]
     data_pd = pd.Series(data_y, index=data_x)
 
@@ -242,7 +251,12 @@ def _generate_trending_traces(in_data, period, moving_win_size=10,
     anomalies = pd.Series()
     anomalies_res = list()
     for idx, item in enumerate(in_data.items()):
-        item_pd = pd.Series([item[1], ], index=[item[0], ])
+        item_pd = pd.Series([item[1], ],
+                            index=["{0}/{1}".
+                            format(item[0],
+                                   build_info[str(item[0])][1].split("~")[-1]),
+                                   ])
+        #item_pd = pd.Series([item[1], ], index=[item[0], ])
         if item[0] in outliers.keys():
             anomalies = anomalies.append(item_pd)
             anomalies_res.append(0.0)
@@ -274,6 +288,8 @@ def _generate_trending_traces(in_data, period, moving_win_size=10,
             "color": color,
             "symbol": "circle",
         },
+        # text=hover_text,
+        # hoverinfo="x+y+text+name"
     )
     traces = [trace_samples, ]
 
@@ -362,12 +378,35 @@ def _generate_all_charts(spec, input_data):
     :type input_data: InputData
     """
 
-    csv_table = list()
+    job_name = spec.cpta["data"].keys()[0]
+
+    builds_lst = list()
+    for build in spec.input["builds"][job_name]:
+        status = build["status"]
+        if status != "failed" and status != "not found":
+            builds_lst.append(str(build["build"]))
+
+    # Get "build ID": "date" dict:
+    build_info = dict()
+    for build in builds_lst:
+        try:
+            build_info[build] = (
+                input_data.metadata(job_name, build)["generated"][:14],
+                input_data.metadata(job_name, build)["version"]
+            )
+        except KeyError:
+            build_info[build] = ("", "")
+
     # Create the header:
-    builds = spec.cpta["data"].values()[0]
-    builds_lst = [str(build) for build in range(builds[0], builds[-1] + 1)]
+    csv_table = list()
     header = "Build Number:," + ",".join(builds_lst) + '\n'
     csv_table.append(header)
+    build_dates = [x[0] for x in build_info.values()]
+    header = "Build Date:," + ",".join(build_dates) + '\n'
+    csv_table.append(header)
+    vpp_versions = [x[1] for x in build_info.values()]
+    header = "VPP Version:," + ",".join(vpp_versions) + '\n'
+    csv_table.append(header)
 
     results = list()
     for chart in spec.cpta["plots"]:
@@ -413,6 +452,7 @@ def _generate_all_charts(spec, input_data):
                 test_name = test_name.split('.')[-1]
                 trace, result = _generate_trending_traces(
                     test_data,
+                    build_info=build_info,
                     period=period,
                     moving_win_size=win_size,
                     fill_missing=True,
@@ -424,6 +464,8 @@ def _generate_all_charts(spec, input_data):
                 idx += 1
 
             # Generate the chart:
+            chart["layout"]["xaxis"]["title"] = \
+                chart["layout"]["xaxis"]["title"].format(job=job_name)
             _generate_chart(traces,
                             chart["layout"],
                             file_name="{0}-{1}-{2}{3}".format(
@@ -442,19 +484,23 @@ def _generate_all_charts(spec, input_data):
     txt_table = None
     with open("{0}.csv".format(file_name), 'rb') as csv_file:
         csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
-        header = True
+        line_nr = 0
         for row in csv_content:
             if txt_table is None:
                 txt_table = prettytable.PrettyTable(row)
-                header = False
             else:
-                if not header:
+                if line_nr > 1:
                     for idx, item in enumerate(row):
                         try:
                             row[idx] = str(round(float(item) / 1000000, 2))
                         except ValueError:
                             pass
-                txt_table.add_row(row)
+                try:
+                    txt_table.add_row(row)
+                except Exception as err:
+                    logging.warning("Error occurred while generating TXT table:"
+                                    "\n{0}".format(err))
+            line_nr += 1
         txt_table.align["Build Number:"] = "l"
     with open("{0}.txt".format(file_name), "w") as txt_file:
         txt_file.write(str(txt_table))