X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_plots.py;h=1b95030818c4f31db2563951ca878f3e1fe2fe44;hp=fb1b4734cfad5c1be1f15a8056b5d48e90621176;hb=8e1dfc4ab336302999f91ea0386093b0bc879974;hpb=c763cfcb064e4f4acf6b8309b08d3800b9bd5331

diff --git a/resources/tools/presentation/generator_plots.py b/resources/tools/presentation/generator_plots.py
index fb1b4734cf..1b95030818 100644
--- a/resources/tools/presentation/generator_plots.py
+++ b/resources/tools/presentation/generator_plots.py
@@ -19,6 +19,7 @@ import re
 import logging
 
 from collections import OrderedDict
+from datetime import datetime
 from copy import deepcopy
 from math import log
 
@@ -27,6 +28,7 @@ import hdrh.codec
 import pandas as pd
 import plotly.offline as ploff
 import plotly.graph_objs as plgo
+import plotly.exceptions as plerr
 
 from plotly.exceptions import PlotlyError
 
@@ -84,7 +86,8 @@ def generate_plots(spec, data):
         u"plot_hdrh_lat_by_percentile": plot_hdrh_lat_by_percentile,
         u"plot_hdrh_lat_by_percentile_x_log": plot_hdrh_lat_by_percentile_x_log,
         u"plot_mrr_box_name": plot_mrr_box_name,
-        u"plot_ndrpdr_box_name": plot_ndrpdr_box_name
+        u"plot_ndrpdr_box_name": plot_ndrpdr_box_name,
+        u"plot_statistics": plot_statistics
     }
 
     logging.info(u"Generating the plots ...")
@@ -102,6 +105,122 @@ def generate_plots(spec, data):
     logging.info(u"Done.")
 
 
+def plot_statistics(plot, input_data):
+    """Generate the plot(s) with algorithm: plot_statistics
+    specified in the specification file.
+
+    :param plot: Plot to generate.
+    :param input_data: Data to process.
+    :type plot: pandas.Series
+    :type input_data: InputData
+    """
+
+    data_x = list()
+    data_y_pass = list()
+    data_y_fail = list()
+    data_y_duration = list()
+    hover_text = list()
+    hover_str = (
+        u"date: {date}<br>"
+        u"passed: {passed}<br>"
+        u"failed: {failed}<br>"
+        u"duration: {duration}<br>"
+        u"{sut}-ref: {build}<br>"
+        u"csit-ref: {test}-{period}-build-{build_nr}<br>"
+        u"testbed: {testbed}"
+    )
+    for job, builds in plot[u"data"].items():
+        for build_nr in builds:
+            try:
+                meta = input_data.metadata(job, str(build_nr))
+                generated = meta[u"generated"]
+                date = datetime(
+                    int(generated[0:4]),
+                    int(generated[4:6]),
+                    int(generated[6:8]),
+                    int(generated[9:11]),
+                    int(generated[12:])
+                )
+                d_y_pass = meta[u"tests_passed"]
+                d_y_fail = meta[u"tests_failed"]
+                minutes = meta[u"elapsedtime"] // 60000
+                duration = f"{(minutes // 60):02d}:{(minutes % 60):02d}"
+                version = meta[u"version"]
+            except (KeyError, IndexError, ValueError, AttributeError):
+                continue
+            data_x.append(date)
+            data_y_pass.append(d_y_pass)
+            data_y_fail.append(d_y_fail)
+            data_y_duration.append(minutes)
+            hover_text.append(hover_str.format(
+                date=date,
+                passed=d_y_pass,
+                failed=d_y_fail,
+                duration=duration,
+                sut=u"vpp" if u"vpp" in job else u"dpdk",
+                build=version,
+                test=u"mrr" if u"mrr" in job else u"ndrpdr",
+                period=u"daily" if u"daily" in job else u"weekly",
+                build_nr=build_nr,
+                testbed=meta[u"testbed"]
+            ))
+
+    traces = [
+        plgo.Bar(
+            x=data_x,
+            y=data_y_pass,
+            name=u"Passed",
+            text=hover_text,
+            hoverinfo=u"text"
+        ),
+        plgo.Bar(
+            x=data_x,
+            y=data_y_fail,
+            name=u"Failed",
+            text=hover_text,
+            hoverinfo=u"text"),
+        plgo.Scatter(
+            x=data_x,
+            y=data_y_duration,
+            name=u"Duration",
+            yaxis=u"y2",
+            text=hover_text,
+            hoverinfo=u"text"
+        )
+    ]
+
+    name_file = f"{plot[u'output-file']}.html"
+
+    logging.info(f"    Writing the file {name_file}")
+    plpl = plgo.Figure(data=traces, layout=plot[u"layout"])
+    tickvals = [0, (max(data_y_duration) // 60) * 60]
+    step = tickvals[1] / 5
+    for i in range(5):
+        tickvals.append(int(tickvals[0] + step * (i + 1)))
+    plpl.update_layout(
+        yaxis2=dict(
+            title=u"Duration [hh:mm]",
+            anchor=u"x",
+            overlaying=u"y",
+            side=u"right",
+            rangemode="tozero",
+            tickmode=u"array",
+            tickvals=tickvals,
+            ticktext=[f"{(val // 60):02d}:{(val % 60):02d}" for val in tickvals]
+        )
+    )
+    plpl.update_layout(barmode=u"stack")
+    try:
+        ploff.plot(
+            plpl,
+            show_link=False,
+            auto_open=False,
+            filename=name_file
+        )
+    except plerr.PlotlyEmptyDataError:
+        logging.warning(u"No data for the plot. Skipped.")
+
+
 def plot_hdrh_lat_by_percentile(plot, input_data):
     """Generate the plot(s) with algorithm: plot_hdrh_lat_by_percentile
     specified in the specification file.
@@ -606,6 +725,19 @@ def plot_perf_box_name(plot, input_data):
                                 )
                             test_type = u"HOSTSTACK"
 
+                        elif test[u"type"] in (u"LDP_NGINX",):
+                            if u"TCP_CPS" in test[u"tags"]:
+                                test_type = u"VSAP_CPS"
+                                y_vals[test[u"parent"]].append(
+                                    test[u"result"][u"cps"]
+                                )
+                            elif u"TCP_RPS" in test[u"tags"]:
+                                test_type = u"VSAP_RPS"
+                                y_vals[test[u"parent"]].append(
+                                    test[u"result"][u"rps"]
+                                )
+                            else:
+                                continue
                         else:
                             continue
 
@@ -632,9 +764,13 @@ def plot_perf_box_name(plot, input_data):
         tst_name = re.sub(REGEX_NIC, u"",
                           col.lower().replace(u'-ndrpdr', u'').
                           replace(u'2n1l-', u''))
+        if test_type in (u"VSAP_CPS", u"VSAP_RPS"):
+            data_y = [y if y else None for y in df_y[col]]
+        else:
+            data_y = [y / 1e6 if y else None for y in df_y[col]]
         kwargs = dict(
             x=[str(i + 1) + u'.'] * len(df_y[col]),
-            y=[y / 1e6 if y else None for y in df_y[col]],
+            y=data_y,
             name=(
                 f"{i + 1}. "
                 f"({nr_of_samples[i]:02d} "
@@ -651,7 +787,7 @@ def plot_perf_box_name(plot, input_data):
         try:
             val_max = max(df_y[col])
             if val_max:
-                y_max.append(int(val_max / 1e6) + 2)
+                y_max.append(int(val_max / 1e6))
         except (ValueError, TypeError) as err:
             logging.error(repr(err))
             continue
@@ -662,10 +798,16 @@ def plot_perf_box_name(plot, input_data):
         if layout.get(u"title", None):
             if test_type in (u"HOSTSTACK", ):
                 layout[u"title"] = f"<b>Bandwidth:</b> {layout[u'title']}"
+            elif test_type == u"VSAP_CPS":
+                layout[u"title"] = f"<b>CPS:</b> {layout[u'title']}"
+                layout[u"yaxis"][u"title"] = u"<b>Connection Rate [cps]</b>"
+            elif test_type == u"VSAP_RPS":
+                layout[u"title"] = f"<b>RPS:</b> {layout[u'title']}"
+                layout[u"yaxis"][u"title"] = u"<b>Connection Rate [rps]</b>"
             else:
-                layout[u"title"] = f"<b>Throughput:</b> {layout[u'title']}"
-        if y_max:
-            layout[u"yaxis"][u"range"] = [0, max(y_max)]
+                layout[u"title"] = f"<b>Tput:</b> {layout[u'title']}"
+        if y_max and max(y_max) > 1:
+            layout[u"yaxis"][u"range"] = [0, max(y_max) + 2]
         plpl = plgo.Figure(data=traces, layout=layout)
 
         # Export Plot
@@ -777,7 +919,7 @@ def plot_ndrpdr_box_name(plot, input_data):
                         layout[u"title"] = f"<b>CPS:</b> {layout[u'title']}"
                     else:
                         layout[u"title"] = \
-                            f"<b>Throughput:</b> {layout[u'title']}"
+                            f"<b>Tput:</b> {layout[u'title']}"
                 if data_y_max:
                     layout[u"yaxis"][u"range"] = [0, max(data_y_max) / 1e6 + 1]
                 plpl = plgo.Figure(data=traces, layout=layout)
@@ -872,7 +1014,7 @@ def plot_mrr_box_name(plot, input_data):
             layout = deepcopy(plot[u"layout"])
             if layout.get(u"title", None):
                 layout[u"title"] = (
-                    f"<b>Throughput:</b> {layout[u'title'].format(core=core)}"
+                    f"<b>Tput:</b> {layout[u'title'].format(core=core)}"
                 )
             if data_y_max:
                 layout[u"yaxis"][u"range"] = [0, max(data_y_max) + 1]