import logging
from collections import OrderedDict
+from datetime import datetime
from copy import deepcopy
from math import log
import pandas as pd
import plotly.offline as ploff
import plotly.graph_objs as plgo
+import plotly.exceptions as plerr
from plotly.exceptions import PlotlyError
u"plot_hdrh_lat_by_percentile": plot_hdrh_lat_by_percentile,
u"plot_hdrh_lat_by_percentile_x_log": plot_hdrh_lat_by_percentile_x_log,
u"plot_mrr_box_name": plot_mrr_box_name,
- u"plot_ndrpdr_box_name": plot_ndrpdr_box_name
+ u"plot_ndrpdr_box_name": plot_ndrpdr_box_name,
+ u"plot_statistics": plot_statistics
}
logging.info(u"Generating the plots ...")
logging.info(u"Done.")
+def plot_statistics(plot, input_data):
+ """Generate the plot(s) with algorithm: plot_statistics
+ specified in the specification file.
+
+ :param plot: Plot to generate.
+ :param input_data: Data to process.
+ :type plot: pandas.Series
+ :type input_data: InputData
+ """
+
+ data_x = list()
+ data_y_pass = list()
+ data_y_fail = list()
+ data_y_duration = list()
+ hover_text = list()
+ hover_str = (
+ u"date: {date}<br>"
+ u"passed: {passed}<br>"
+ u"failed: {failed}<br>"
+ u"duration: {duration}<br>"
+ u"{sut}-ref: {build}<br>"
+ u"csit-ref: {test}-{period}-build-{build_nr}<br>"
+ u"testbed: {testbed}"
+ )
+ for job, builds in plot[u"data"].items():
+ for build_nr in builds:
+ try:
+ meta = input_data.metadata(job, str(build_nr))
+ generated = meta[u"generated"]
+ date = datetime(
+ int(generated[0:4]),
+ int(generated[4:6]),
+ int(generated[6:8]),
+ int(generated[9:11]),
+ int(generated[12:])
+ )
+ d_y_pass = meta[u"tests_passed"]
+ d_y_fail = meta[u"tests_failed"]
+ minutes = meta[u"elapsedtime"] // 60000
+ duration = f"{(minutes // 60):02d}:{(minutes % 60):02d}"
+ version = meta.get(u"version", u"")
+ except (KeyError, IndexError, ValueError, AttributeError):
+ continue
+ data_x.append(date)
+ data_y_pass.append(d_y_pass)
+ data_y_fail.append(d_y_fail)
+ data_y_duration.append(minutes)
+ if u"vpp" in job:
+ sut = u"vpp"
+ elif u"dpdk" in job:
+ sut = u"dpdk"
+ elif u"trex" in job:
+ sut = u"trex"
+ else:
+ sut = u""
+ hover_text.append(hover_str.format(
+ date=date,
+ passed=d_y_pass,
+ failed=d_y_fail,
+ duration=duration,
+ sut=sut,
+ build=version,
+ test=u"mrr" if u"mrr" in job else u"ndrpdr",
+ period=u"daily" if u"daily" in job else u"weekly",
+ build_nr=build_nr,
+ testbed=meta.get(u"testbed", u"")
+ ))
+
+ traces = [
+ plgo.Bar(
+ x=data_x,
+ y=data_y_pass,
+ name=u"Passed",
+ text=hover_text,
+ hoverinfo=u"text"
+ ),
+ plgo.Bar(
+ x=data_x,
+ y=data_y_fail,
+ name=u"Failed",
+ text=hover_text,
+ hoverinfo=u"text"),
+ plgo.Scatter(
+ x=data_x,
+ y=data_y_duration,
+ name=u"Duration",
+ yaxis=u"y2",
+ text=hover_text,
+ hoverinfo=u"text"
+ )
+ ]
+
+ name_file = f"{plot[u'output-file']}.html"
+
+ logging.info(f" Writing the file {name_file}")
+ plpl = plgo.Figure(data=traces, layout=plot[u"layout"])
+ tickvals = [0, (max(data_y_duration) // 60) * 60]
+ step = tickvals[1] / 5
+ for i in range(5):
+ tickvals.append(int(tickvals[0] + step * (i + 1)))
+ plpl.update_layout(
+ yaxis2=dict(
+ title=u"Duration [hh:mm]",
+ anchor=u"x",
+ overlaying=u"y",
+ side=u"right",
+ rangemode="tozero",
+ tickmode=u"array",
+ tickvals=tickvals,
+ ticktext=[f"{(val // 60):02d}:{(val % 60):02d}" for val in tickvals]
+ )
+ )
+ plpl.update_layout(barmode=u"stack")
+ try:
+ ploff.plot(
+ plpl,
+ show_link=False,
+ auto_open=False,
+ filename=name_file
+ )
+ except plerr.PlotlyEmptyDataError:
+ logging.warning(u"No data for the plot. Skipped.")
+
+
def plot_hdrh_lat_by_percentile(plot, input_data):
"""Generate the plot(s) with algorithm: plot_hdrh_lat_by_percentile
specified in the specification file.
)
test_type = u"HOSTSTACK"
+ elif test[u"type"] in (u"LDP_NGINX",):
+ if u"TCP_CPS" in test[u"tags"]:
+ test_type = u"VSAP_CPS"
+ y_vals[test[u"parent"]].append(
+ test[u"result"][u"cps"]
+ )
+ elif u"TCP_RPS" in test[u"tags"]:
+ test_type = u"VSAP_RPS"
+ y_vals[test[u"parent"]].append(
+ test[u"result"][u"rps"]
+ )
+ else:
+ continue
else:
continue
tst_name = re.sub(REGEX_NIC, u"",
col.lower().replace(u'-ndrpdr', u'').
replace(u'2n1l-', u''))
+ if test_type in (u"VSAP_CPS", u"VSAP_RPS"):
+ data_y = [y if y else None for y in df_y[col]]
+ else:
+ data_y = [y / 1e6 if y else None for y in df_y[col]]
kwargs = dict(
- x=[str(i + 1) + u'.'] * len(df_y[col]),
- y=[y / 1e6 if y else None for y in df_y[col]],
+ y=data_y,
name=(
f"{i + 1}. "
f"({nr_of_samples[i]:02d} "
)
if test_type in (u"SOAK", ):
kwargs[u"boxpoints"] = u"all"
+ kwargs[u"jitter"] = 0.3
traces.append(plgo.Box(**kwargs))
try:
val_max = max(df_y[col])
if val_max:
- y_max.append(int(val_max / 1e6) + 2)
+ y_max.append(int(val_max / 1e6))
except (ValueError, TypeError) as err:
logging.error(repr(err))
continue
try:
# Create plot
layout = deepcopy(plot[u"layout"])
+ layout[u"xaxis"][u"tickvals"] = [i for i in range(len(y_vals))]
+ layout[u"xaxis"][u"ticktext"] = [str(i + 1) for i in range(len(y_vals))]
if layout.get(u"title", None):
if test_type in (u"HOSTSTACK", ):
layout[u"title"] = f"<b>Bandwidth:</b> {layout[u'title']}"
+ elif test_type == u"VSAP_CPS":
+ layout[u"title"] = f"<b>CPS:</b> {layout[u'title']}"
+ layout[u"yaxis"][u"title"] = u"<b>Connection Rate [cps]</b>"
+ elif test_type == u"VSAP_RPS":
+ layout[u"title"] = f"<b>RPS:</b> {layout[u'title']}"
+ layout[u"yaxis"][u"title"] = u"<b>Connection Rate [rps]</b>"
else:
- layout[u"title"] = f"<b>Throughput:</b> {layout[u'title']}"
- if y_max:
- layout[u"yaxis"][u"range"] = [0, max(y_max)]
+ layout[u"title"] = f"<b>Tput:</b> {layout[u'title']}"
+ if y_max and max(y_max) > 1:
+ layout[u"yaxis"][u"range"] = [0, max(y_max) + 2]
plpl = plgo.Figure(data=traces, layout=layout)
# Export Plot
REGEX_NIC, u'', key.lower().replace(u'-ndrpdr', u'').
replace(u'2n1l-', u'')
)
- traces.append(
- plgo.Box(
- x=[data_x[idx], ] * len(data_x),
- y=[y / 1e6 if y else None for y in vals],
- name=(
- f"{idx+1}."
- f"({len(vals):02d} "
- f"run"
- f"{u's' if len(vals) > 1 else u''}) "
- f"{name}"
- ),
- hoverinfo=u"y+name"
- )
+ kwargs = dict(
+ y=[y / 1e6 if y else None for y in vals],
+ name=(
+ f"{idx + 1}."
+ f"({len(vals):02d} "
+ f"run"
+ f"{u's' if len(vals) > 1 else u''}) "
+ f"{name}"
+ ),
+ hoverinfo=u"y+name"
)
+ box_points = plot.get(u"boxpoints", None)
+ if box_points and box_points in \
+ (u"all", u"outliers", u"suspectedoutliers", False):
+ kwargs[u"boxpoints"] = box_points
+ kwargs[u"jitter"] = 0.3
+ traces.append(plgo.Box(**kwargs))
try:
data_y_max.append(max(vals))
except ValueError as err:
try:
# Create plot
layout = deepcopy(plot[u"layout"])
+ layout[u"xaxis"][u"tickvals"] = [i for i in range(len(data_y))]
+ layout[u"xaxis"][u"ticktext"] = \
+ [str(i + 1) for i in range(len(data_y))]
if layout.get(u"title", None):
layout[u"title"] = \
layout[u'title'].format(core=core, test_type=ttype)
layout[u"title"] = f"<b>CPS:</b> {layout[u'title']}"
else:
layout[u"title"] = \
- f"<b>Throughput:</b> {layout[u'title']}"
+ f"<b>Tput:</b> {layout[u'title']}"
if data_y_max:
layout[u"yaxis"][u"range"] = [0, max(data_y_max) / 1e6 + 1]
plpl = plgo.Figure(data=traces, layout=layout)
# Add plot traces
traces = list()
for idx, x_item in enumerate(data_x):
- traces.append(
- plgo.Box(
- x=[x_item, ] * len(data_y[idx]),
- y=data_y[idx],
- name=data_names[idx],
- hoverinfo=u"y+name"
- )
+ kwargs = dict(
+ y=data_y[idx],
+ name=data_names[idx],
+ hoverinfo=u"y+name"
)
+ box_points = plot.get(u"boxpoints", None)
+ if box_points and box_points in \
+ (u"all", u"outliers", u"suspectedoutliers", False):
+ kwargs[u"boxpoints"] = box_points
+ kwargs["jitter"] = 0.3
+ traces.append(plgo.Box(**kwargs))
try:
# Create plot
layout = deepcopy(plot[u"layout"])
+ layout[u"xaxis"][u"tickvals"] = [i for i in range(len(data_y))]
+ layout[u"xaxis"][u"ticktext"] = \
+ [str(i + 1) for i in range(len(data_y))]
if layout.get(u"title", None):
layout[u"title"] = (
- f"<b>Throughput:</b> {layout[u'title'].format(core=core)}"
+ f"<b>Tput:</b> {layout[u'title'].format(core=core)}"
)
if data_y_max:
layout[u"yaxis"][u"range"] = [0, max(data_y_max) + 1]
regex_test_name = re.compile(r'^.*-(\d+ch|\d+pl)-'
r'(\d+mif|\d+vh)-'
r'(\d+vm\d+t|\d+dcr\d+t|\d+dcr\d+c).*$')
- vals = dict()
-
# Transform the data
logging.info(
f" Creating the data set for the {plot.get(u'type', u'')} "
for ttype in plot.get(u"test-type", (u"ndr", u"pdr")):
for core in plot.get(u"core", tuple()):
+ vals = dict()
for item in plot.get(u"include", tuple()):
reg_ex = re.compile(str(item.format(core=core)).lower())
for job in in_data:
Code Review / csit.git / blobdiff