Code Review
/
csit.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
review
|
tree
raw
|
inline
| side by side
Report: data
[csit.git]
/
resources
/
tools
/
presentation
/
generator_CPTA.py
diff --git
a/resources/tools/presentation/generator_CPTA.py
b/resources/tools/presentation/generator_CPTA.py
index
567b889
..
e27a521
100644
(file)
--- a/
resources/tools/presentation/generator_CPTA.py
+++ b/
resources/tools/presentation/generator_CPTA.py
@@
-144,55
+144,53
@@
def _select_data(in_data, period, fill_missing=False, use_first=False):
return OrderedDict(sorted(data_dict.items(), key=lambda t: t[0]))
return OrderedDict(sorted(data_dict.items(), key=lambda t: t[0]))
-def _evaluate_results(
in_data,
trimmed_data, window=10):
+def _evaluate_results(trimmed_data, window=10):
"""Evaluates if the sample value is regress, normal or progress compared to
previous data within the window.
We use the intervals defined as:
"""Evaluates if the sample value is regress, normal or progress compared to
previous data within the window.
We use the intervals defined as:
- - regress: less than median - 3 * stdev
- - normal: between median - 3 * stdev and median + 3 * stdev
- - progress: more than median + 3 * stdev
+ - regress: less than trimmed moving median - 3 * stdev
+ - normal: between trimmed moving median - 3 * stdev and median + 3 * stdev
+ - progress: more than trimmed moving median + 3 * stdev
+ where stdev is trimmed moving standard deviation.
- :param in_data: Full data set.
- :param trimmed_data: Full data set without the outliers.
- :param window: Window size used to calculate moving median and moving stdev.
- :type in_data: pandas.Series
+ :param trimmed_data: Full data set with the outliers replaced by nan.
+ :param window: Window size used to calculate moving average and moving stdev.
:type trimmed_data: pandas.Series
:type window: int
:returns: Evaluated results.
:rtype: list
"""
:type trimmed_data: pandas.Series
:type window: int
:returns: Evaluated results.
:rtype: list
"""
- if len(
in
_data) > 2:
- win_size =
in_data.size if in
_data.size < window else window
+ if len(
trimmed
_data) > 2:
+ win_size =
trimmed_data.size if trimmed
_data.size < window else window
results = [0.66, ]
results = [0.66, ]
-
median = in
_data.rolling(window=win_size, min_periods=2).median()
-
stdev_t
= trimmed_data.rolling(window=win_size, min_periods=2).std()
+
tmm = trimmed
_data.rolling(window=win_size, min_periods=2).median()
+
tmstd
= trimmed_data.rolling(window=win_size, min_periods=2).std()
first = True
first = True
- for build_nr, value in
in
_data.iteritems():
+ for build_nr, value in
trimmed
_data.iteritems():
if first:
first = False
continue
if first:
first = False
continue
- if np.isnan(trimmed_data[build_nr]) \
- or np.isnan(median[build_nr]) \
- or np.isnan(stdev_t[build_nr]) \
- or np.isnan(value):
+ if (np.isnan(value)
+ or np.isnan(tmm[build_nr])
+ or np.isnan(tmstd[build_nr])):
results.append(0.0)
results.append(0.0)
- elif value < (
median[build_nr] - 3 * stdev_t
[build_nr]):
+ elif value < (
tmm[build_nr] - 3 * tmstd
[build_nr]):
results.append(0.33)
results.append(0.33)
- elif value > (
median[build_nr] + 3 * stdev_t
[build_nr]):
+ elif value > (
tmm[build_nr] + 3 * tmstd
[build_nr]):
results.append(1.0)
else:
results.append(0.66)
else:
results = [0.0, ]
try:
results.append(1.0)
else:
results.append(0.66)
else:
results = [0.0, ]
try:
-
median = np.median(in
_data)
-
stdev = np.std(in
_data)
- if
in_data.values[-1] < (median - 3 * stdev
):
+
tmm = np.median(trimmed
_data)
+
tmstd = np.std(trimmed
_data)
+ if
trimmed_data.values[-1] < (tmm - 3 * tmstd
):
results.append(0.33)
results.append(0.33)
- elif (
median - 3 * stdev) <= in
_data.values[-1] <= (
-
median + 3 * stdev
):
+ elif (
tmm - 3 * tmstd) <= trimmed
_data.values[-1] <= (
+
tmm + 3 * tmstd
):
results.append(0.66)
else:
results.append(1.0)
results.append(0.66)
else:
results.append(1.0)
@@
-203,10
+201,10
@@
def _evaluate_results(in_data, trimmed_data, window=10):
def _generate_trending_traces(in_data, build_info, period, moving_win_size=10,
fill_missing=True, use_first=False,
def _generate_trending_traces(in_data, build_info, period, moving_win_size=10,
fill_missing=True, use_first=False,
- show_
moving_median
=True, name="", color=""):
+ show_
trend_line
=True, name="", color=""):
"""Generate the trending traces:
- samples,
"""Generate the trending traces:
- samples,
- -
moving median (trending plot
)
+ -
trimmed moving median (trending line
)
- outliers, regress, progress
:param in_data: Full data set.
- outliers, regress, progress
:param in_data: Full data set.
@@
-214,9
+212,9
@@
def _generate_trending_traces(in_data, build_info, period, moving_win_size=10,
:param period: Sampling period.
:param moving_win_size: Window size.
:param fill_missing: If the chosen sample is missing in the full set, its
:param period: Sampling period.
:param moving_win_size: Window size.
:param fill_missing: If the chosen sample is missing in the full set, its
- nearest neighbour is used.
+
nearest neighbour is used.
:param use_first: Use the first sample even though it is not chosen.
:param use_first: Use the first sample even though it is not chosen.
- :param show_
moving_median
: Show moving median (trending plot).
+ :param show_
trend_line
: Show moving median (trending plot).
:param name: Name of the plot
:param color: Name of the color for the plot.
:type in_data: OrderedDict
:param name: Name of the plot
:param color: Name of the color for the plot.
:type in_data: OrderedDict
@@
-225,7
+223,7
@@
def _generate_trending_traces(in_data, build_info, period, moving_win_size=10,
:type moving_win_size: int
:type fill_missing: bool
:type use_first: bool
:type moving_win_size: int
:type fill_missing: bool
:type use_first: bool
- :type show_
moving_median
: bool
+ :type show_
trend_line
: bool
:type name: str
:type color: str
:returns: Generated traces (list) and the evaluated result (float).
:type name: str
:type color: str
:returns: Generated traces (list) and the evaluated result (float).
@@
-237,8
+235,8
@@
def _generate_trending_traces(in_data, build_info, period, moving_win_size=10,
fill_missing=fill_missing,
use_first=use_first)
fill_missing=fill_missing,
use_first=use_first)
- data_x =
[key for key in in_data.keys()]
- data_y =
[val for val in in_data.values()]
+ data_x =
list(in_data.keys())
+ data_y =
list(in_data.values())
hover_text = list()
for idx in data_x:
hover_text = list()
for idx in data_x:
@@
-249,7
+247,7
@@
def _generate_trending_traces(in_data, build_info, period, moving_win_size=10,
t_data, outliers = split_outliers(data_pd, outlier_const=1.5,
window=moving_win_size)
t_data, outliers = split_outliers(data_pd, outlier_const=1.5,
window=moving_win_size)
- results = _evaluate_results(
data_pd,
t_data, window=moving_win_size)
+ results = _evaluate_results(t_data, window=moving_win_size)
anomalies = pd.Series()
anomalies_res = list()
anomalies = pd.Series()
anomalies_res = list()
@@
-328,12
+326,12
@@
def _generate_trending_traces(in_data, build_info, period, moving_win_size=10,
)
traces.append(trace_anomalies)
)
traces.append(trace_anomalies)
- if show_
moving_median
:
- data_
mean_y = pd.Series(data_y).rolling(
-
window=moving_win_size,
min_periods=2).median()
- trace_
median
= plgo.Scatter(
- x=data_
x
,
- y=data_
mean_y
,
+ if show_
trend_line
:
+ data_
trend = t_data.rolling(window=moving_win_size,
+
min_periods=2).median()
+ trace_
trend
= plgo.Scatter(
+ x=data_
trend.keys()
,
+ y=data_
trend.tolist()
,
mode='lines',
line={
"shape": "spline",
mode='lines',
line={
"shape": "spline",
@@
-342,7
+340,7
@@
def _generate_trending_traces(in_data, build_info, period, moving_win_size=10,
},
name='{name}-trend'.format(name=name)
)
},
name='{name}-trend'.format(name=name)
)
- traces.append(trace_
median
)
+ traces.append(trace_
trend
)
return traces, results[-1]
return traces, results[-1]
@@
-437,7
+435,8
@@
def _generate_all_charts(spec, input_data):
tst_lst = list()
for build in builds_lst:
item = tst_data.get(int(build), '')
tst_lst = list()
for build in builds_lst:
item = tst_data.get(int(build), '')
- tst_lst.append(str(item) if item else '')
+ tst_lst.append(str(item))
+ # tst_lst.append(str(item) if item else '')
csv_table.append("{0},".format(tst_name) + ",".join(tst_lst) + '\n')
for period in chart["periods"]:
csv_table.append("{0},".format(tst_name) + ",".join(tst_lst) + '\n')
for period in chart["periods"]: