Code Review
/
csit.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
review
|
tree
raw
|
inline
| side by side
CSIT-1104: Trending: Speed-up plots generation
[csit.git]
/
resources
/
tools
/
presentation
/
generator_tables.py
diff --git
a/resources/tools/presentation/generator_tables.py
b/resources/tools/presentation/generator_tables.py
index
38439ba
..
5246952
100644
(file)
--- a/
resources/tools/presentation/generator_tables.py
+++ b/
resources/tools/presentation/generator_tables.py
@@
-21,9
+21,8
@@
import prettytable
import pandas as pd
from string import replace
import pandas as pd
from string import replace
-from math import isnan
from collections import OrderedDict
from collections import OrderedDict
-from numpy import nan
+from numpy import nan
, isnan
from xml.etree import ElementTree as ET
from errors import PresentationError
from xml.etree import ElementTree as ET
from errors import PresentationError
@@
-63,6
+62,8
@@
def table_details(table, input_data):
format(table.get("title", "")))
# Transform the data
format(table.get("title", "")))
# Transform the data
+ logging.info(" Creating the data set for the {0} '{1}'.".
+ format(table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table)
# Prepare the header of the tables
data = input_data.filter_data(table)
# Prepare the header of the tables
@@
-129,10
+130,14
@@
def table_merged_details(table, input_data):
format(table.get("title", "")))
# Transform the data
format(table.get("title", "")))
# Transform the data
+ logging.info(" Creating the data set for the {0} '{1}'.".
+ format(table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table)
data = input_data.merge_data(data)
data.sort_index(inplace=True)
data = input_data.filter_data(table)
data = input_data.merge_data(data)
data.sort_index(inplace=True)
+ logging.info(" Creating the data set for the {0} '{1}'.".
+ format(table.get("type", ""), table.get("title", "")))
suites = input_data.filter_data(table, data_set="suites")
suites = input_data.merge_data(suites)
suites = input_data.filter_data(table, data_set="suites")
suites = input_data.merge_data(suites)
@@
-226,6
+231,8
@@
def table_performance_improvements(table, input_data):
return None
# Transform the data
return None
# Transform the data
+ logging.info(" Creating the data set for the {0} '{1}'.".
+ format(table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table)
# Prepare the header of the tables
data = input_data.filter_data(table)
# Prepare the header of the tables
@@
-357,6
+364,8
@@
def table_performance_comparison(table, input_data):
format(table.get("title", "")))
# Transform the data
format(table.get("title", "")))
# Transform the data
+ logging.info(" Creating the data set for the {0} '{1}'.".
+ format(table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the tables
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the tables
@@
-595,6
+604,8
@@
def table_performance_comparison_mrr(table, input_data):
format(table.get("title", "")))
# Transform the data
format(table.get("title", "")))
# Transform the data
+ logging.info(" Creating the data set for the {0} '{1}'.".
+ format(table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the tables
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the tables
@@
-727,16
+738,18
@@
def table_performance_trending_dashboard(table, input_data):
format(table.get("title", "")))
# Transform the data
format(table.get("title", "")))
# Transform the data
+ logging.info(" Creating the data set for the {0} '{1}'.".
+ format(table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the tables
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the tables
- header = ["
Test Case",
+ header = ["Test Case",
"Trend [Mpps]",
"Trend [Mpps]",
- "
Short-Term
Change [%]",
- "
Long-Term
Change [%]",
- "
Regressions [#]",
- "
Progressions [#]",
- "
Outliers [#]"
+ "
Short-Term
Change [%]",
+ "
Long-Term
Change [%]",
+ "Regressions [#]",
+ "Progressions [#]",
+ "Outliers [#]"
]
header_str = ",".join(header) + "\n"
]
header_str = ",".join(header) + "\n"
@@
-752,7
+765,7
@@
def table_performance_trending_dashboard(table, input_data):
"-".join(tst_data["name"].
split("-")[1:]))
tbl_dict[tst_name] = {"name": name,
"-".join(tst_data["name"].
split("-")[1:]))
tbl_dict[tst_name] = {"name": name,
- "data":
d
ict()}
+ "data":
OrderedD
ict()}
try:
tbl_dict[tst_name]["data"][str(build)] = \
tst_data["result"]["throughput"]
try:
tbl_dict[tst_name]["data"][str(build)] = \
tst_data["result"]["throughput"]
@@
-764,18
+777,16
@@
def table_performance_trending_dashboard(table, input_data):
if len(tbl_dict[tst_name]["data"]) > 2:
pd_data = pd.Series(tbl_dict[tst_name]["data"])
if len(tbl_dict[tst_name]["data"]) > 2:
pd_data = pd.Series(tbl_dict[tst_name]["data"])
- last_key = pd_data.keys()[-1]
- win_size = min(pd_data.size, table["window"])
- win_first_idx = pd_data.size - win_size
- key_14 = pd_data.keys()[win_first_idx]
- long_win_size = min(pd_data.size, table["long-trend-window"])
-
data_t, _ = split_outliers(pd_data, outlier_const=1.5,
data_t, _ = split_outliers(pd_data, outlier_const=1.5,
- window=win_size)
-
+ window=table["window"])
+ last_key = data_t.keys()[-1]
+ win_size = min(data_t.size, table["window"])
+ win_first_idx = data_t.size - win_size
+ key_14 = data_t.keys()[win_first_idx]
+ long_win_size = min(data_t.size, table["long-trend-window"])
median_t = data_t.rolling(window=win_size, min_periods=2).median()
stdev_t = data_t.rolling(window=win_size, min_periods=2).std()
median_t = data_t.rolling(window=win_size, min_periods=2).median()
stdev_t = data_t.rolling(window=win_size, min_periods=2).std()
- median_first_idx =
pd_data
.size - long_win_size
+ median_first_idx =
median_t
.size - long_win_size
try:
max_median = max(
[x for x in median_t.values[median_first_idx:-win_size]
try:
max_median = max(
[x for x in median_t.values[median_first_idx:-win_size]
@@
-791,15
+802,10
@@
def table_performance_trending_dashboard(table, input_data):
except KeyError:
median_t_14 = nan
except KeyError:
median_t_14 = nan
- # Test name:
- name = tbl_dict[tst_name]["name"]
-
# Classification list:
classification_lst = list()
# Classification list:
classification_lst = list()
- for build_nr, value in pd_data.iteritems():
-
- if isnan(data_t[build_nr]) \
- or isnan(median_t[build_nr]) \
+ for build_nr, value in data_t.iteritems():
+ if isnan(median_t[build_nr]) \
or isnan(stdev_t[build_nr]) \
or isnan(value):
classification_lst.append("outlier")
or isnan(stdev_t[build_nr]) \
or isnan(value):
classification_lst.append("outlier")
@@
-823,7
+829,7
@@
def table_performance_trending_dashboard(table, input_data):
((last_median_t - max_median) / max_median) * 100, 2)
tbl_lst.append(
((last_median_t - max_median) / max_median) * 100, 2)
tbl_lst.append(
- [
name
,
+ [
tbl_dict[tst_name]["name"]
,
'-' if isnan(last_median_t) else
round(last_median_t / 1000000, 2),
'-' if isnan(rel_change_last) else rel_change_last,
'-' if isnan(last_median_t) else
round(last_median_t / 1000000, 2),
'-' if isnan(rel_change_last) else rel_change_last,