X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=csit.infra.dash%2Fapp%2Fcdash%2Fcomparisons%2Ftables.py;h=8c19d3c7762e2a57b0758c19b5a554a460762dd7;hb=refs%2Fchanges%2F05%2F38705%2F5;hp=14d5d552af18882d0b26edfe0c9ba32866c5dfac;hpb=0fc5aff9887fa7a3125c71d0662475a3f9a763ba;p=csit.git diff --git a/csit.infra.dash/app/cdash/comparisons/tables.py b/csit.infra.dash/app/cdash/comparisons/tables.py index 14d5d552af..8c19d3c776 100644 --- a/csit.infra.dash/app/cdash/comparisons/tables.py +++ b/csit.infra.dash/app/cdash/comparisons/tables.py @@ -18,6 +18,7 @@ import pandas as pd from numpy import mean, std from copy import deepcopy + from ..utils.constants import Constants as C from ..utils.utils import relative_change_stdev @@ -70,23 +71,34 @@ def select_comparison_data( for itm in data_in["test_id"].unique().tolist(): itm_lst = itm.split(".") test = itm_lst[-1].rsplit("-", 1)[0] + if "hoststack" in itm: + test_type = f"hoststack-{ttype}" + else: + test_type = ttype df = data_in.loc[(data_in["test_id"] == itm)] - l_df = df[C.VALUE_ITER[ttype]].to_list() + l_df = df[C.VALUE_ITER[test_type]].to_list() if len(l_df) and isinstance(l_df[0], list): tmp_df = list() for l_itm in l_df: tmp_df.extend(l_itm) l_df = tmp_df + try: + mean_val = mean(l_df) + std_val = std(l_df) + except (TypeError, ValueError): + continue d_data["name"].append(f"{test.replace(f'{drv}-', '')}-{ttype}") - d_data["mean"].append(int(mean(l_df) * norm_factor)) - d_data["stdev"].append(int(std(l_df) * norm_factor)) - d_data["unit"].append(df[C.UNIT[ttype]].to_list()[0]) + d_data["mean"].append(int(mean_val * norm_factor)) + d_data["stdev"].append(int(std_val * norm_factor)) + d_data["unit"].append(df[C.UNIT[test_type]].to_list()[0]) return pd.DataFrame(d_data) lst_df = list() for itm in selected: - if itm["ttype"] in ("NDR", "PDR"): + if itm["ttype"] in ("NDR", "PDR", "Latency"): test_type = "ndrpdr" + elif itm["ttype"] in ("CPS", "RPS", "BPS"): + test_type = "hoststack" else: test_type = itm["ttype"].lower() @@ -101,11 +113,17 @@ def select_comparison_data( drv = "" if itm["driver"] == "dpdk" else itm["driver"].replace("_", "-") core = str() if itm["dut"] == "trex" else itm["core"].lower() - reg_id = \ - f"^.*[.|-]{itm['nic']}.*{itm['frmsize'].lower()}-{core}-{drv}.*$" + ttype = "ndrpdr" if itm["ttype"] in ("NDR", "PDR", "Latency") \ + else itm["ttype"].lower() tmp_df = tmp_df[ (tmp_df.job.str.endswith(itm["tbed"])) & - (tmp_df.test_id.str.contains(reg_id, regex=True)) + (tmp_df.test_id.str.contains( + ( + f"^.*[.|-]{itm['nic']}.*{itm['frmsize'].lower()}-" + f"{core}-{drv}.*-{ttype}$" + ), + regex=True + )) ] if itm["driver"] == "dpdk": for drv in C.DRIVERS: @@ -114,16 +132,23 @@ def select_comparison_data( inplace=True ) - # Change the data type from ndrpdr to one of ("NDR", "PDR") + # Change the data type from ndrpdr to one of ("NDR", "PDR", "Latency") if test_type == "ndrpdr": tmp_df = tmp_df.assign(test_type=itm["ttype"].lower()) if not tmp_df.empty: + if normalize: + if itm["ttype"] == "Latency": + norm_factor = C.FREQUENCY[itm["tbed"]] / C.NORM_FREQUENCY + else: + norm_factor = C.NORM_FREQUENCY / C.FREQUENCY[itm["tbed"]] + else: + norm_factor = 1.0 tmp_df = _calculate_statistics( tmp_df, itm["ttype"].lower(), itm["driver"], - C.NORM_FREQUENCY / C.FREQUENCY[itm["tbed"]] if normalize else 1 + norm_factor ) lst_df.append(tmp_df) @@ -194,12 +219,15 @@ def comparison_table( }) return selection - unit_factor, s_unit_factor = (1e6, "M") if format == "html" else (1, str()) - r_sel = deepcopy(selected["reference"]["selection"]) c_params = selected["compare"] r_selection = _create_selection(r_sel) + if format == "html" and "Latency" not in r_sel["ttype"]: + unit_factor, s_unit_factor = (1e6, "M") + else: + unit_factor, s_unit_factor = (1, str()) + # Create Table title and titles of columns with data params = list(r_sel) params.remove(c_params["parameter"]) @@ -281,3 +309,65 @@ def comparison_table( ) return (title, df_cmp) + + +def filter_table_data( + store_table_data: list, + table_filter: str + ) -> list: + """Filter table data using user specified filter. + + :param store_table_data: Table data represented as a list of records. + :param table_filter: User specified filter. + :type store_table_data: list + :type table_filter: str + :returns: A new table created by filtering of table data represented as + a list of records. + :rtype: list + """ + + # Checks: + if not any((table_filter, store_table_data, )): + return store_table_data + + def _split_filter_part(filter_part: str) -> tuple: + """Split a part of filter into column name, operator and value. + A "part of filter" is a sting berween "&&" operator. + + :param filter_part: A part of filter. + :type filter_part: str + :returns: Column name, operator, value + :rtype: tuple[str, str, str|float] + """ + for operator_type in C.OPERATORS: + for operator in operator_type: + if operator in filter_part: + name_p, val_p = filter_part.split(operator, 1) + name = name_p[name_p.find("{") + 1 : name_p.rfind("}")] + val_p = val_p.strip() + if (val_p[0] == val_p[-1] and val_p[0] in ("'", '"', '`')): + value = val_p[1:-1].replace("\\" + val_p[0], val_p[0]) + else: + try: + value = float(val_p) + except ValueError: + value = val_p + + return name, operator_type[0].strip(), value + return (None, None, None) + + df = pd.DataFrame.from_records(store_table_data) + for filter_part in table_filter.split(" && "): + col_name, operator, filter_value = _split_filter_part(filter_part) + if operator == "contains": + df = df.loc[df[col_name].str.contains(filter_value, regex=True)] + elif operator in ("eq", "ne", "lt", "le", "gt", "ge"): + # These operators match pandas series operator method names. + df = df.loc[getattr(df[col_name], operator)(filter_value)] + elif operator == "datestartswith": + # This is a simplification of the front-end filtering logic, + # only works with complete fields in standard format. + # Currently not used in comparison tables. + df = df.loc[df[col_name].str.startswith(filter_value)] + + return df.to_dict("records")