From 7669ddf94b93df73c339d41b8094bf95a5247775 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Tue, 13 Jun 2023 11:07:20 +0000 Subject: [PATCH] C-Dash: Telemetry - Add option to ignore hosts Change-Id: Ife92be275a18b07a2b78f57095843d2a65c4bcad Signed-off-by: Tibor Frank --- csit.infra.dash/app/cdash/trending/layout.py | 568 ++++++++++------------ csit.infra.dash/app/cdash/utils/telemetry_data.py | 75 +-- 2 files changed, 290 insertions(+), 353 deletions(-) diff --git a/csit.infra.dash/app/cdash/trending/layout.py b/csit.infra.dash/app/cdash/trending/layout.py index 38381459b8..74c39eb8c0 100644 --- a/csit.infra.dash/app/cdash/trending/layout.py +++ b/csit.infra.dash/app/cdash/trending/layout.py @@ -279,15 +279,8 @@ class Layout: ) else: return html.Div( - id="div-main-error", - children=[ - dbc.Alert( - [ - "An Error Occured" - ], - color="danger" - ) - ] + dbc.Alert("An Error Occured", color="danger"), + id="div-main-error" ) def _add_navbar(self): @@ -297,17 +290,15 @@ class Layout: :rtype: dbc.NavbarSimple """ return dbc.NavbarSimple( - id="navbarsimple-main", - children=[ - dbc.NavItem( - dbc.NavLink( - C.TREND_TITLE, - disabled=True, - external_link=True, - href="#" - ) + dbc.NavItem( + dbc.NavLink( + C.TREND_TITLE, + disabled=True, + external_link=True, + href="#" ) - ], + ), + id="navbarsimple-main", brand=C.BRAND, brand_href="/", brand_external_link=True, @@ -321,12 +312,7 @@ class Layout: :returns: Column with the control panel. :rtype: dbc.Col """ - return dbc.Col([ - html.Div( - children=self._add_ctrl_panel(), - className="sticky-top" - ) - ]) + return dbc.Col(html.Div(self._add_ctrl_panel(), className="sticky-top")) def _add_ctrl_panel(self) -> list: """Add control panel. @@ -336,316 +322,226 @@ class Layout: """ return [ dbc.Row( - class_name="g-0 p-1", - children=[ - dbc.InputGroup( - [ - dbc.InputGroupText( - children=show_tooltip( - self._tooltips, - "help-dut", - "DUT" - ) - ), - dbc.Select( - id={"type": "ctrl-dd", "index": "dut"}, - placeholder="Select a Device under Test...", - options=sorted( - [ - {"label": k, "value": k} \ - for k in self._spec_tbs.keys() - ], - key=lambda d: d["label"] - ) + dbc.InputGroup( + [ + dbc.InputGroupText( + show_tooltip(self._tooltips, "help-dut", "DUT") + ), + dbc.Select( + id={"type": "ctrl-dd", "index": "dut"}, + placeholder="Select a Device under Test...", + options=sorted( + [ + {"label": k, "value": k} \ + for k in self._spec_tbs.keys() + ], + key=lambda d: d["label"] ) - ], - size="sm" - ) - ] + ) + ], + size="sm" + ), + class_name="g-0 p-1" ), dbc.Row( - class_name="g-0 p-1", - children=[ - dbc.InputGroup( - [ - dbc.InputGroupText( - children=show_tooltip( - self._tooltips, - "help-infra", - "Infra" - ) - ), - dbc.Select( - id={"type": "ctrl-dd", "index": "phy"}, - placeholder=\ - "Select a Physical Test Bed Topology..." - ) - ], - size="sm" - ) - ] + dbc.InputGroup( + [ + dbc.InputGroupText( + show_tooltip(self._tooltips, "help-infra", "Infra") + ), + dbc.Select( + id={"type": "ctrl-dd", "index": "phy"}, + placeholder="Select a Physical Test Bed Topology..." + ) + ], + size="sm" + ), + class_name="g-0 p-1" ), dbc.Row( - class_name="g-0 p-1", - children=[ - dbc.InputGroup( - [ - dbc.InputGroupText( - children=show_tooltip( - self._tooltips, - "help-area", - "Area" - ) - ), - dbc.Select( - id={"type": "ctrl-dd", "index": "area"}, - placeholder="Select an Area..." - ) - ], - size="sm" - ) - ] + dbc.InputGroup( + [ + dbc.InputGroupText( + show_tooltip(self._tooltips, "help-area", "Area") + ), + dbc.Select( + id={"type": "ctrl-dd", "index": "area"}, + placeholder="Select an Area..." + ) + ], + size="sm" + ), + class_name="g-0 p-1" ), dbc.Row( - class_name="g-0 p-1", - children=[ - dbc.InputGroup( - [ - dbc.InputGroupText( - children=show_tooltip( - self._tooltips, - "help-test", - "Test" - ) - ), - dbc.Select( - id={"type": "ctrl-dd", "index": "test"}, - placeholder="Select a Test..." - ) - ], - size="sm" - ) - ] + dbc.InputGroup( + [ + dbc.InputGroupText( + show_tooltip(self._tooltips, "help-test", "Test") + ), + dbc.Select( + id={"type": "ctrl-dd", "index": "test"}, + placeholder="Select a Test..." + ) + ], + size="sm" + ), + class_name="g-0 p-1" ), dbc.Row( - class_name="g-0 p-1", - children=[ - dbc.InputGroup( - [ - dbc.InputGroupText( - children=show_tooltip( - self._tooltips, - "help-framesize", - "Frame Size" - ) - ), - dbc.Col( - children=[ - dbc.Checklist( - id={ - "type": "ctrl-cl", - "index": "frmsize-all" - }, - options=C.CL_ALL_DISABLED, - inline=True, - class_name="ms-2" - ) - ], - width=2 + dbc.InputGroup( + [ + dbc.InputGroupText(show_tooltip( + self._tooltips, + "help-framesize", + "Frame Size" + )), + dbc.Col( + dbc.Checklist( + id={"type": "ctrl-cl", "index": "frmsize-all"}, + options=C.CL_ALL_DISABLED, + inline=True, + class_name="ms-2" ), - dbc.Col( - children=[ - dbc.Checklist( - id={ - "type": "ctrl-cl", - "index": "frmsize" - }, - inline=True - ) - ] + width=2 + ), + dbc.Col( + dbc.Checklist( + id={"type": "ctrl-cl", "index": "frmsize"}, + inline=True ) - ], - style={"align-items": "center"}, - size="sm" - ) - ] + ) + ], + style={"align-items": "center"}, + size="sm" + ), + class_name="g-0 p-1" ), dbc.Row( - class_name="g-0 p-1", - children=[ - dbc.InputGroup( - [ - dbc.InputGroupText( - children=show_tooltip( - self._tooltips, - "help-cores", - "Number of Cores" - ) - ), - dbc.Col( - children=[ - dbc.Checklist( - id={ - "type": "ctrl-cl", - "index": "core-all" - }, - options=C.CL_ALL_DISABLED, - inline=True, - class_name="ms-2" - ) - ], - width=2 + dbc.InputGroup( + [ + dbc.InputGroupText(show_tooltip( + self._tooltips, + "help-cores", + "Number of Cores" + )), + dbc.Col( + dbc.Checklist( + id={"type": "ctrl-cl", "index": "core-all"}, + options=C.CL_ALL_DISABLED, + inline=True, + class_name="ms-2" ), - dbc.Col( - children=[ - dbc.Checklist( - id={ - "type": "ctrl-cl", - "index": "core" - }, - inline=True - ) - ] + width=2 + ), + dbc.Col( + dbc.Checklist( + id={"type": "ctrl-cl", "index": "core"}, + inline=True ) - ], - style={"align-items": "center"}, - size="sm" - ) - ] + ) + ], + style={"align-items": "center"}, + size="sm" + ), + class_name="g-0 p-1" ), dbc.Row( - class_name="g-0 p-1", - children=[ - dbc.InputGroup( - [ - dbc.InputGroupText( - children=show_tooltip( - self._tooltips, - "help-ttype", - "Test Type" - ) - ), - dbc.Col( - children=[ - dbc.Checklist( - id={ - "type": "ctrl-cl", - "index": "tsttype-all" - }, - options=C.CL_ALL_DISABLED, - inline=True, - class_name="ms-2" - ) - ], - width=2 + dbc.InputGroup( + [ + dbc.InputGroupText(show_tooltip( + self._tooltips, + "help-ttype", + "Test Type" + )), + dbc.Col( + dbc.Checklist( + id={"type": "ctrl-cl", "index": "tsttype-all"}, + options=C.CL_ALL_DISABLED, + inline=True, + class_name="ms-2" ), - dbc.Col( - children=[ - dbc.Checklist( - id={ - "type": "ctrl-cl", - "index": "tsttype" - }, - inline=True - ) - ] + width=2 + ), + dbc.Col( + dbc.Checklist( + id={"type": "ctrl-cl", "index": "tsttype"}, + inline=True ) - ], - style={"align-items": "center"}, - size="sm" - ) - ] + ) + ], + style={"align-items": "center"}, + size="sm" + ), + class_name="g-0 p-1" ), dbc.Row( - class_name="g-0 p-1", - children=[ - dbc.InputGroup( - [ - dbc.InputGroupText( - children=show_tooltip( - self._tooltips, - "help-normalize", - "Normalization" - ) - ), - dbc.Col( - children=[ - dbc.Checklist( - id="normalize", - options=[{ - "value": "normalize", - "label": ( - "Normalize to CPU frequency " - "2GHz" - ) - }], - value=[], - inline=True, - class_name="ms-2" - ) - ] - ) - ], - style={"align-items": "center"}, - size="sm" - ) - ] + dbc.InputGroup( + [ + dbc.InputGroupText(show_tooltip( + self._tooltips, + "help-normalize", + "Normalization" + )), + dbc.Col(dbc.Checklist( + id="normalize", + options=[{ + "value": "normalize", + "label": "Normalize to CPU frequency 2GHz" + }], + value=[], + inline=True, + class_name="ms-2" + )) + ], + style={"align-items": "center"}, + size="sm" + ), + class_name="g-0 p-1" ), dbc.Row( - class_name="g-0 p-1", - children=[ - dbc.Button( - id={"type": "ctrl-btn", "index": "add-test"}, - children="Add Selected", - color="info" - ) - ] + dbc.Button( + id={"type": "ctrl-btn", "index": "add-test"}, + children="Add Selected", + color="info" + ), + class_name="g-0 p-1" ), dbc.Row( + dbc.ListGroup( + class_name="overflow-auto p-0", + id="lg-selected", + children=[], + style={"max-height": "20em"}, + flush=True + ), id="row-card-sel-tests", class_name="g-0 p-1", style=C.STYLE_DISABLED, - children=[ - dbc.ListGroup( - class_name="overflow-auto p-0", - id="lg-selected", - children=[], - style={"max-height": "20em"}, - flush=True - ) - ] ), dbc.Row( + dbc.ButtonGroup([ + dbc.Button( + "Remove Selected", + id={"type": "ctrl-btn", "index": "rm-test"}, + class_name="w-100", + color="info", + disabled=False + ), + dbc.Button( + "Remove All", + id={"type": "ctrl-btn", "index": "rm-test-all"}, + class_name="w-100", + color="info", + disabled=False + ) + ]), id="row-btns-sel-tests", class_name="g-0 p-1", style=C.STYLE_DISABLED, - children=[ - dbc.ButtonGroup( - children=[ - dbc.Button( - id={"type": "ctrl-btn", "index": "rm-test"}, - children="Remove Selected", - class_name="w-100", - color="info", - disabled=False - ), - dbc.Button( - id={"type": "ctrl-btn", "index": "rm-test-all"}, - children="Remove All", - class_name="w-100", - color="info", - disabled=False - ) - ] - ) - ] ), dbc.Stack( - id="row-btns-add-tm", - class_name="g-0 p-1", - style=C.STYLE_DISABLED, - gap=2, - children=[ + [ dbc.Button( "Add Telemetry Panel", id={"type": "telemetry-btn", "index": "open"}, @@ -662,7 +558,11 @@ class Layout: is_open=False, scrollable=True ) - ] + ], + id="row-btns-add-tm", + class_name="g-0 p-1", + style=C.STYLE_DISABLED, + gap=2 ) ] @@ -980,30 +880,38 @@ class Layout: """ return [ dbc.Row( + "Add content here.", id={"type": "tm-container", "index": 0}, - class_name="g-0 p-1", - children=["Add content here."] + class_name="g-0 p-1" ), dbc.Row( - class_name="g-0 p-2", - children=[ - dbc.Checkbox( - id={"type": "cb-all-in-one", "index": 0}, - label="All Metrics in one Graph" + [ + dbc.Col( + dbc.Checkbox( + id={"type": "cb-all-in-one", "index": 0}, + label="All Metrics in one Graph" + ), + width=6 ), - ] + dbc.Col( + dbc.Checkbox( + id={"type": "cb-ignore-host", "index": 0}, + label="Ignore Host" + ), + width=6 + ) + ], + class_name="g-0 p-2" ), dbc.Row( - class_name="g-0 p-1", - children=[ - dbc.Textarea( - id={"type": "tm-list-metrics", "index": 0}, - rows=20, - size="sm", - wrap="off", - readonly=True - ) - ] + dbc.Textarea( + id={"type": "tm-list-metrics", "index": 0}, + rows=20, + size="sm", + wrap="off", + readonly=True + ), + class_name="g-0 p-1" ) ] @@ -1056,6 +964,7 @@ class Layout: State("store", "data"), State({"type": "sel-cl", "index": ALL}, "value"), State({"type": "cb-all-in-one", "index": ALL}, "value"), + State({"type": "cb-ignore-host", "index": ALL}, "value"), State({"type": "telemetry-search-out", "index": ALL}, "children"), State({"type": "plot-mod-telemetry", "index": ALL}, "is_open"), State({"type": "telemetry-btn", "index": ALL}, "disabled"), @@ -1080,6 +989,7 @@ class Layout: store: dict, lst_sel: list, all_in_one: list, + ignore_host: list, search_out: list, is_open: list, tm_btns_disabled: list, @@ -1102,6 +1012,7 @@ class Layout: "selected-metrics": dict(), "telemetry-panels": list(), "telemetry-all-in-one": list(), + "telemetry-ignore-host": list(), "telemetry-graphs": list(), "url": str() } @@ -1115,6 +1026,7 @@ class Layout: tm_user = store["selected-metrics"] tm_panels = store["telemetry-panels"] tm_all_in_one = store["telemetry-all-in-one"] + tm_ignore_host = store["telemetry-ignore-host"] plotting_area_telemetry = no_update on_draw = [False, False] # 0 --> trending, 1 --> telemetry @@ -1154,10 +1066,12 @@ class Layout: store_sel = literal_eval(url_params["store_sel"][0]) normalize = literal_eval(url_params["norm"][0]) telemetry = literal_eval(url_params["telemetry"][0]) - tm_all_in_one = literal_eval(url_params["all-in-one"][0]) + url_p = url_params.get("all-in-one", ["[[None]]"]) + tm_all_in_one = literal_eval(url_p[0]) + url_p = url_params.get("ignore-host", ["[[None]]"]) + tm_ignore_host = literal_eval(url_p[0]) if not isinstance(telemetry, list): telemetry = [telemetry, ] - tm_all_in_one = [tm_all_in_one, ] except (KeyError, IndexError, AttributeError, ValueError): pass if store_sel: @@ -1359,6 +1273,7 @@ class Layout: elif trigger.type == "ctrl-btn": tm_panels = list() tm_all_in_one = list() + tm_ignore_host = list() store["trending-graphs"] = None store["telemetry-graphs"] = list() # on_draw[0] = True @@ -1446,6 +1361,7 @@ class Layout: tm.from_json(tm_data) tm_panels.append(tm_user["selected_metrics_with_labels"]) tm_all_in_one.append(all_in_one) + tm_ignore_host.append(ignore_host) is_open = (False, False) tm_btns_disabled[1], tm_btns_disabled[5] = True, True on_draw = [True, True] @@ -1455,6 +1371,7 @@ class Layout: elif trigger.idx == "rm-all": tm_panels = list() tm_all_in_one = list() + tm_ignore_host = list() tm_user = None is_open = (False, False) tm_btns_disabled[1], tm_btns_disabled[5] = True, True @@ -1501,6 +1418,7 @@ class Layout: elif trigger.type == "tm-btn-remove": del tm_panels[trigger.idx] del tm_all_in_one[trigger.idx] + del tm_ignore_host[trigger.idx] del store["telemetry-graphs"][trigger.idx] tm.from_json(tm_data) on_draw = [True, True] @@ -1512,6 +1430,7 @@ class Layout: if tm_panels: new_url_params["telemetry"] = tm_panels new_url_params["all-in-one"] = tm_all_in_one + new_url_params["ignore-host"] = tm_ignore_host if on_draw[0]: # Trending if store_sel: @@ -1538,7 +1457,10 @@ class Layout: elif on_draw[1] and (end_idx >= start_idx): for idx in range(start_idx, end_idx): store["telemetry-graphs"].append(graph_tm_trending( - tm.select_tm_trending_data(tm_panels[idx]), + tm.select_tm_trending_data( + tm_panels[idx], + ignore_host=bool(tm_ignore_host[idx][0]) + ), self._graph_layout, bool(tm_all_in_one[idx][0]) )) @@ -1561,6 +1483,7 @@ class Layout: store_sel = list() tm_panels = list() tm_all_in_one = list() + tm_ignore_host = list() tm_user = None else: plotting_area_trending = no_update @@ -1577,6 +1500,7 @@ class Layout: store["selected-metrics"] = tm_user store["telemetry-panels"] = tm_panels store["telemetry-all-in-one"] = tm_all_in_one + store["telemetry-ignore-host"] = tm_ignore_host ret_val = [ store, plotting_area_trending, diff --git a/csit.infra.dash/app/cdash/utils/telemetry_data.py b/csit.infra.dash/app/cdash/utils/telemetry_data.py index 9c2e45f9a1..80187967fa 100644 --- a/csit.infra.dash/app/cdash/utils/telemetry_data.py +++ b/csit.infra.dash/app/cdash/utils/telemetry_data.py @@ -52,15 +52,17 @@ class TelemetryData: if in_data.empty: return - df = pd.DataFrame() metrics = set() # A set of unique metrics # Create a dataframe with metrics for selected tests: + lst_items = list() for itm in self._tests: sel_data = select_trending_data(in_data, itm) if sel_data is not None: sel_data["test_name"] = itm["id"] - df = pd.concat([df, sel_data], ignore_index=True, copy=False) + lst_items.append(sel_data) + df = pd.concat(lst_items, ignore_index=True, copy=False) + # Use only neccessary data: df = df[[ "job", @@ -182,23 +184,20 @@ class TelemetryData: :rtype: dict """ - df_labels = pd.DataFrame() + lst_labels = list() tmp_labels = dict() for _, row in self._data.iterrows(): telemetry = row["telemetry"] for itm in metrics: df = telemetry.loc[(telemetry["metric"] == itm)] - df_labels = pd.concat( - [df_labels, df], - ignore_index=True, - copy=False - ) + lst_labels.append(df) for _, tm in df.iterrows(): for label in tm["labels"]: if label[0] not in tmp_labels: tmp_labels[label[0]] = set() tmp_labels[label[0]].add(label[1]) + df_labels = pd.concat(lst_labels, ignore_index=True, copy=False) selected_labels = dict() for key in sorted(tmp_labels): selected_labels[key] = sorted(tmp_labels[key]) @@ -279,17 +278,19 @@ class TelemetryData: return bool(passed and all(passed)) self._selected_metrics_labels = pd.DataFrame() + lst_items = list() for _, row in self._unique_metrics_labels.iterrows(): if _is_selected(row["labels"], selection): - self._selected_metrics_labels = pd.concat( - [self._selected_metrics_labels, row.to_frame().T], - ignore_index=True, - axis=0, - copy=False - ) + lst_items.append(row.to_frame().T) + self._selected_metrics_labels = \ + pd.concat(lst_items, ignore_index=True, axis=0, copy=False) return self._selected_metrics_labels - def select_tm_trending_data(self, selection: dict) -> pd.DataFrame: + def select_tm_trending_data( + self, + selection: dict, + ignore_host: bool = False + ) -> pd.DataFrame: """Select telemetry data for trending based on user's 'selection'. The output dataframe includes these columns: @@ -313,37 +314,49 @@ class TelemetryData: - "tm_value". :param selection: User's selection (metrics and labels). + :param ignore_host: Ignore 'hostname' and 'hook' labels in metrics. :type selection: dict + :type ignore_host: bool :returns: Dataframe with selected data. :rtype: pandas.DataFrame """ - df = pd.DataFrame() - if self._data is None: - return df + return pd.DataFrame() if self._data.empty: - return df + return pd.DataFrame() if not selection: - return df + return pd.DataFrame() df_sel = pd.DataFrame.from_dict(selection) + lst_rows = list() for _, row in self._data.iterrows(): tm_row = row["telemetry"] for _, tm_sel in df_sel.iterrows(): df_tmp = tm_row.loc[tm_row["metric"] == tm_sel["metric"]] for _, tm in df_tmp.iterrows(): - if tm["labels"] == tm_sel["labels"]: - labels = ','.join( - [f"{itm[0]}='{itm[1]}'" for itm in tm["labels"]] - ) + do_it = False + if ignore_host: + if tm["labels"][2:] == tm_sel["labels"][2:]: + labels = ','.join( + [f"{i[0]}='{i[1]}'" for i in tm["labels"][2:]] + ) + do_it = True + else: + if tm["labels"] == tm_sel["labels"]: + labels = ','.join( + [f"{i[0]}='{i[1]}'" for i in tm["labels"]] + ) + do_it = True + if do_it: row["tm_metric"] = f"{tm['metric']}{{{labels}}}" row["tm_value"] = tm["value"] - new_row = row.drop(labels=["telemetry", ]) - df = pd.concat( - [df, new_row.to_frame().T], - ignore_index=True, - axis=0, - copy=False + lst_rows.append( + row.drop(labels=["telemetry", ]).to_frame().T ) - return df + if lst_rows: + return pd.concat( + lst_rows, ignore_index=True, axis=0, copy=False + ).drop_duplicates() + else: + return pd.DataFrame() -- 2.16.6