1 # Copyright (c) 2023 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Function used by Dash applications.
18 import dash_bootstrap_components as dbc
21 from numpy import isnan
23 from datetime import datetime
25 from ..jumpavg import classify
26 from ..utils.constants import Constants as C
27 from ..utils.url_processing import url_encode
30 def classify_anomalies(data):
31 """Process the data and return anomalies and trending values.
33 Gather data into groups with average as trend value.
34 Decorate values within groups to be normal,
35 the first value of changed average as a regression, or a progression.
37 :param data: Full data set with unavailable samples replaced by nan.
38 :type data: OrderedDict
39 :returns: Classification and trend values
40 :rtype: 3-tuple, list of strings, list of floats and list of floats
42 # NaN means something went wrong.
43 # Use 0.0 to cause that being reported as a severe regression.
44 bare_data = [0.0 if isnan(sample) else sample for sample in data.values()]
45 # TODO: Make BitCountingGroupList a subclass of list again?
46 group_list = classify(bare_data).group_list
47 group_list.reverse() # Just to use .pop() for FIFO.
48 classification = list()
55 for sample in data.values():
57 classification.append("outlier")
61 if values_left < 1 or active_group is None:
63 while values_left < 1: # Ignore empty groups (should not happen).
64 active_group = group_list.pop()
65 values_left = len(active_group.run_list)
66 avg = active_group.stats.avg
67 stdv = active_group.stats.stdev
68 classification.append(active_group.comment)
73 classification.append("normal")
77 return classification, avgs, stdevs
80 def get_color(idx: int) -> str:
81 """Returns a color from the list defined in Constants.PLOT_COLORS defined by
84 :param idx: Index of the color.
86 :returns: Color defined by hex code.
89 return C.PLOT_COLORS[idx % len(C.PLOT_COLORS)]
92 def show_tooltip(tooltips:dict, id: str, title: str,
93 clipboard_id: str=None) -> list:
94 """Generate list of elements to display a text (e.g. a title) with a
95 tooltip and optionaly with Copy&Paste icon and the clipboard
96 functionality enabled.
98 :param tooltips: Dictionary with tooltips.
99 :param id: Tooltip ID.
100 :param title: A text for which the tooltip will be displayed.
101 :param clipboard_id: If defined, a Copy&Paste icon is displayed and the
102 clipboard functionality is enabled.
106 :type clipboard_id: str
107 :returns: List of elements to display a text with a tooltip and
108 optionaly with Copy&Paste icon.
113 dcc.Clipboard(target_id=clipboard_id, title="Copy URL") \
114 if clipboard_id else str(),
122 class_name="border ms-1",
125 children=tooltips.get(id, str()),
132 def label(key: str) -> str:
133 """Returns a label for input elements (dropdowns, ...).
135 If the label is not defined, the function returns the provided key.
137 :param key: The key to the label defined in Constants.LABELS.
142 return C.LABELS.get(key, key)
145 def sync_checklists(options: list, sel: list, all: list, id: str) -> tuple:
146 """Synchronize a checklist with defined "options" with its "All" checklist.
148 :param options: List of options for the cheklist.
149 :param sel: List of selected options.
150 :param all: List of selected option from "All" checklist.
151 :param id: ID of a checklist to be used for synchronization.
152 :returns: Tuple of lists with otions for both checklists.
153 :rtype: tuple of lists
155 opts = {v["value"] for v in options}
157 sel = list(opts) if all else list()
159 all = ["all", ] if set(sel) == opts else list()
163 def list_tests(selection: dict) -> list:
164 """Transform list of tests to a list of dictionaries usable by checkboxes.
166 :param selection: List of tests to be displayed in "Selected tests" window.
167 :type selection: list
168 :returns: List of dictionaries with "label", "value" pairs for a checkbox.
172 return [{"label": v["id"], "value": v["id"]} for v in selection]
177 def get_date(s_date: str) -> datetime:
178 """Transform string reprezentation of date to datetime.datetime data type.
180 :param s_date: String reprezentation of date.
182 :returns: Date as datetime.datetime.
183 :rtype: datetime.datetime
185 return datetime(int(s_date[0:4]), int(s_date[5:7]), int(s_date[8:10]))
188 def gen_new_url(url_components: dict, params: dict) -> str:
189 """Generate a new URL with encoded parameters.
191 :param url_components: Dictionary with URL elements. It should contain
192 "scheme", "netloc" and "path".
193 :param url_components: URL parameters to be encoded to the URL.
194 :type parsed_url: dict
196 :returns Encoded URL with parameters.
203 "scheme": url_components.get("scheme", ""),
204 "netloc": url_components.get("netloc", ""),
205 "path": url_components.get("path", ""),
213 def get_duts(df: pd.DataFrame) -> list:
214 """Get the list of DUTs from the pre-processed information about jobs.
216 :param df: DataFrame with information about jobs.
217 :type df: pandas.DataFrame
218 :returns: Alphabeticaly sorted list of DUTs.
221 return sorted(list(df["dut"].unique()))
224 def get_ttypes(df: pd.DataFrame, dut: str) -> list:
225 """Get the list of test types from the pre-processed information about
228 :param df: DataFrame with information about jobs.
229 :param dut: The DUT for which the list of test types will be populated.
230 :type df: pandas.DataFrame
232 :returns: Alphabeticaly sorted list of test types.
235 return sorted(list(df.loc[(df["dut"] == dut)]["ttype"].unique()))
238 def get_cadences(df: pd.DataFrame, dut: str, ttype: str) -> list:
239 """Get the list of cadences from the pre-processed information about
242 :param df: DataFrame with information about jobs.
243 :param dut: The DUT for which the list of cadences will be populated.
244 :param ttype: The test type for which the list of cadences will be
246 :type df: pandas.DataFrame
249 :returns: Alphabeticaly sorted list of cadences.
252 return sorted(list(df.loc[(
254 (df["ttype"] == ttype)
255 )]["cadence"].unique()))
258 def get_test_beds(df: pd.DataFrame, dut: str, ttype: str, cadence: str) -> list:
259 """Get the list of test beds from the pre-processed information about
262 :param df: DataFrame with information about jobs.
263 :param dut: The DUT for which the list of test beds will be populated.
264 :param ttype: The test type for which the list of test beds will be
266 :param cadence: The cadence for which the list of test beds will be
268 :type df: pandas.DataFrame
272 :returns: Alphabeticaly sorted list of test beds.
275 return sorted(list(df.loc[(
277 (df["ttype"] == ttype) &
278 (df["cadence"] == cadence)
279 )]["tbed"].unique()))
282 def get_job(df: pd.DataFrame, dut, ttype, cadence, testbed):
283 """Get the name of a job defined by dut, ttype, cadence, test bed.
284 Input information comes from the control panel.
286 :param df: DataFrame with information about jobs.
287 :param dut: The DUT for which the job name will be created.
288 :param ttype: The test type for which the job name will be created.
289 :param cadence: The cadence for which the job name will be created.
290 :param testbed: The test bed for which the job name will be created.
291 :type df: pandas.DataFrame
301 (df["ttype"] == ttype) &
302 (df["cadence"] == cadence) &
303 (df["tbed"] == testbed)
307 def generate_options(opts: list, sort: bool=True) -> list:
308 """Return list of options for radio items in control panel. The items in
309 the list are dictionaries with keys "label" and "value".
311 :params opts: List of options (str) to be used for the generated list.
313 :returns: List of options (dict).
318 return [{"label": i, "value": i} for i in opts]
321 def set_job_params(df: pd.DataFrame, job: str) -> dict:
322 """Create a dictionary with all options and values for (and from) the
325 :param df: DataFrame with information about jobs.
326 :params job: The name of job for and from which the dictionary will be
328 :type df: pandas.DataFrame
330 :returns: Dictionary with all options and values for (and from) the
335 l_job = job.split("-")
341 "tbed": "-".join(l_job[-2:]),
342 "duts": generate_options(get_duts(df)),
343 "ttypes": generate_options(get_ttypes(df, l_job[1])),
344 "cadences": generate_options(get_cadences(df, l_job[1], l_job[3])),
345 "tbeds": generate_options(
346 get_test_beds(df, l_job[1], l_job[3], l_job[4]))
350 def get_list_group_items(
354 add_index: bool=False
356 """Generate list of ListGroupItems with checkboxes with selected items.
358 :param items: List of items to be displayed in the ListGroup.
359 :param type: The type part of an element ID.
360 :param colorize: If True, the color of labels is set, otherwise the default
362 :param add_index: Add index to the list items.
366 :type add_index: bool
367 :returns: List of ListGroupItems with checkboxes with selected items.
372 for i, l in enumerate(items):
373 idx = f"{i + 1}. " if add_index else str()
374 label = f"{idx}{l['id']}" if isinstance(l, dict) else f"{idx}{l}"
379 id={"type": type, "index": i},
382 label_class_name="m-0 p-0",
384 "font-size": ".875em",
385 "color": get_color(i) if colorize else "#55595c"
396 def relative_change_stdev(mean1, mean2, std1, std2):
397 """Compute relative standard deviation of change of two values.
399 The "1" values are the base for comparison.
400 Results are returned as percentage (and percentual points for stdev).
401 Linearized theory is used, so results are wrong for relatively large stdev.
403 :param mean1: Mean of the first number.
404 :param mean2: Mean of the second number.
405 :param std1: Standard deviation estimate of the first number.
406 :param std2: Standard deviation estimate of the second number.
411 :returns: Relative change and its stdev.
414 mean1, mean2 = float(mean1), float(mean2)
415 quotient = mean2 / mean1
417 second = std2 / mean2
418 std = quotient * sqrt(first * first + second * second)
419 return (quotient - 1) * 100, std * 100