csit.infra.dash/app/cdash/trending/graphs.py

   1 # Copyright (c) 2024 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Implementation of graphs for trending data.
  15 """
  16
  17 import logging
  18 import plotly.graph_objects as go
  19 import pandas as pd
  20
  21 from numpy import nan
  22
  23 from ..utils.constants import Constants as C
  24 from ..utils.utils import get_color, get_hdrh_latencies
  25 from ..utils.anomalies import classify_anomalies
  26
  27
  28 def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
  29     """Select the data for graphs from the provided data frame.
  30
  31     :param data: Data frame with data for graphs.
  32     :param itm: Item (in this case job name) which data will be selected from
  33         the input data frame.
  34     :type data: pandas.DataFrame
  35     :type itm: dict
  36     :returns: A data frame with selected data.
  37     :rtype: pandas.DataFrame
  38     """
  39
  40     phy = itm["phy"].split("-")
  41     if len(phy) == 4:
  42         topo, arch, nic, drv = phy
  43         if drv == "dpdk":
  44             drv = ""
  45         else:
  46             drv += "-"
  47             drv = drv.replace("_", "-")
  48     else:
  49         return None
  50
  51     if itm["testtype"] in ("ndr", "pdr"):
  52         test_type = "ndrpdr"
  53     elif itm["testtype"] == "mrr":
  54         test_type = "mrr"
  55     elif itm["testtype"] == "soak":
  56         test_type = "soak"
  57     elif itm["area"] == "hoststack":
  58         test_type = "hoststack"
  59     df = data.loc[(
  60         (data["test_type"] == test_type) &
  61         (data["passed"] == True)
  62     )]
  63     df = df[df.job.str.endswith(f"{topo}-{arch}")]
  64     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
  65     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
  66     df = df[df.test_id.str.contains(
  67         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
  68         regex=True
  69     )].sort_values(by="start_time", ignore_index=True)
  70
  71     return df
  72
  73
  74 def graph_trending(
  75         data: pd.DataFrame,
  76         sel: dict,
  77         layout: dict,
  78         normalize: bool=False,
  79         trials: bool=False
  80     ) -> tuple:
  81     """Generate the trending graph(s) - MRR, NDR, PDR and for PDR also Latences
  82     (result_latency_forward_pdr_50_avg).
  83
  84     :param data: Data frame with test results.
  85     :param sel: Selected tests.
  86     :param layout: Layout of plot.ly graph.
  87     :param normalize: If True, the data is normalized to CPU frquency
  88         Constants.NORM_FREQUENCY.
  89     :param trials: If True, MRR trials are displayed in the trending graph.
  90     :type data: pandas.DataFrame
  91     :type sel: dict
  92     :type layout: dict
  93     :type normalize: bool
  94     :type: trials: bool
  95     :returns: Trending graph(s)
  96     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
  97     """
  98
  99     if not sel:
 100         return None, None
 101
 102
 103     def _generate_trending_traces(
 104             ttype: str,
 105             name: str,
 106             df: pd.DataFrame,
 107             color: str,
 108             nf: float
 109         ) -> list:
 110         """Generate the trending traces for the trending graph.
 111
 112         :param ttype: Test type (MRR, NDR, PDR).
 113         :param name: The test name to be displayed as the graph title.
 114         :param df: Data frame with test data.
 115         :param color: The color of the trace (samples and trend line).
 116         :param nf: The factor used for normalization of the results to
 117             CPU frequency set to Constants.NORM_FREQUENCY.
 118         :type ttype: str
 119         :type name: str
 120         :type df: pandas.DataFrame
 121         :type color: str
 122         :type nf: float
 123         :returns: Traces (samples, trending line, anomalies)
 124         :rtype: list
 125         """
 126
 127         df = df.dropna(subset=[C.VALUE[ttype], ])
 128         if df.empty:
 129             return list(), list()
 130
 131         hover = list()
 132         customdata = list()
 133         customdata_samples = list()
 134         name_lst = name.split("-")
 135         for _, row in df.iterrows():
 136             h_tput, h_band, h_lat = str(), str(), str()
 137             if ttype in ("mrr", "mrr-bandwidth"):
 138                 h_tput = (
 139                     f"tput avg [{row['result_receive_rate_rate_unit']}]: "
 140                     f"{row['result_receive_rate_rate_avg'] * nf:,.0f}<br>"
 141                     f"tput stdev [{row['result_receive_rate_rate_unit']}]: "
 142                     f"{row['result_receive_rate_rate_stdev'] * nf:,.0f}<br>"
 143                 )
 144                 if pd.notna(row["result_receive_rate_bandwidth_avg"]):
 145                     h_band = (
 146                         f"bandwidth avg "
 147                         f"[{row['result_receive_rate_bandwidth_unit']}]: "
 148                         f"{row['result_receive_rate_bandwidth_avg'] * nf:,.0f}"
 149                         "<br>"
 150                         f"bandwidth stdev "
 151                         f"[{row['result_receive_rate_bandwidth_unit']}]: "
 152                         f"{row['result_receive_rate_bandwidth_stdev']* nf:,.0f}"
 153                         "<br>"
 154                     )
 155             elif ttype in ("ndr", "ndr-bandwidth"):
 156                 h_tput = (
 157                     f"tput [{row['result_ndr_lower_rate_unit']}]: "
 158                     f"{row['result_ndr_lower_rate_value'] * nf:,.0f}<br>"
 159                 )
 160                 if pd.notna(row["result_ndr_lower_bandwidth_value"]):
 161                     h_band = (
 162                         f"bandwidth [{row['result_ndr_lower_bandwidth_unit']}]:"
 163                         f" {row['result_ndr_lower_bandwidth_value'] * nf:,.0f}"
 164                         "<br>"
 165                     )
 166             elif ttype in ("pdr", "pdr-bandwidth", "latency"):
 167                 h_tput = (
 168                     f"tput [{row['result_pdr_lower_rate_unit']}]: "
 169                     f"{row['result_pdr_lower_rate_value'] * nf:,.0f}<br>"
 170                 )
 171                 if pd.notna(row["result_pdr_lower_bandwidth_value"]):
 172                     h_band = (
 173                         f"bandwidth [{row['result_pdr_lower_bandwidth_unit']}]:"
 174                         f" {row['result_pdr_lower_bandwidth_value'] * nf:,.0f}"
 175                         "<br>"
 176                     )
 177                 if pd.notna(row["result_latency_forward_pdr_50_avg"]):
 178                     h_lat = (
 179                         f"latency "
 180                         f"[{row['result_latency_forward_pdr_50_unit']}]: "
 181                         f"{row['result_latency_forward_pdr_50_avg'] / nf:,.0f}"
 182                         "<br>"
 183                     )
 184             elif ttype in ("hoststack-cps", "hoststack-rps",
 185                            "hoststack-cps-bandwidth",
 186                            "hoststack-rps-bandwidth", "hoststack-latency"):
 187                 h_tput = (
 188                     f"tput [{row['result_rate_unit']}]: "
 189                     f"{row['result_rate_value'] * nf:,.0f}<br>"
 190                 )
 191                 h_band = (
 192                     f"bandwidth [{row['result_bandwidth_unit']}]: "
 193                     f"{row['result_bandwidth_value'] * nf:,.0f}<br>"
 194                 )
 195                 h_lat = (
 196                     f"latency [{row['result_latency_unit']}]: "
 197                     f"{row['result_latency_value'] / nf:,.0f}<br>"
 198                 )
 199             elif ttype in ("hoststack-bps", ):
 200                 h_band = (
 201                     f"bandwidth [{row['result_bandwidth_unit']}]: "
 202                     f"{row['result_bandwidth_value'] * nf:,.0f}<br>"
 203                 )
 204             elif ttype in ("soak", "soak-bandwidth"):
 205                 h_tput = (
 206                     f"tput [{row['result_critical_rate_lower_rate_unit']}]: "
 207                     f"{row['result_critical_rate_lower_rate_value'] * nf:,.0f}"
 208                     "<br>"
 209                 )
 210                 if pd.notna(row["result_critical_rate_lower_bandwidth_value"]):
 211                     bv = row['result_critical_rate_lower_bandwidth_value']
 212                     h_band = (
 213                         "bandwidth "
 214                         f"[{row['result_critical_rate_lower_bandwidth_unit']}]:"
 215                         f" {bv * nf:,.0f}"
 216                         "<br>"
 217                     )
 218             try:
 219                 hosts = f"<br>hosts: {', '.join(row['hosts'])}"
 220             except (KeyError, TypeError):
 221                 hosts = str()
 222             hover_itm = (
 223                 f"dut: {name_lst[0]}<br>"
 224                 f"infra: {'-'.join(name_lst[1:5])}<br>"
 225                 f"test: {'-'.join(name_lst[5:])}<br>"
 226                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 227                 f"{h_tput}{h_band}{h_lat}"
 228                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 229                 f"csit-ref: {row['job']}/{row['build']}"
 230                 f"{hosts}"
 231             )
 232             hover.append(hover_itm)
 233             if ttype == "latency":
 234                 customdata_samples.append(get_hdrh_latencies(row, name))
 235                 customdata.append({"name": name})
 236             else:
 237                 customdata_samples.append(
 238                     {"name": name, "show_telemetry": True}
 239                 )
 240                 customdata.append({"name": name})
 241
 242         x_axis = df["start_time"].tolist()
 243         if "latency" in ttype:
 244             y_data = [(v / nf) for v in df[C.VALUE[ttype]].tolist()]
 245         else:
 246             y_data = [(v * nf) for v in df[C.VALUE[ttype]].tolist()]
 247         units = df[C.UNIT[ttype]].unique().tolist()
 248
 249         try:
 250             anomalies, trend_avg, trend_stdev = classify_anomalies(
 251                 {k: v for k, v in zip(x_axis, y_data)}
 252             )
 253         except ValueError as err:
 254             logging.error(err)
 255             return list(), list()
 256
 257         hover_trend = list()
 258         for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 259             try:
 260                 hosts = f"<br>hosts: {', '.join(row['hosts'])}"
 261             except (KeyError, TypeError):
 262                 hosts = str()
 263             hover_itm = (
 264                 f"dut: {name_lst[0]}<br>"
 265                 f"infra: {'-'.join(name_lst[1:5])}<br>"
 266                 f"test: {'-'.join(name_lst[5:])}<br>"
 267                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 268                 f"trend [{row[C.UNIT[ttype]]}]: {avg:,.0f}<br>"
 269                 f"stdev [{row[C.UNIT[ttype]]}]: {stdev:,.0f}<br>"
 270                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 271                 f"csit-ref: {row['job']}/{row['build']}"
 272                 f"{hosts}"
 273             )
 274             if ttype == "latency":
 275                 hover_itm = hover_itm.replace("[pps]", "[us]")
 276             hover_trend.append(hover_itm)
 277
 278         traces = [
 279             go.Scatter(  # Samples
 280                 x=x_axis,
 281                 y=y_data,
 282                 name=name,
 283                 mode="markers",
 284                 marker={
 285                     "size": 5,
 286                     "color": color,
 287                     "symbol": "circle"
 288                 },
 289                 text=hover,
 290                 hoverinfo="text",
 291                 showlegend=True,
 292                 legendgroup=name,
 293                 customdata=customdata_samples
 294             ),
 295             go.Scatter(  # Trend line
 296                 x=x_axis,
 297                 y=trend_avg,
 298                 name=name,
 299                 mode="lines",
 300                 line={
 301                     "shape": "linear",
 302                     "width": 1,
 303                     "color": color,
 304                 },
 305                 text=hover_trend,
 306                 hoverinfo="text",
 307                 showlegend=False,
 308                 legendgroup=name,
 309                 customdata=customdata
 310             )
 311         ]
 312
 313         if anomalies:
 314             anomaly_x = list()
 315             anomaly_y = list()
 316             anomaly_color = list()
 317             hover = list()
 318             for idx, anomaly in enumerate(anomalies):
 319                 if anomaly in ("regression", "progression"):
 320                     anomaly_x.append(x_axis[idx])
 321                     anomaly_y.append(trend_avg[idx])
 322                     anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 323                     hover_itm = (
 324                         f"dut: {name_lst[0]}<br>"
 325                         f"infra: {'-'.join(name_lst[1:5])}<br>"
 326                         f"test: {'-'.join(name_lst[5:])}<br>"
 327                         f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 328                         f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
 329                         f"classification: {anomaly}"
 330                     )
 331                     if ttype == "latency":
 332                         hover_itm = hover_itm.replace("[pps]", "[us]")
 333                     hover.append(hover_itm)
 334             anomaly_color.extend([0.0, 0.5, 1.0])
 335             traces.append(
 336                 go.Scatter(
 337                     x=anomaly_x,
 338                     y=anomaly_y,
 339                     mode="markers",
 340                     text=hover,
 341                     hoverinfo="text",
 342                     showlegend=False,
 343                     legendgroup=name,
 344                     name=name,
 345                     customdata=customdata,
 346                     marker={
 347                         "size": 15,
 348                         "symbol": "circle-open",
 349                         "color": anomaly_color,
 350                         "colorscale": C.COLORSCALE_LAT \
 351                             if ttype == "latency" else C.COLORSCALE_TPUT,
 352                         "showscale": True,
 353                         "line": {
 354                             "width": 2
 355                         },
 356                         "colorbar": {
 357                             "y": 0.5,
 358                             "len": 0.8,
 359                             "title": "Circles Marking Data Classification",
 360                             "titleside": "right",
 361                             "tickmode": "array",
 362                             "tickvals": [0.167, 0.500, 0.833],
 363                             "ticktext": C.TICK_TEXT_LAT \
 364                                 if ttype == "latency" else C.TICK_TEXT_TPUT,
 365                             "ticks": "",
 366                             "ticklen": 0,
 367                             "tickangle": -90,
 368                             "thickness": 10
 369                         }
 370                     }
 371                 )
 372             )
 373
 374         return traces, units
 375
 376
 377     def _add_mrr_trials_traces(
 378             ttype: str,
 379             name: str,
 380             df: pd.DataFrame,
 381             color: str,
 382             nf: float
 383         ) -> list:
 384         """Add the traces with mrr trials.
 385
 386         :param ttype: Test type (mrr, mrr-bandwidth).
 387         :param name: The test name to be displayed in hover.
 388         :param df: Data frame with test data.
 389         :param color: The color of the trace.
 390         :param nf: The factor used for normalization of the results to
 391             CPU frequency set to Constants.NORM_FREQUENCY.
 392         :type ttype: str
 393         :type name: str
 394         :type df: pandas.DataFrame
 395         :type color: str
 396         :type nf: float
 397         :returns: list of Traces
 398         :rtype: list
 399         """
 400         traces = list()
 401         x_axis = df["start_time"].tolist()
 402         y_data = df[C.VALUE[ttype].replace("avg", "values")].tolist()
 403
 404         for idx_trial in range(10):
 405             y_axis = list()
 406             for idx_run in range(len(x_axis)):
 407                 try:
 408                     y_axis.append(y_data[idx_run][idx_trial] * nf)
 409                 except IndexError:
 410                     y_axis.append(nan)
 411             traces.append(go.Scatter(
 412                 x=x_axis,
 413                 y=y_axis,
 414                 name=name,
 415                 mode="markers",
 416                 marker={
 417                     "size": 2,
 418                     "color": color,
 419                     "symbol": "circle"
 420                 },
 421                 showlegend=True,
 422                 legendgroup=name
 423             ))
 424         return traces
 425
 426
 427     fig_tput = None
 428     fig_lat = None
 429     fig_band = None
 430     y_units = set()
 431     for idx, itm in enumerate(sel):
 432         df = select_trending_data(data, itm)
 433         if df is None or df.empty:
 434             continue
 435
 436         if normalize:
 437             phy = itm["phy"].split("-")
 438             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 439             norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY.get(topo_arch, 1.0)) \
 440                 if topo_arch else 1.0
 441         else:
 442             norm_factor = 1.0
 443
 444         if itm["area"] == "hoststack":
 445             ttype = f"hoststack-{itm['testtype']}"
 446         else:
 447             ttype = itm["testtype"]
 448
 449         traces, units = _generate_trending_traces(
 450             ttype,
 451             itm["id"],
 452             df,
 453             get_color(idx),
 454             norm_factor
 455         )
 456         if traces:
 457             if not fig_tput:
 458                 fig_tput = go.Figure()
 459             if trials and "mrr" in ttype:
 460                 traces.extend(_add_mrr_trials_traces(
 461                     ttype,
 462                     itm["id"],
 463                     df,
 464                     get_color(idx),
 465                     norm_factor
 466                 ))
 467             fig_tput.add_traces(traces)
 468
 469         if ttype in C.TESTS_WITH_BANDWIDTH:
 470             traces, _ = _generate_trending_traces(
 471                 f"{ttype}-bandwidth",
 472                 itm["id"],
 473                 df,
 474                 get_color(idx),
 475                 norm_factor
 476             )
 477             if traces:
 478                 if not fig_band:
 479                     fig_band = go.Figure()
 480                 if trials and "mrr" in ttype:
 481                     traces.extend(_add_mrr_trials_traces(
 482                         f"{ttype}-bandwidth",
 483                         itm["id"],
 484                         df,
 485                         get_color(idx),
 486                         norm_factor
 487                     ))
 488                 fig_band.add_traces(traces)
 489
 490         if ttype in C.TESTS_WITH_LATENCY:
 491             traces, _ = _generate_trending_traces(
 492                 "latency" if ttype == "pdr" else "hoststack-latency",
 493                 itm["id"],
 494                 df,
 495                 get_color(idx),
 496                 norm_factor
 497             )
 498             if traces:
 499                 if not fig_lat:
 500                     fig_lat = go.Figure()
 501                 fig_lat.add_traces(traces)
 502
 503         y_units.update(units)
 504
 505     if fig_tput:
 506         fig_layout = layout.get("plot-trending-tput", dict())
 507         fig_layout["yaxis"]["title"] = \
 508             f"Throughput [{'|'.join(sorted(y_units))}]"
 509         fig_tput.update_layout(fig_layout)
 510     if fig_band:
 511         fig_band.update_layout(layout.get("plot-trending-bandwidth", dict()))
 512     if fig_lat:
 513         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 514
 515     return fig_tput, fig_band, fig_lat
 516
 517
 518 def graph_tm_trending(
 519         data: pd.DataFrame,
 520         layout: dict,
 521         all_in_one: bool=False
 522     ) -> list:
 523     """Generates one trending graph per test, each graph includes all selected
 524     metrics.
 525
 526     :param data: Data frame with telemetry data.
 527     :param layout: Layout of plot.ly graph.
 528     :param all_in_one: If True, all telemetry traces are placed in one graph,
 529         otherwise they are split to separate graphs grouped by test ID.
 530     :type data: pandas.DataFrame
 531     :type layout: dict
 532     :type all_in_one: bool
 533     :returns: List of generated graphs together with test names.
 534         list(tuple(plotly.graph_objects.Figure(), str()), tuple(...), ...)
 535     :rtype: list
 536     """
 537
 538     if data.empty:
 539         return list()
 540
 541     def _generate_traces(
 542             data: pd.DataFrame,
 543             test: str,
 544             all_in_one: bool,
 545             color_index: int
 546         ) -> list:
 547         """Generates a trending graph for given test with all metrics.
 548
 549         :param data: Data frame with telemetry data for the given test.
 550         :param test: The name of the test.
 551         :param all_in_one: If True, all telemetry traces are placed in one
 552             graph, otherwise they are split to separate graphs grouped by
 553             test ID.
 554         :param color_index: The index of the test used if all_in_one is True.
 555         :type data: pandas.DataFrame
 556         :type test: str
 557         :type all_in_one: bool
 558         :type color_index: int
 559         :returns: List of traces.
 560         :rtype: list
 561         """
 562         traces = list()
 563         metrics = data.tm_metric.unique().tolist()
 564         for idx, metric in enumerate(metrics):
 565             if "-pdr" in test and "='pdr'" not in metric:
 566                 continue
 567             if "-ndr" in test and "='ndr'" not in metric:
 568                 continue
 569
 570             df = data.loc[(data["tm_metric"] == metric)]
 571             x_axis = df["start_time"].tolist()
 572             y_data = [float(itm) for itm in df["tm_value"].tolist()]
 573             hover = list()
 574             for i, (_, row) in enumerate(df.iterrows()):
 575                 if row["test_type"] == "mrr":
 576                     rate = (
 577                         f"mrr avg [{row[C.UNIT['mrr']]}]: "
 578                         f"{row[C.VALUE['mrr']]:,.0f}<br>"
 579                         f"mrr stdev [{row[C.UNIT['mrr']]}]: "
 580                         f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 581                     )
 582                 elif row["test_type"] == "ndrpdr":
 583                     if "-pdr" in test:
 584                         rate = (
 585                             f"pdr [{row[C.UNIT['pdr']]}]: "
 586                             f"{row[C.VALUE['pdr']]:,.0f}<br>"
 587                         )
 588                     elif "-ndr" in test:
 589                         rate = (
 590                             f"ndr [{row[C.UNIT['ndr']]}]: "
 591                             f"{row[C.VALUE['ndr']]:,.0f}<br>"
 592                         )
 593                     else:
 594                         rate = str()
 595                 else:
 596                     rate = str()
 597                 hover.append(
 598                     f"date: "
 599                     f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 600                     f"value: {y_data[i]:,.2f}<br>"
 601                     f"{rate}"
 602                     f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 603                     f"csit-ref: {row['job']}/{row['build']}<br>"
 604                 )
 605             if any(y_data):
 606                 anomalies, trend_avg, trend_stdev = classify_anomalies(
 607                     {k: v for k, v in zip(x_axis, y_data)}
 608                 )
 609                 hover_trend = list()
 610                 for avg, stdev, (_, row) in \
 611                         zip(trend_avg, trend_stdev, df.iterrows()):
 612                     hover_trend.append(
 613                         f"date: "
 614                         f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 615                         f"trend: {avg:,.2f}<br>"
 616                         f"stdev: {stdev:,.2f}<br>"
 617                         f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 618                         f"csit-ref: {row['job']}/{row['build']}"
 619                     )
 620             else:
 621                 anomalies = None
 622             if all_in_one:
 623                 color = get_color(color_index * len(metrics) + idx)
 624                 metric_name = f"{test}<br>{metric}"
 625             else:
 626                 color = get_color(idx)
 627                 metric_name = metric
 628
 629             traces.append(
 630                 go.Scatter(  # Samples
 631                     x=x_axis,
 632                     y=y_data,
 633                     name=metric_name,
 634                     mode="markers",
 635                     marker={
 636                         "size": 5,
 637                         "color": color,
 638                         "symbol": "circle",
 639                     },
 640                     text=hover,
 641                     hoverinfo="text+name",
 642                     showlegend=True,
 643                     legendgroup=metric_name
 644                 )
 645             )
 646             if anomalies:
 647                 traces.append(
 648                     go.Scatter(  # Trend line
 649                         x=x_axis,
 650                         y=trend_avg,
 651                         name=metric_name,
 652                         mode="lines",
 653                         line={
 654                             "shape": "linear",
 655                             "width": 1,
 656                             "color": color,
 657                         },
 658                         text=hover_trend,
 659                         hoverinfo="text+name",
 660                         showlegend=False,
 661                         legendgroup=metric_name
 662                     )
 663                 )
 664
 665                 anomaly_x = list()
 666                 anomaly_y = list()
 667                 anomaly_color = list()
 668                 hover = list()
 669                 for idx, anomaly in enumerate(anomalies):
 670                     if anomaly in ("regression", "progression"):
 671                         anomaly_x.append(x_axis[idx])
 672                         anomaly_y.append(trend_avg[idx])
 673                         anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 674                         hover_itm = (
 675                             f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}"
 676                             f"<br>trend: {trend_avg[idx]:,.2f}"
 677                             f"<br>classification: {anomaly}"
 678                         )
 679                         hover.append(hover_itm)
 680                 anomaly_color.extend([0.0, 0.5, 1.0])
 681                 traces.append(
 682                     go.Scatter(
 683                         x=anomaly_x,
 684                         y=anomaly_y,
 685                         mode="markers",
 686                         text=hover,
 687                         hoverinfo="text+name",
 688                         showlegend=False,
 689                         legendgroup=metric_name,
 690                         name=metric_name,
 691                         marker={
 692                             "size": 15,
 693                             "symbol": "circle-open",
 694                             "color": anomaly_color,
 695                             "colorscale": C.COLORSCALE_TPUT,
 696                             "showscale": True,
 697                             "line": {
 698                                 "width": 2
 699                             },
 700                             "colorbar": {
 701                                 "y": 0.5,
 702                                 "len": 0.8,
 703                                 "title": "Circles Marking Data Classification",
 704                                 "titleside": "right",
 705                                 "tickmode": "array",
 706                                 "tickvals": [0.167, 0.500, 0.833],
 707                                 "ticktext": C.TICK_TEXT_TPUT,
 708                                 "ticks": "",
 709                                 "ticklen": 0,
 710                                 "tickangle": -90,
 711                                 "thickness": 10
 712                             }
 713                         }
 714                     )
 715                 )
 716
 717         unique_metrics = set()
 718         for itm in metrics:
 719             unique_metrics.add(itm.split("{", 1)[0])
 720         return traces, unique_metrics
 721
 722     tm_trending_graphs = list()
 723     graph_layout = layout.get("plot-trending-telemetry", dict())
 724
 725     if all_in_one:
 726         all_traces = list()
 727
 728     all_metrics = set()
 729     all_tests = list()
 730     for idx, test in enumerate(data.test_name.unique()):
 731         df = data.loc[(data["test_name"] == test)]
 732         traces, metrics = _generate_traces(df, test, all_in_one, idx)
 733         if traces:
 734             all_metrics.update(metrics)
 735             if all_in_one:
 736                 all_traces.extend(traces)
 737                 all_tests.append(test)
 738             else:
 739                 graph = go.Figure()
 740                 graph.add_traces(traces)
 741                 graph.update_layout(graph_layout)
 742                 tm_trending_graphs.append((graph, [test, ], ))
 743
 744     if all_in_one:
 745         graph = go.Figure()
 746         graph.add_traces(all_traces)
 747         graph.update_layout(graph_layout)
 748         tm_trending_graphs.append((graph, all_tests, ))
 749
 750     return tm_trending_graphs, list(all_metrics)