csit.infra.dash/app/cdash/trending/graphs.py

   1 # Copyright (c) 2024 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Implementation of graphs for trending data.
  15 """
  16
  17 import logging
  18 import plotly.graph_objects as go
  19 import pandas as pd
  20
  21 from ..utils.constants import Constants as C
  22 from ..utils.utils import get_color, get_hdrh_latencies
  23 from ..utils.anomalies import classify_anomalies
  24
  25
  26 def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
  27     """Select the data for graphs from the provided data frame.
  28
  29     :param data: Data frame with data for graphs.
  30     :param itm: Item (in this case job name) which data will be selected from
  31         the input data frame.
  32     :type data: pandas.DataFrame
  33     :type itm: dict
  34     :returns: A data frame with selected data.
  35     :rtype: pandas.DataFrame
  36     """
  37
  38     phy = itm["phy"].split("-")
  39     if len(phy) == 4:
  40         topo, arch, nic, drv = phy
  41         if drv == "dpdk":
  42             drv = ""
  43         else:
  44             drv += "-"
  45             drv = drv.replace("_", "-")
  46     else:
  47         return None
  48
  49     if itm["testtype"] in ("ndr", "pdr"):
  50         test_type = "ndrpdr"
  51     elif itm["testtype"] == "mrr":
  52         test_type = "mrr"
  53     elif itm["testtype"] == "soak":
  54         test_type = "soak"
  55     elif itm["area"] == "hoststack":
  56         test_type = "hoststack"
  57     df = data.loc[(
  58         (data["test_type"] == test_type) &
  59         (data["passed"] == True)
  60     )]
  61     df = df[df.job.str.endswith(f"{topo}-{arch}")]
  62     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
  63     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
  64     df = df[df.test_id.str.contains(
  65         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
  66         regex=True
  67     )].sort_values(by="start_time", ignore_index=True)
  68
  69     return df
  70
  71
  72 def graph_trending(
  73         data: pd.DataFrame,
  74         sel: dict,
  75         layout: dict,
  76         normalize: bool=False
  77     ) -> tuple:
  78     """Generate the trending graph(s) - MRR, NDR, PDR and for PDR also Latences
  79     (result_latency_forward_pdr_50_avg).
  80
  81     :param data: Data frame with test results.
  82     :param sel: Selected tests.
  83     :param layout: Layout of plot.ly graph.
  84     :param normalize: If True, the data is normalized to CPU frquency
  85         Constants.NORM_FREQUENCY.
  86     :type data: pandas.DataFrame
  87     :type sel: dict
  88     :type layout: dict
  89     :type normalize: bool
  90     :returns: Trending graph(s)
  91     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
  92     """
  93
  94     if not sel:
  95         return None, None
  96
  97
  98     def _generate_trending_traces(
  99             ttype: str,
 100             name: str,
 101             df: pd.DataFrame,
 102             color: str,
 103             nf: float
 104         ) -> list:
 105         """Generate the trending traces for the trending graph.
 106
 107         :param ttype: Test type (MRR, NDR, PDR).
 108         :param name: The test name to be displayed as the graph title.
 109         :param df: Data frame with test data.
 110         :param color: The color of the trace (samples and trend line).
 111         :param nf: The factor used for normalization of the results to
 112             CPU frequency set to Constants.NORM_FREQUENCY.
 113         :type ttype: str
 114         :type name: str
 115         :type df: pandas.DataFrame
 116         :type color: str
 117         :type nf: float
 118         :returns: Traces (samples, trending line, anomalies)
 119         :rtype: list
 120         """
 121
 122         df = df.dropna(subset=[C.VALUE[ttype], ])
 123         if df.empty:
 124             return list(), list()
 125
 126         hover = list()
 127         customdata = list()
 128         customdata_samples = list()
 129         name_lst = name.split("-")
 130         for _, row in df.iterrows():
 131             h_tput, h_band, h_lat = str(), str(), str()
 132             if ttype in ("mrr", "mrr-bandwidth"):
 133                 h_tput = (
 134                     f"tput avg [{row['result_receive_rate_rate_unit']}]: "
 135                     f"{row['result_receive_rate_rate_avg'] * nf:,.0f}<br>"
 136                     f"tput stdev [{row['result_receive_rate_rate_unit']}]: "
 137                     f"{row['result_receive_rate_rate_stdev'] * nf:,.0f}<br>"
 138                 )
 139                 if pd.notna(row["result_receive_rate_bandwidth_avg"]):
 140                     h_band = (
 141                         f"bandwidth avg "
 142                         f"[{row['result_receive_rate_bandwidth_unit']}]: "
 143                         f"{row['result_receive_rate_bandwidth_avg'] * nf:,.0f}"
 144                         "<br>"
 145                         f"bandwidth stdev "
 146                         f"[{row['result_receive_rate_bandwidth_unit']}]: "
 147                         f"{row['result_receive_rate_bandwidth_stdev']* nf:,.0f}"
 148                         "<br>"
 149                     )
 150             elif ttype in ("ndr", "ndr-bandwidth"):
 151                 h_tput = (
 152                     f"tput [{row['result_ndr_lower_rate_unit']}]: "
 153                     f"{row['result_ndr_lower_rate_value'] * nf:,.0f}<br>"
 154                 )
 155                 if pd.notna(row["result_ndr_lower_bandwidth_value"]):
 156                     h_band = (
 157                         f"bandwidth [{row['result_ndr_lower_bandwidth_unit']}]:"
 158                         f" {row['result_ndr_lower_bandwidth_value'] * nf:,.0f}"
 159                         "<br>"
 160                     )
 161             elif ttype in ("pdr", "pdr-bandwidth", "latency"):
 162                 h_tput = (
 163                     f"tput [{row['result_pdr_lower_rate_unit']}]: "
 164                     f"{row['result_pdr_lower_rate_value'] * nf:,.0f}<br>"
 165                 )
 166                 if pd.notna(row["result_pdr_lower_bandwidth_value"]):
 167                     h_band = (
 168                         f"bandwidth [{row['result_pdr_lower_bandwidth_unit']}]:"
 169                         f" {row['result_pdr_lower_bandwidth_value'] * nf:,.0f}"
 170                         "<br>"
 171                     )
 172                 if pd.notna(row["result_latency_forward_pdr_50_avg"]):
 173                     h_lat = (
 174                         f"latency "
 175                         f"[{row['result_latency_forward_pdr_50_unit']}]: "
 176                         f"{row['result_latency_forward_pdr_50_avg'] / nf:,.0f}"
 177                         "<br>"
 178                     )
 179             elif ttype in ("hoststack-cps", "hoststack-rps",
 180                            "hoststack-cps-bandwidth",
 181                            "hoststack-rps-bandwidth", "hoststack-latency"):
 182                 h_tput = (
 183                     f"tput [{row['result_rate_unit']}]: "
 184                     f"{row['result_rate_value'] * nf:,.0f}<br>"
 185                 )
 186                 h_band = (
 187                     f"bandwidth [{row['result_bandwidth_unit']}]: "
 188                     f"{row['result_bandwidth_value'] * nf:,.0f}<br>"
 189                 )
 190                 h_lat = (
 191                     f"latency [{row['result_latency_unit']}]: "
 192                     f"{row['result_latency_value'] / nf:,.0f}<br>"
 193                 )
 194             elif ttype in ("hoststack-bps", ):
 195                 h_band = (
 196                     f"bandwidth [{row['result_bandwidth_unit']}]: "
 197                     f"{row['result_bandwidth_value'] * nf:,.0f}<br>"
 198                 )
 199             elif ttype in ("soak", "soak-bandwidth"):
 200                 h_tput = (
 201                     f"tput [{row['result_critical_rate_lower_rate_unit']}]: "
 202                     f"{row['result_critical_rate_lower_rate_value'] * nf:,.0f}"
 203                     "<br>"
 204                 )
 205                 if pd.notna(row["result_critical_rate_lower_bandwidth_value"]):
 206                     bv = row['result_critical_rate_lower_bandwidth_value']
 207                     h_band = (
 208                         "bandwidth "
 209                         f"[{row['result_critical_rate_lower_bandwidth_unit']}]:"
 210                         f" {bv * nf:,.0f}"
 211                         "<br>"
 212                     )
 213             try:
 214                 hosts = f"<br>hosts: {', '.join(row['hosts'])}"
 215             except (KeyError, TypeError):
 216                 hosts = str()
 217             hover_itm = (
 218                 f"dut: {name_lst[0]}<br>"
 219                 f"infra: {'-'.join(name_lst[1:5])}<br>"
 220                 f"test: {'-'.join(name_lst[5:])}<br>"
 221                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 222                 f"{h_tput}{h_band}{h_lat}"
 223                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 224                 f"csit-ref: {row['job']}/{row['build']}"
 225                 f"{hosts}"
 226             )
 227             hover.append(hover_itm)
 228             if ttype == "latency":
 229                 customdata_samples.append(get_hdrh_latencies(row, name))
 230                 customdata.append({"name": name})
 231             else:
 232                 customdata_samples.append(
 233                     {"name": name, "show_telemetry": True}
 234                 )
 235                 customdata.append({"name": name})
 236
 237         x_axis = df["start_time"].tolist()
 238         if "latency" in ttype:
 239             y_data = [(v / nf) for v in df[C.VALUE[ttype]].tolist()]
 240         else:
 241             y_data = [(v * nf) for v in df[C.VALUE[ttype]].tolist()]
 242         units = df[C.UNIT[ttype]].unique().tolist()
 243
 244         try:
 245             anomalies, trend_avg, trend_stdev = classify_anomalies(
 246                 {k: v for k, v in zip(x_axis, y_data)}
 247             )
 248         except ValueError as err:
 249             logging.error(err)
 250             return list(), list()
 251
 252         hover_trend = list()
 253         for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 254             try:
 255                 hosts = f"<br>hosts: {', '.join(row['hosts'])}"
 256             except (KeyError, TypeError):
 257                 hosts = str()
 258             hover_itm = (
 259                 f"dut: {name_lst[0]}<br>"
 260                 f"infra: {'-'.join(name_lst[1:5])}<br>"
 261                 f"test: {'-'.join(name_lst[5:])}<br>"
 262                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 263                 f"trend [{row[C.UNIT[ttype]]}]: {avg:,.0f}<br>"
 264                 f"stdev [{row[C.UNIT[ttype]]}]: {stdev:,.0f}<br>"
 265                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 266                 f"csit-ref: {row['job']}/{row['build']}"
 267                 f"{hosts}"
 268             )
 269             if ttype == "latency":
 270                 hover_itm = hover_itm.replace("[pps]", "[us]")
 271             hover_trend.append(hover_itm)
 272
 273         traces = [
 274             go.Scatter(  # Samples
 275                 x=x_axis,
 276                 y=y_data,
 277                 name=name,
 278                 mode="markers",
 279                 marker={
 280                     "size": 5,
 281                     "color": color,
 282                     "symbol": "circle",
 283                 },
 284                 text=hover,
 285                 hoverinfo="text",
 286                 showlegend=True,
 287                 legendgroup=name,
 288                 customdata=customdata_samples
 289             ),
 290             go.Scatter(  # Trend line
 291                 x=x_axis,
 292                 y=trend_avg,
 293                 name=name,
 294                 mode="lines",
 295                 line={
 296                     "shape": "linear",
 297                     "width": 1,
 298                     "color": color,
 299                 },
 300                 text=hover_trend,
 301                 hoverinfo="text",
 302                 showlegend=False,
 303                 legendgroup=name,
 304                 customdata=customdata
 305             )
 306         ]
 307
 308         if anomalies:
 309             anomaly_x = list()
 310             anomaly_y = list()
 311             anomaly_color = list()
 312             hover = list()
 313             for idx, anomaly in enumerate(anomalies):
 314                 if anomaly in ("regression", "progression"):
 315                     anomaly_x.append(x_axis[idx])
 316                     anomaly_y.append(trend_avg[idx])
 317                     anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 318                     hover_itm = (
 319                         f"dut: {name_lst[0]}<br>"
 320                         f"infra: {'-'.join(name_lst[1:5])}<br>"
 321                         f"test: {'-'.join(name_lst[5:])}<br>"
 322                         f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 323                         f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
 324                         f"classification: {anomaly}"
 325                     )
 326                     if ttype == "latency":
 327                         hover_itm = hover_itm.replace("[pps]", "[us]")
 328                     hover.append(hover_itm)
 329             anomaly_color.extend([0.0, 0.5, 1.0])
 330             traces.append(
 331                 go.Scatter(
 332                     x=anomaly_x,
 333                     y=anomaly_y,
 334                     mode="markers",
 335                     text=hover,
 336                     hoverinfo="text",
 337                     showlegend=False,
 338                     legendgroup=name,
 339                     name=name,
 340                     customdata=customdata,
 341                     marker={
 342                         "size": 15,
 343                         "symbol": "circle-open",
 344                         "color": anomaly_color,
 345                         "colorscale": C.COLORSCALE_LAT \
 346                             if ttype == "latency" else C.COLORSCALE_TPUT,
 347                         "showscale": True,
 348                         "line": {
 349                             "width": 2
 350                         },
 351                         "colorbar": {
 352                             "y": 0.5,
 353                             "len": 0.8,
 354                             "title": "Circles Marking Data Classification",
 355                             "titleside": "right",
 356                             "tickmode": "array",
 357                             "tickvals": [0.167, 0.500, 0.833],
 358                             "ticktext": C.TICK_TEXT_LAT \
 359                                 if ttype == "latency" else C.TICK_TEXT_TPUT,
 360                             "ticks": "",
 361                             "ticklen": 0,
 362                             "tickangle": -90,
 363                             "thickness": 10
 364                         }
 365                     }
 366                 )
 367             )
 368
 369         return traces, units
 370
 371
 372     fig_tput = None
 373     fig_lat = None
 374     fig_band = None
 375     y_units = set()
 376     for idx, itm in enumerate(sel):
 377         df = select_trending_data(data, itm)
 378         if df is None or df.empty:
 379             continue
 380
 381         if normalize:
 382             phy = itm["phy"].split("-")
 383             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 384             norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY.get(topo_arch, 1.0)) \
 385                 if topo_arch else 1.0
 386         else:
 387             norm_factor = 1.0
 388
 389         if itm["area"] == "hoststack":
 390             ttype = f"hoststack-{itm['testtype']}"
 391         else:
 392             ttype = itm["testtype"]
 393
 394         traces, units = _generate_trending_traces(
 395             ttype,
 396             itm["id"],
 397             df,
 398             get_color(idx),
 399             norm_factor
 400         )
 401         if traces:
 402             if not fig_tput:
 403                 fig_tput = go.Figure()
 404             fig_tput.add_traces(traces)
 405
 406         if ttype in C.TESTS_WITH_BANDWIDTH:
 407             traces, _ = _generate_trending_traces(
 408                 f"{ttype}-bandwidth",
 409                 itm["id"],
 410                 df,
 411                 get_color(idx),
 412                 norm_factor
 413             )
 414             if traces:
 415                 if not fig_band:
 416                     fig_band = go.Figure()
 417                 fig_band.add_traces(traces)
 418
 419         if ttype in C.TESTS_WITH_LATENCY:
 420             traces, _ = _generate_trending_traces(
 421                 "latency" if ttype == "pdr" else "hoststack-latency",
 422                 itm["id"],
 423                 df,
 424                 get_color(idx),
 425                 norm_factor
 426             )
 427             if traces:
 428                 if not fig_lat:
 429                     fig_lat = go.Figure()
 430                 fig_lat.add_traces(traces)
 431
 432         y_units.update(units)
 433
 434     if fig_tput:
 435         fig_layout = layout.get("plot-trending-tput", dict())
 436         fig_layout["yaxis"]["title"] = \
 437             f"Throughput [{'|'.join(sorted(y_units))}]"
 438         fig_tput.update_layout(fig_layout)
 439     if fig_band:
 440         fig_band.update_layout(layout.get("plot-trending-bandwidth", dict()))
 441     if fig_lat:
 442         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 443
 444     return fig_tput, fig_band, fig_lat
 445
 446
 447 def graph_tm_trending(
 448         data: pd.DataFrame,
 449         layout: dict,
 450         all_in_one: bool=False
 451     ) -> list:
 452     """Generates one trending graph per test, each graph includes all selected
 453     metrics.
 454
 455     :param data: Data frame with telemetry data.
 456     :param layout: Layout of plot.ly graph.
 457     :param all_in_one: If True, all telemetry traces are placed in one graph,
 458         otherwise they are split to separate graphs grouped by test ID.
 459     :type data: pandas.DataFrame
 460     :type layout: dict
 461     :type all_in_one: bool
 462     :returns: List of generated graphs together with test names.
 463         list(tuple(plotly.graph_objects.Figure(), str()), tuple(...), ...)
 464     :rtype: list
 465     """
 466
 467     if data.empty:
 468         return list()
 469
 470     def _generate_traces(
 471             data: pd.DataFrame,
 472             test: str,
 473             all_in_one: bool,
 474             color_index: int
 475         ) -> list:
 476         """Generates a trending graph for given test with all metrics.
 477
 478         :param data: Data frame with telemetry data for the given test.
 479         :param test: The name of the test.
 480         :param all_in_one: If True, all telemetry traces are placed in one
 481             graph, otherwise they are split to separate graphs grouped by
 482             test ID.
 483         :param color_index: The index of the test used if all_in_one is True.
 484         :type data: pandas.DataFrame
 485         :type test: str
 486         :type all_in_one: bool
 487         :type color_index: int
 488         :returns: List of traces.
 489         :rtype: list
 490         """
 491         traces = list()
 492         metrics = data.tm_metric.unique().tolist()
 493         for idx, metric in enumerate(metrics):
 494             if "-pdr" in test and "='pdr'" not in metric:
 495                 continue
 496             if "-ndr" in test and "='ndr'" not in metric:
 497                 continue
 498
 499             df = data.loc[(data["tm_metric"] == metric)]
 500             x_axis = df["start_time"].tolist()
 501             y_data = [float(itm) for itm in df["tm_value"].tolist()]
 502             hover = list()
 503             for i, (_, row) in enumerate(df.iterrows()):
 504                 if row["test_type"] == "mrr":
 505                     rate = (
 506                         f"mrr avg [{row[C.UNIT['mrr']]}]: "
 507                         f"{row[C.VALUE['mrr']]:,.0f}<br>"
 508                         f"mrr stdev [{row[C.UNIT['mrr']]}]: "
 509                         f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 510                     )
 511                 elif row["test_type"] == "ndrpdr":
 512                     if "-pdr" in test:
 513                         rate = (
 514                             f"pdr [{row[C.UNIT['pdr']]}]: "
 515                             f"{row[C.VALUE['pdr']]:,.0f}<br>"
 516                         )
 517                     elif "-ndr" in test:
 518                         rate = (
 519                             f"ndr [{row[C.UNIT['ndr']]}]: "
 520                             f"{row[C.VALUE['ndr']]:,.0f}<br>"
 521                         )
 522                     else:
 523                         rate = str()
 524                 else:
 525                     rate = str()
 526                 hover.append(
 527                     f"date: "
 528                     f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 529                     f"value: {y_data[i]:,.2f}<br>"
 530                     f"{rate}"
 531                     f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 532                     f"csit-ref: {row['job']}/{row['build']}<br>"
 533                 )
 534             if any(y_data):
 535                 anomalies, trend_avg, trend_stdev = classify_anomalies(
 536                     {k: v for k, v in zip(x_axis, y_data)}
 537                 )
 538                 hover_trend = list()
 539                 for avg, stdev, (_, row) in \
 540                         zip(trend_avg, trend_stdev, df.iterrows()):
 541                     hover_trend.append(
 542                         f"date: "
 543                         f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 544                         f"trend: {avg:,.2f}<br>"
 545                         f"stdev: {stdev:,.2f}<br>"
 546                         f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 547                         f"csit-ref: {row['job']}/{row['build']}"
 548                     )
 549             else:
 550                 anomalies = None
 551             if all_in_one:
 552                 color = get_color(color_index * len(metrics) + idx)
 553                 metric_name = f"{test}<br>{metric}"
 554             else:
 555                 color = get_color(idx)
 556                 metric_name = metric
 557
 558             traces.append(
 559                 go.Scatter(  # Samples
 560                     x=x_axis,
 561                     y=y_data,
 562                     name=metric_name,
 563                     mode="markers",
 564                     marker={
 565                         "size": 5,
 566                         "color": color,
 567                         "symbol": "circle",
 568                     },
 569                     text=hover,
 570                     hoverinfo="text+name",
 571                     showlegend=True,
 572                     legendgroup=metric_name
 573                 )
 574             )
 575             if anomalies:
 576                 traces.append(
 577                     go.Scatter(  # Trend line
 578                         x=x_axis,
 579                         y=trend_avg,
 580                         name=metric_name,
 581                         mode="lines",
 582                         line={
 583                             "shape": "linear",
 584                             "width": 1,
 585                             "color": color,
 586                         },
 587                         text=hover_trend,
 588                         hoverinfo="text+name",
 589                         showlegend=False,
 590                         legendgroup=metric_name
 591                     )
 592                 )
 593
 594                 anomaly_x = list()
 595                 anomaly_y = list()
 596                 anomaly_color = list()
 597                 hover = list()
 598                 for idx, anomaly in enumerate(anomalies):
 599                     if anomaly in ("regression", "progression"):
 600                         anomaly_x.append(x_axis[idx])
 601                         anomaly_y.append(trend_avg[idx])
 602                         anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 603                         hover_itm = (
 604                             f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}"
 605                             f"<br>trend: {trend_avg[idx]:,.2f}"
 606                             f"<br>classification: {anomaly}"
 607                         )
 608                         hover.append(hover_itm)
 609                 anomaly_color.extend([0.0, 0.5, 1.0])
 610                 traces.append(
 611                     go.Scatter(
 612                         x=anomaly_x,
 613                         y=anomaly_y,
 614                         mode="markers",
 615                         text=hover,
 616                         hoverinfo="text+name",
 617                         showlegend=False,
 618                         legendgroup=metric_name,
 619                         name=metric_name,
 620                         marker={
 621                             "size": 15,
 622                             "symbol": "circle-open",
 623                             "color": anomaly_color,
 624                             "colorscale": C.COLORSCALE_TPUT,
 625                             "showscale": True,
 626                             "line": {
 627                                 "width": 2
 628                             },
 629                             "colorbar": {
 630                                 "y": 0.5,
 631                                 "len": 0.8,
 632                                 "title": "Circles Marking Data Classification",
 633                                 "titleside": "right",
 634                                 "tickmode": "array",
 635                                 "tickvals": [0.167, 0.500, 0.833],
 636                                 "ticktext": C.TICK_TEXT_TPUT,
 637                                 "ticks": "",
 638                                 "ticklen": 0,
 639                                 "tickangle": -90,
 640                                 "thickness": 10
 641                             }
 642                         }
 643                     )
 644                 )
 645
 646         unique_metrics = set()
 647         for itm in metrics:
 648             unique_metrics.add(itm.split("{", 1)[0])
 649         return traces, unique_metrics
 650
 651     tm_trending_graphs = list()
 652     graph_layout = layout.get("plot-trending-telemetry", dict())
 653
 654     if all_in_one:
 655         all_traces = list()
 656
 657     all_metrics = set()
 658     all_tests = list()
 659     for idx, test in enumerate(data.test_name.unique()):
 660         df = data.loc[(data["test_name"] == test)]
 661         traces, metrics = _generate_traces(df, test, all_in_one, idx)
 662         if traces:
 663             all_metrics.update(metrics)
 664             if all_in_one:
 665                 all_traces.extend(traces)
 666                 all_tests.append(test)
 667             else:
 668                 graph = go.Figure()
 669                 graph.add_traces(traces)
 670                 graph.update_layout(graph_layout)
 671                 tm_trending_graphs.append((graph, [test, ], ))
 672
 673     if all_in_one:
 674         graph = go.Figure()
 675         graph.add_traces(all_traces)
 676         graph.update_layout(graph_layout)
 677         tm_trending_graphs.append((graph, all_tests, ))
 678
 679     return tm_trending_graphs, list(all_metrics)