csit.infra.dash/app/cdash/trending/graphs.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Implementation of graphs for trending data.
  15 """
  16
  17 import logging
  18 import plotly.graph_objects as go
  19 import pandas as pd
  20
  21 from ..utils.constants import Constants as C
  22 from ..utils.utils import get_color, get_hdrh_latencies
  23 from ..utils.anomalies import classify_anomalies
  24
  25
  26 def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
  27     """Select the data for graphs from the provided data frame.
  28
  29     :param data: Data frame with data for graphs.
  30     :param itm: Item (in this case job name) which data will be selected from
  31         the input data frame.
  32     :type data: pandas.DataFrame
  33     :type itm: dict
  34     :returns: A data frame with selected data.
  35     :rtype: pandas.DataFrame
  36     """
  37
  38     phy = itm["phy"].split("-")
  39     if len(phy) == 4:
  40         topo, arch, nic, drv = phy
  41         if drv == "dpdk":
  42             drv = ""
  43         else:
  44             drv += "-"
  45             drv = drv.replace("_", "-")
  46     else:
  47         return None
  48
  49     if itm["testtype"] in ("ndr", "pdr"):
  50         test_type = "ndrpdr"
  51     elif itm["testtype"] == "mrr":
  52         test_type = "mrr"
  53     elif itm["area"] == "hoststack":
  54         test_type = "hoststack"
  55     df = data.loc[(
  56         (data["test_type"] == test_type) &
  57         (data["passed"] == True)
  58     )]
  59     df = df[df.job.str.endswith(f"{topo}-{arch}")]
  60     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
  61     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
  62     df = df[df.test_id.str.contains(
  63         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
  64         regex=True
  65     )].sort_values(by="start_time", ignore_index=True)
  66
  67     return df
  68
  69
  70 def graph_trending(
  71         data: pd.DataFrame,
  72         sel: dict,
  73         layout: dict,
  74         normalize: bool
  75     ) -> tuple:
  76     """Generate the trending graph(s) - MRR, NDR, PDR and for PDR also Latences
  77     (result_latency_forward_pdr_50_avg).
  78
  79     :param data: Data frame with test results.
  80     :param sel: Selected tests.
  81     :param layout: Layout of plot.ly graph.
  82     :param normalize: If True, the data is normalized to CPU frquency
  83         Constants.NORM_FREQUENCY.
  84     :type data: pandas.DataFrame
  85     :type sel: dict
  86     :type layout: dict
  87     :type normalize: bool
  88     :returns: Trending graph(s)
  89     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
  90     """
  91
  92     if not sel:
  93         return None, None
  94
  95
  96     def _generate_trending_traces(
  97             ttype: str,
  98             name: str,
  99             df: pd.DataFrame,
 100             color: str,
 101             nf: float
 102         ) -> list:
 103         """Generate the trending traces for the trending graph.
 104
 105         :param ttype: Test type (MRR, NDR, PDR).
 106         :param name: The test name to be displayed as the graph title.
 107         :param df: Data frame with test data.
 108         :param color: The color of the trace (samples and trend line).
 109         :param nf: The factor used for normalization of the results to
 110             CPU frequency set to Constants.NORM_FREQUENCY.
 111         :type ttype: str
 112         :type name: str
 113         :type df: pandas.DataFrame
 114         :type color: str
 115         :type nf: float
 116         :returns: Traces (samples, trending line, anomalies)
 117         :rtype: list
 118         """
 119
 120         df = df.dropna(subset=[C.VALUE[ttype], ])
 121         if df.empty:
 122             return list(), list()
 123
 124         hover = list()
 125         customdata = list()
 126         customdata_samples = list()
 127         name_lst = name.split("-")
 128         for _, row in df.iterrows():
 129             h_tput, h_band, h_lat = str(), str(), str()
 130             if ttype in ("mrr", "mrr-bandwidth"):
 131                 h_tput = (
 132                     f"tput avg [{row['result_receive_rate_rate_unit']}]: "
 133                     f"{row['result_receive_rate_rate_avg'] * nf:,.0f}<br>"
 134                     f"tput stdev [{row['result_receive_rate_rate_unit']}]: "
 135                     f"{row['result_receive_rate_rate_stdev'] * nf:,.0f}<br>"
 136                 )
 137                 if pd.notna(row["result_receive_rate_bandwidth_avg"]):
 138                     h_band = (
 139                         f"bandwidth avg "
 140                         f"[{row['result_receive_rate_bandwidth_unit']}]: "
 141                         f"{row['result_receive_rate_bandwidth_avg'] * nf:,.0f}"
 142                         "<br>"
 143                         f"bandwidth stdev "
 144                         f"[{row['result_receive_rate_bandwidth_unit']}]: "
 145                         f"{row['result_receive_rate_bandwidth_stdev']* nf:,.0f}"
 146                         "<br>"
 147                     )
 148             elif ttype in ("ndr", "ndr-bandwidth"):
 149                 h_tput = (
 150                     f"tput [{row['result_ndr_lower_rate_unit']}]: "
 151                     f"{row['result_ndr_lower_rate_value'] * nf:,.0f}<br>"
 152                 )
 153                 if pd.notna(row["result_ndr_lower_bandwidth_value"]):
 154                     h_band = (
 155                         f"bandwidth [{row['result_ndr_lower_bandwidth_unit']}]:"
 156                         f" {row['result_ndr_lower_bandwidth_value'] * nf:,.0f}"
 157                         "<br>"
 158                     )
 159             elif ttype in ("pdr", "pdr-bandwidth", "latency"):
 160                 h_tput = (
 161                     f"tput [{row['result_pdr_lower_rate_unit']}]: "
 162                     f"{row['result_pdr_lower_rate_value'] * nf:,.0f}<br>"
 163                 )
 164                 if pd.notna(row["result_pdr_lower_bandwidth_value"]):
 165                     h_band = (
 166                         f"bandwidth [{row['result_pdr_lower_bandwidth_unit']}]:"
 167                         f" {row['result_pdr_lower_bandwidth_value'] * nf:,.0f}"
 168                         "<br>"
 169                     )
 170                 if pd.notna(row["result_latency_forward_pdr_50_avg"]):
 171                     h_lat = (
 172                         f"latency "
 173                         f"[{row['result_latency_forward_pdr_50_unit']}]: "
 174                         f"{row['result_latency_forward_pdr_50_avg'] / nf:,.0f}"
 175                         "<br>"
 176                     )
 177             elif ttype in ("hoststack-cps", "hoststack-rps",
 178                            "hoststack-cps-bandwidth",
 179                            "hoststack-rps-bandwidth", "hoststack-latency"):
 180                 h_tput = (
 181                     f"tput [{row['result_rate_unit']}]: "
 182                     f"{row['result_rate_value'] * nf:,.0f}<br>"
 183                 )
 184                 h_band = (
 185                     f"bandwidth [{row['result_bandwidth_unit']}]: "
 186                     f"{row['result_bandwidth_value'] * nf:,.0f}<br>"
 187                 )
 188                 h_lat = (
 189                     f"latency [{row['result_latency_unit']}]: "
 190                     f"{row['result_latency_value'] / nf:,.0f}<br>"
 191                 )
 192             elif ttype in ("hoststack-bps", ):
 193                 h_band = (
 194                     f"bandwidth [{row['result_bandwidth_unit']}]: "
 195                     f"{row['result_bandwidth_value'] * nf:,.0f}<br>"
 196                 )
 197             hover_itm = (
 198                 f"dut: {name_lst[0]}<br>"
 199                 f"infra: {'-'.join(name_lst[1:5])}<br>"
 200                 f"test: {'-'.join(name_lst[5:])}<br>"
 201                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 202                 f"{h_tput}{h_band}{h_lat}"
 203                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 204                 f"csit-ref: {row['job']}/{row['build']}<br>"
 205                 f"hosts: {', '.join(row['hosts'])}"
 206             )
 207             hover.append(hover_itm)
 208             if ttype == "latency":
 209                 customdata_samples.append(get_hdrh_latencies(row, name))
 210                 customdata.append({"name": name})
 211             else:
 212                 customdata_samples.append(
 213                     {"name": name, "show_telemetry": True}
 214                 )
 215                 customdata.append({"name": name})
 216
 217         x_axis = df["start_time"].tolist()
 218         if "latency" in ttype:
 219             y_data = [(v / nf) for v in df[C.VALUE[ttype]].tolist()]
 220         else:
 221             y_data = [(v * nf) for v in df[C.VALUE[ttype]].tolist()]
 222         units = df[C.UNIT[ttype]].unique().tolist()
 223
 224         try:
 225             anomalies, trend_avg, trend_stdev = classify_anomalies(
 226                 {k: v for k, v in zip(x_axis, y_data)}
 227             )
 228         except ValueError as err:
 229             logging.error(err)
 230             return list(), list()
 231
 232         hover_trend = list()
 233         for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 234             hover_itm = (
 235                 f"dut: {name_lst[0]}<br>"
 236                 f"infra: {'-'.join(name_lst[1:5])}<br>"
 237                 f"test: {'-'.join(name_lst[5:])}<br>"
 238                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 239                 f"trend [{row[C.UNIT[ttype]]}]: {avg:,.0f}<br>"
 240                 f"stdev [{row[C.UNIT[ttype]]}]: {stdev:,.0f}<br>"
 241                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 242                 f"csit-ref: {row['job']}/{row['build']}<br>"
 243                 f"hosts: {', '.join(row['hosts'])}"
 244             )
 245             if ttype == "latency":
 246                 hover_itm = hover_itm.replace("[pps]", "[us]")
 247             hover_trend.append(hover_itm)
 248
 249         traces = [
 250             go.Scatter(  # Samples
 251                 x=x_axis,
 252                 y=y_data,
 253                 name=name,
 254                 mode="markers",
 255                 marker={
 256                     "size": 5,
 257                     "color": color,
 258                     "symbol": "circle",
 259                 },
 260                 text=hover,
 261                 hoverinfo="text",
 262                 showlegend=True,
 263                 legendgroup=name,
 264                 customdata=customdata_samples
 265             ),
 266             go.Scatter(  # Trend line
 267                 x=x_axis,
 268                 y=trend_avg,
 269                 name=name,
 270                 mode="lines",
 271                 line={
 272                     "shape": "linear",
 273                     "width": 1,
 274                     "color": color,
 275                 },
 276                 text=hover_trend,
 277                 hoverinfo="text",
 278                 showlegend=False,
 279                 legendgroup=name,
 280                 customdata=customdata
 281             )
 282         ]
 283
 284         if anomalies:
 285             anomaly_x = list()
 286             anomaly_y = list()
 287             anomaly_color = list()
 288             hover = list()
 289             for idx, anomaly in enumerate(anomalies):
 290                 if anomaly in ("regression", "progression"):
 291                     anomaly_x.append(x_axis[idx])
 292                     anomaly_y.append(trend_avg[idx])
 293                     anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 294                     hover_itm = (
 295                         f"dut: {name_lst[0]}<br>"
 296                         f"infra: {'-'.join(name_lst[1:5])}<br>"
 297                         f"test: {'-'.join(name_lst[5:])}<br>"
 298                         f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 299                         f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
 300                         f"classification: {anomaly}"
 301                     )
 302                     if ttype == "latency":
 303                         hover_itm = hover_itm.replace("[pps]", "[us]")
 304                     hover.append(hover_itm)
 305             anomaly_color.extend([0.0, 0.5, 1.0])
 306             traces.append(
 307                 go.Scatter(
 308                     x=anomaly_x,
 309                     y=anomaly_y,
 310                     mode="markers",
 311                     text=hover,
 312                     hoverinfo="text",
 313                     showlegend=False,
 314                     legendgroup=name,
 315                     name=name,
 316                     customdata=customdata,
 317                     marker={
 318                         "size": 15,
 319                         "symbol": "circle-open",
 320                         "color": anomaly_color,
 321                         "colorscale": C.COLORSCALE_LAT \
 322                             if ttype == "latency" else C.COLORSCALE_TPUT,
 323                         "showscale": True,
 324                         "line": {
 325                             "width": 2
 326                         },
 327                         "colorbar": {
 328                             "y": 0.5,
 329                             "len": 0.8,
 330                             "title": "Circles Marking Data Classification",
 331                             "titleside": "right",
 332                             "tickmode": "array",
 333                             "tickvals": [0.167, 0.500, 0.833],
 334                             "ticktext": C.TICK_TEXT_LAT \
 335                                 if ttype == "latency" else C.TICK_TEXT_TPUT,
 336                             "ticks": "",
 337                             "ticklen": 0,
 338                             "tickangle": -90,
 339                             "thickness": 10
 340                         }
 341                     }
 342                 )
 343             )
 344
 345         return traces, units
 346
 347
 348     fig_tput = None
 349     fig_lat = None
 350     fig_band = None
 351     y_units = set()
 352     for idx, itm in enumerate(sel):
 353         df = select_trending_data(data, itm)
 354         if df is None or df.empty:
 355             continue
 356
 357         if normalize:
 358             phy = itm["phy"].split("-")
 359             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 360             norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY.get(topo_arch, 1.0)) \
 361                 if topo_arch else 1.0
 362         else:
 363             norm_factor = 1.0
 364
 365         if itm["area"] == "hoststack":
 366             ttype = f"hoststack-{itm['testtype']}"
 367         else:
 368             ttype = itm["testtype"]
 369
 370         traces, units = _generate_trending_traces(
 371             ttype,
 372             itm["id"],
 373             df,
 374             get_color(idx),
 375             norm_factor
 376         )
 377         if traces:
 378             if not fig_tput:
 379                 fig_tput = go.Figure()
 380             fig_tput.add_traces(traces)
 381
 382         if ttype in ("ndr", "pdr", "mrr", "hoststack-cps", "hoststack-rps"):
 383             traces, _ = _generate_trending_traces(
 384                 f"{ttype}-bandwidth",
 385                 itm["id"],
 386                 df,
 387                 get_color(idx),
 388                 norm_factor
 389             )
 390             if traces:
 391                 if not fig_band:
 392                     fig_band = go.Figure()
 393                 fig_band.add_traces(traces)
 394
 395         if ttype in ("pdr", "hoststack-cps", "hoststack-rps"):
 396             traces, _ = _generate_trending_traces(
 397                 "latency" if ttype == "pdr" else "hoststack-latency",
 398                 itm["id"],
 399                 df,
 400                 get_color(idx),
 401                 norm_factor
 402             )
 403             if traces:
 404                 if not fig_lat:
 405                     fig_lat = go.Figure()
 406                 fig_lat.add_traces(traces)
 407
 408         y_units.update(units)
 409
 410     if fig_tput:
 411         fig_layout = layout.get("plot-trending-tput", dict())
 412         fig_layout["yaxis"]["title"] = \
 413             f"Throughput [{'|'.join(sorted(y_units))}]"
 414         fig_tput.update_layout(fig_layout)
 415     if fig_band:
 416         fig_band.update_layout(layout.get("plot-trending-bandwidth", dict()))
 417     if fig_lat:
 418         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 419
 420     return fig_tput, fig_band, fig_lat
 421
 422
 423 def graph_tm_trending(
 424         data: pd.DataFrame,
 425         layout: dict,
 426         all_in_one: bool=False
 427     ) -> list:
 428     """Generates one trending graph per test, each graph includes all selected
 429     metrics.
 430
 431     :param data: Data frame with telemetry data.
 432     :param layout: Layout of plot.ly graph.
 433     :param all_in_one: If True, all telemetry traces are placed in one graph,
 434         otherwise they are split to separate graphs grouped by test ID.
 435     :type data: pandas.DataFrame
 436     :type layout: dict
 437     :type all_in_one: bool
 438     :returns: List of generated graphs together with test names.
 439         list(tuple(plotly.graph_objects.Figure(), str()), tuple(...), ...)
 440     :rtype: list
 441     """
 442
 443     if data.empty:
 444         return list()
 445
 446     def _generate_traces(
 447             data: pd.DataFrame,
 448             test: str,
 449             all_in_one: bool,
 450             color_index: int
 451         ) -> list:
 452         """Generates a trending graph for given test with all metrics.
 453
 454         :param data: Data frame with telemetry data for the given test.
 455         :param test: The name of the test.
 456         :param all_in_one: If True, all telemetry traces are placed in one
 457             graph, otherwise they are split to separate graphs grouped by
 458             test ID.
 459         :param color_index: The index of the test used if all_in_one is True.
 460         :type data: pandas.DataFrame
 461         :type test: str
 462         :type all_in_one: bool
 463         :type color_index: int
 464         :returns: List of traces.
 465         :rtype: list
 466         """
 467         traces = list()
 468         metrics = data.tm_metric.unique().tolist()
 469         for idx, metric in enumerate(metrics):
 470             if "-pdr" in test and "='pdr'" not in metric:
 471                 continue
 472             if "-ndr" in test and "='ndr'" not in metric:
 473                 continue
 474
 475             df = data.loc[(data["tm_metric"] == metric)]
 476             x_axis = df["start_time"].tolist()
 477             y_data = [float(itm) for itm in df["tm_value"].tolist()]
 478             hover = list()
 479             for i, (_, row) in enumerate(df.iterrows()):
 480                 if row["test_type"] == "mrr":
 481                     rate = (
 482                         f"mrr avg [{row[C.UNIT['mrr']]}]: "
 483                         f"{row[C.VALUE['mrr']]:,.0f}<br>"
 484                         f"mrr stdev [{row[C.UNIT['mrr']]}]: "
 485                         f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 486                     )
 487                 elif row["test_type"] == "ndrpdr":
 488                     if "-pdr" in test:
 489                         rate = (
 490                             f"pdr [{row[C.UNIT['pdr']]}]: "
 491                             f"{row[C.VALUE['pdr']]:,.0f}<br>"
 492                         )
 493                     elif "-ndr" in test:
 494                         rate = (
 495                             f"ndr [{row[C.UNIT['ndr']]}]: "
 496                             f"{row[C.VALUE['ndr']]:,.0f}<br>"
 497                         )
 498                     else:
 499                         rate = str()
 500                 else:
 501                     rate = str()
 502                 hover.append(
 503                     f"date: "
 504                     f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 505                     f"value: {y_data[i]:,.2f}<br>"
 506                     f"{rate}"
 507                     f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 508                     f"csit-ref: {row['job']}/{row['build']}<br>"
 509                 )
 510             if any(y_data):
 511                 anomalies, trend_avg, trend_stdev = classify_anomalies(
 512                     {k: v for k, v in zip(x_axis, y_data)}
 513                 )
 514                 hover_trend = list()
 515                 for avg, stdev, (_, row) in \
 516                         zip(trend_avg, trend_stdev, df.iterrows()):
 517                     hover_trend.append(
 518                         f"date: "
 519                         f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 520                         f"trend: {avg:,.2f}<br>"
 521                         f"stdev: {stdev:,.2f}<br>"
 522                         f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 523                         f"csit-ref: {row['job']}/{row['build']}"
 524                     )
 525             else:
 526                 anomalies = None
 527             if all_in_one:
 528                 color = get_color(color_index * len(metrics) + idx)
 529                 metric_name = f"{test}<br>{metric}"
 530             else:
 531                 color = get_color(idx)
 532                 metric_name = metric
 533
 534             traces.append(
 535                 go.Scatter(  # Samples
 536                     x=x_axis,
 537                     y=y_data,
 538                     name=metric_name,
 539                     mode="markers",
 540                     marker={
 541                         "size": 5,
 542                         "color": color,
 543                         "symbol": "circle",
 544                     },
 545                     text=hover,
 546                     hoverinfo="text+name",
 547                     showlegend=True,
 548                     legendgroup=metric_name
 549                 )
 550             )
 551             if anomalies:
 552                 traces.append(
 553                     go.Scatter(  # Trend line
 554                         x=x_axis,
 555                         y=trend_avg,
 556                         name=metric_name,
 557                         mode="lines",
 558                         line={
 559                             "shape": "linear",
 560                             "width": 1,
 561                             "color": color,
 562                         },
 563                         text=hover_trend,
 564                         hoverinfo="text+name",
 565                         showlegend=False,
 566                         legendgroup=metric_name
 567                     )
 568                 )
 569
 570                 anomaly_x = list()
 571                 anomaly_y = list()
 572                 anomaly_color = list()
 573                 hover = list()
 574                 for idx, anomaly in enumerate(anomalies):
 575                     if anomaly in ("regression", "progression"):
 576                         anomaly_x.append(x_axis[idx])
 577                         anomaly_y.append(trend_avg[idx])
 578                         anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 579                         hover_itm = (
 580                             f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}"
 581                             f"<br>trend: {trend_avg[idx]:,.2f}"
 582                             f"<br>classification: {anomaly}"
 583                         )
 584                         hover.append(hover_itm)
 585                 anomaly_color.extend([0.0, 0.5, 1.0])
 586                 traces.append(
 587                     go.Scatter(
 588                         x=anomaly_x,
 589                         y=anomaly_y,
 590                         mode="markers",
 591                         text=hover,
 592                         hoverinfo="text+name",
 593                         showlegend=False,
 594                         legendgroup=metric_name,
 595                         name=metric_name,
 596                         marker={
 597                             "size": 15,
 598                             "symbol": "circle-open",
 599                             "color": anomaly_color,
 600                             "colorscale": C.COLORSCALE_TPUT,
 601                             "showscale": True,
 602                             "line": {
 603                                 "width": 2
 604                             },
 605                             "colorbar": {
 606                                 "y": 0.5,
 607                                 "len": 0.8,
 608                                 "title": "Circles Marking Data Classification",
 609                                 "titleside": "right",
 610                                 "tickmode": "array",
 611                                 "tickvals": [0.167, 0.500, 0.833],
 612                                 "ticktext": C.TICK_TEXT_TPUT,
 613                                 "ticks": "",
 614                                 "ticklen": 0,
 615                                 "tickangle": -90,
 616                                 "thickness": 10
 617                             }
 618                         }
 619                     )
 620                 )
 621
 622         unique_metrics = set()
 623         for itm in metrics:
 624             unique_metrics.add(itm.split("{", 1)[0])
 625         return traces, unique_metrics
 626
 627     tm_trending_graphs = list()
 628     graph_layout = layout.get("plot-trending-telemetry", dict())
 629
 630     if all_in_one:
 631         all_traces = list()
 632
 633     all_metrics = set()
 634     all_tests = list()
 635     for idx, test in enumerate(data.test_name.unique()):
 636         df = data.loc[(data["test_name"] == test)]
 637         traces, metrics = _generate_traces(df, test, all_in_one, idx)
 638         if traces:
 639             all_metrics.update(metrics)
 640             if all_in_one:
 641                 all_traces.extend(traces)
 642                 all_tests.append(test)
 643             else:
 644                 graph = go.Figure()
 645                 graph.add_traces(traces)
 646                 graph.update_layout(graph_layout)
 647                 tm_trending_graphs.append((graph, [test, ], ))
 648
 649     if all_in_one:
 650         graph = go.Figure()
 651         graph.add_traces(all_traces)
 652         graph.update_layout(graph_layout)
 653         tm_trending_graphs.append((graph, all_tests, ))
 654
 655     return tm_trending_graphs, list(all_metrics)