csit.infra.dash/app/cdash/trending/graphs.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Implementation of graphs for trending data.
  15 """
  16
  17 import logging
  18 import plotly.graph_objects as go
  19 import pandas as pd
  20
  21 from ..utils.constants import Constants as C
  22 from ..utils.utils import get_color, get_hdrh_latencies
  23 from ..utils.anomalies import classify_anomalies
  24
  25
  26 def select_trending_data(data: pd.DataFrame, itm: dict) -> pd.DataFrame:
  27     """Select the data for graphs from the provided data frame.
  28
  29     :param data: Data frame with data for graphs.
  30     :param itm: Item (in this case job name) which data will be selected from
  31         the input data frame.
  32     :type data: pandas.DataFrame
  33     :type itm: dict
  34     :returns: A data frame with selected data.
  35     :rtype: pandas.DataFrame
  36     """
  37
  38     phy = itm["phy"].split("-")
  39     if len(phy) == 4:
  40         topo, arch, nic, drv = phy
  41         if drv == "dpdk":
  42             drv = ""
  43         else:
  44             drv += "-"
  45             drv = drv.replace("_", "-")
  46     else:
  47         return None
  48
  49     if itm["testtype"] in ("ndr", "pdr"):
  50         test_type = "ndrpdr"
  51     elif itm["testtype"] == "mrr":
  52         test_type = "mrr"
  53     elif itm["area"] == "hoststack":
  54         test_type = "hoststack"
  55     df = data.loc[(
  56         (data["test_type"] == test_type) &
  57         (data["passed"] == True)
  58     )]
  59     df = df[df.job.str.endswith(f"{topo}-{arch}")]
  60     core = str() if itm["dut"] == "trex" else f"{itm['core']}"
  61     ttype = "ndrpdr" if itm["testtype"] in ("ndr", "pdr") else itm["testtype"]
  62     df = df[df.test_id.str.contains(
  63         f"^.*[.|-]{nic}.*{itm['framesize']}-{core}-{drv}{itm['test']}-{ttype}$",
  64         regex=True
  65     )].sort_values(by="start_time", ignore_index=True)
  66
  67     return df
  68
  69
  70 def graph_trending(
  71         data: pd.DataFrame,
  72         sel: dict,
  73         layout: dict,
  74         normalize: bool
  75     ) -> tuple:
  76     """Generate the trending graph(s) - MRR, NDR, PDR and for PDR also Latences
  77     (result_latency_forward_pdr_50_avg).
  78
  79     :param data: Data frame with test results.
  80     :param sel: Selected tests.
  81     :param layout: Layout of plot.ly graph.
  82     :param normalize: If True, the data is normalized to CPU frquency
  83         Constants.NORM_FREQUENCY.
  84     :type data: pandas.DataFrame
  85     :type sel: dict
  86     :type layout: dict
  87     :type normalize: bool
  88     :returns: Trending graph(s)
  89     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
  90     """
  91
  92     if not sel:
  93         return None, None
  94
  95
  96     def _generate_trending_traces(
  97             ttype: str,
  98             name: str,
  99             df: pd.DataFrame,
 100             color: str,
 101             norm_factor: float
 102         ) -> list:
 103         """Generate the trending traces for the trending graph.
 104
 105         :param ttype: Test type (MRR, NDR, PDR).
 106         :param name: The test name to be displayed as the graph title.
 107         :param df: Data frame with test data.
 108         :param color: The color of the trace (samples and trend line).
 109         :param norm_factor: The factor used for normalization of the results to
 110             CPU frequency set to Constants.NORM_FREQUENCY.
 111         :type ttype: str
 112         :type name: str
 113         :type df: pandas.DataFrame
 114         :type color: str
 115         :type norm_factor: float
 116         :returns: Traces (samples, trending line, anomalies)
 117         :rtype: list
 118         """
 119
 120         df = df.dropna(subset=[C.VALUE[ttype], ])
 121         if df.empty:
 122             return list(), list()
 123
 124         x_axis = df["start_time"].tolist()
 125         if ttype == "latency":
 126             y_data = [(v / norm_factor) for v in df[C.VALUE[ttype]].tolist()]
 127         else:
 128             y_data = [(v * norm_factor) for v in df[C.VALUE[ttype]].tolist()]
 129         units = df[C.UNIT[ttype]].unique().tolist()
 130
 131         try:
 132             anomalies, trend_avg, trend_stdev = classify_anomalies(
 133                 {k: v for k, v in zip(x_axis, y_data)}
 134             )
 135         except ValueError as err:
 136             logging.error(err)
 137             return list(), list()
 138
 139         hover = list()
 140         customdata = list()
 141         customdata_samples = list()
 142         name_lst = name.split("-")
 143         for idx, (_, row) in enumerate(df.iterrows()):
 144             h_tput, h_band, h_lat = str(), str(), str()
 145             if ttype in ("mrr", "mrr-bandwidth"):
 146                 h_tput = (
 147                     f"tput avg [{row['result_receive_rate_rate_unit']}]: "
 148                     f"{row['result_receive_rate_rate_avg']:,.0f}<br>"
 149                     f"tput stdev [{row['result_receive_rate_rate_unit']}]: "
 150                     f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 151                 )
 152                 if pd.notna(row["result_receive_rate_bandwidth_avg"]):
 153                     h_band = (
 154                         f"bandwidth avg "
 155                         f"[{row['result_receive_rate_bandwidth_unit']}]: "
 156                         f"{row['result_receive_rate_bandwidth_avg']:,.0f}<br>"
 157                         f"bandwidth stdev "
 158                         f"[{row['result_receive_rate_bandwidth_unit']}]: "
 159                         f"{row['result_receive_rate_bandwidth_stdev']:,.0f}<br>"
 160                     )
 161             elif ttype in ("ndr", "ndr-bandwidth"):
 162                 h_tput = (
 163                     f"tput [{row['result_ndr_lower_rate_unit']}]: "
 164                     f"{row['result_ndr_lower_rate_value']:,.0f}<br>"
 165                 )
 166                 if pd.notna(row["result_ndr_lower_bandwidth_value"]):
 167                     h_band = (
 168                         f"bandwidth [{row['result_ndr_lower_bandwidth_unit']}]:"
 169                         f" {row['result_ndr_lower_bandwidth_value']:,.0f}<br>"
 170                     )
 171             elif ttype in ("pdr", "pdr-bandwidth", "latency"):
 172                 h_tput = (
 173                     f"tput [{row['result_pdr_lower_rate_unit']}]: "
 174                     f"{row['result_pdr_lower_rate_value']:,.0f}<br>"
 175                 )
 176                 if pd.notna(row["result_pdr_lower_bandwidth_value"]):
 177                     h_band = (
 178                         f"bandwidth [{row['result_pdr_lower_bandwidth_unit']}]:"
 179                         f" {row['result_pdr_lower_bandwidth_value']:,.0f}<br>"
 180                     )
 181                 if pd.notna(row["result_latency_forward_pdr_50_avg"]):
 182                     h_lat = (
 183                         f"latency "
 184                         f"[{row['result_latency_forward_pdr_50_unit']}]: "
 185                         f"{row['result_latency_forward_pdr_50_avg']:,.0f}<br>"
 186                     )
 187             elif ttype in ("hoststack-cps", "hoststack-rps",
 188                            "hoststack-cps-bandwidth",
 189                            "hoststack-rps-bandwidth", "hoststack-latency"):
 190                 h_tput = (
 191                     f"tput [{row['result_rate_unit']}]: "
 192                     f"{row['result_rate_value']:,.0f}<br>"
 193                 )
 194                 h_band = (
 195                     f"bandwidth [{row['result_bandwidth_unit']}]: "
 196                     f"{row['result_bandwidth_value']:,.0f}<br>"
 197                 )
 198                 h_lat = (
 199                     f"latency [{row['result_latency_unit']}]: "
 200                     f"{row['result_latency_value']:,.0f}<br>"
 201                 )
 202             elif ttype in ("hoststack-bps", ):
 203                 h_band = (
 204                     f"bandwidth [{row['result_bandwidth_unit']}]: "
 205                     f"{row['result_bandwidth_value']:,.0f}<br>"
 206                 )
 207             hover_itm = (
 208                 f"dut: {name_lst[0]}<br>"
 209                 f"infra: {'-'.join(name_lst[1:5])}<br>"
 210                 f"test: {'-'.join(name_lst[5:])}<br>"
 211                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 212                 f"{h_tput}{h_band}{h_lat}"
 213                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 214                 f"csit-ref: {row['job']}/{row['build']}<br>"
 215                 f"hosts: {', '.join(row['hosts'])}"
 216             )
 217             hover.append(hover_itm)
 218             if ttype == "latency":
 219                 customdata_samples.append(get_hdrh_latencies(row, name))
 220                 customdata.append({"name": name})
 221             else:
 222                 customdata_samples.append(
 223                     {"name": name, "show_telemetry": True}
 224                 )
 225                 customdata.append({"name": name})
 226
 227         hover_trend = list()
 228         for avg, stdev, (_, row) in zip(trend_avg, trend_stdev, df.iterrows()):
 229             hover_itm = (
 230                 f"dut: {name_lst[0]}<br>"
 231                 f"infra: {'-'.join(name_lst[1:5])}<br>"
 232                 f"test: {'-'.join(name_lst[5:])}<br>"
 233                 f"date: {row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 234                 f"trend [{row[C.UNIT[ttype]]}]: {avg:,.0f}<br>"
 235                 f"stdev [{row[C.UNIT[ttype]]}]: {stdev:,.0f}<br>"
 236                 f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 237                 f"csit-ref: {row['job']}/{row['build']}<br>"
 238                 f"hosts: {', '.join(row['hosts'])}"
 239             )
 240             if ttype == "latency":
 241                 hover_itm = hover_itm.replace("[pps]", "[us]")
 242             hover_trend.append(hover_itm)
 243
 244         traces = [
 245             go.Scatter(  # Samples
 246                 x=x_axis,
 247                 y=y_data,
 248                 name=name,
 249                 mode="markers",
 250                 marker={
 251                     "size": 5,
 252                     "color": color,
 253                     "symbol": "circle",
 254                 },
 255                 text=hover,
 256                 hoverinfo="text",
 257                 showlegend=True,
 258                 legendgroup=name,
 259                 customdata=customdata_samples
 260             ),
 261             go.Scatter(  # Trend line
 262                 x=x_axis,
 263                 y=trend_avg,
 264                 name=name,
 265                 mode="lines",
 266                 line={
 267                     "shape": "linear",
 268                     "width": 1,
 269                     "color": color,
 270                 },
 271                 text=hover_trend,
 272                 hoverinfo="text",
 273                 showlegend=False,
 274                 legendgroup=name,
 275                 customdata=customdata
 276             )
 277         ]
 278
 279         if anomalies:
 280             anomaly_x = list()
 281             anomaly_y = list()
 282             anomaly_color = list()
 283             hover = list()
 284             for idx, anomaly in enumerate(anomalies):
 285                 if anomaly in ("regression", "progression"):
 286                     anomaly_x.append(x_axis[idx])
 287                     anomaly_y.append(trend_avg[idx])
 288                     anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 289                     hover_itm = (
 290                         f"dut: {name_lst[0]}<br>"
 291                         f"infra: {'-'.join(name_lst[1:5])}<br>"
 292                         f"test: {'-'.join(name_lst[5:])}<br>"
 293                         f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 294                         f"trend [pps]: {trend_avg[idx]:,.0f}<br>"
 295                         f"classification: {anomaly}"
 296                     )
 297                     if ttype == "latency":
 298                         hover_itm = hover_itm.replace("[pps]", "[us]")
 299                     hover.append(hover_itm)
 300             anomaly_color.extend([0.0, 0.5, 1.0])
 301             traces.append(
 302                 go.Scatter(
 303                     x=anomaly_x,
 304                     y=anomaly_y,
 305                     mode="markers",
 306                     text=hover,
 307                     hoverinfo="text",
 308                     showlegend=False,
 309                     legendgroup=name,
 310                     name=name,
 311                     customdata=customdata,
 312                     marker={
 313                         "size": 15,
 314                         "symbol": "circle-open",
 315                         "color": anomaly_color,
 316                         "colorscale": C.COLORSCALE_LAT \
 317                             if ttype == "latency" else C.COLORSCALE_TPUT,
 318                         "showscale": True,
 319                         "line": {
 320                             "width": 2
 321                         },
 322                         "colorbar": {
 323                             "y": 0.5,
 324                             "len": 0.8,
 325                             "title": "Circles Marking Data Classification",
 326                             "titleside": "right",
 327                             "tickmode": "array",
 328                             "tickvals": [0.167, 0.500, 0.833],
 329                             "ticktext": C.TICK_TEXT_LAT \
 330                                 if ttype == "latency" else C.TICK_TEXT_TPUT,
 331                             "ticks": "",
 332                             "ticklen": 0,
 333                             "tickangle": -90,
 334                             "thickness": 10
 335                         }
 336                     }
 337                 )
 338             )
 339
 340         return traces, units
 341
 342
 343     fig_tput = None
 344     fig_lat = None
 345     fig_band = None
 346     y_units = set()
 347     for idx, itm in enumerate(sel):
 348         df = select_trending_data(data, itm)
 349         if df is None or df.empty:
 350             continue
 351
 352         if normalize:
 353             phy = itm["phy"].split("-")
 354             topo_arch = f"{phy[0]}-{phy[1]}" if len(phy) == 4 else str()
 355             norm_factor = (C.NORM_FREQUENCY / C.FREQUENCY[topo_arch]) \
 356                 if topo_arch else 1.0
 357         else:
 358             norm_factor = 1.0
 359
 360         if itm["area"] == "hoststack":
 361             ttype = f"hoststack-{itm['testtype']}"
 362         else:
 363             ttype = itm["testtype"]
 364
 365         traces, units = _generate_trending_traces(
 366             ttype,
 367             itm["id"],
 368             df,
 369             get_color(idx),
 370             norm_factor
 371         )
 372         if traces:
 373             if not fig_tput:
 374                 fig_tput = go.Figure()
 375             fig_tput.add_traces(traces)
 376
 377         if ttype in ("ndr", "pdr", "mrr", "hoststack-cps", "hoststack-rps"):
 378             traces, _ = _generate_trending_traces(
 379                 f"{ttype}-bandwidth",
 380                 itm["id"],
 381                 df,
 382                 get_color(idx),
 383                 norm_factor
 384             )
 385             if traces:
 386                 if not fig_band:
 387                     fig_band = go.Figure()
 388                 fig_band.add_traces(traces)
 389
 390         if ttype in ("pdr", "hoststack-cps", "hoststack-rps"):
 391             traces, _ = _generate_trending_traces(
 392                 "latency" if ttype == "pdr" else "hoststack-latency",
 393                 itm["id"],
 394                 df,
 395                 get_color(idx),
 396                 norm_factor
 397             )
 398             if traces:
 399                 if not fig_lat:
 400                     fig_lat = go.Figure()
 401                 fig_lat.add_traces(traces)
 402
 403         y_units.update(units)
 404
 405     if fig_tput:
 406         fig_layout = layout.get("plot-trending-tput", dict())
 407         fig_layout["yaxis"]["title"] = \
 408             f"Throughput [{'|'.join(sorted(y_units))}]"
 409         fig_tput.update_layout(fig_layout)
 410     if fig_band:
 411         fig_band.update_layout(layout.get("plot-trending-bandwidth", dict()))
 412     if fig_lat:
 413         fig_lat.update_layout(layout.get("plot-trending-lat", dict()))
 414
 415     return fig_tput, fig_band, fig_lat
 416
 417
 418 def graph_tm_trending(
 419         data: pd.DataFrame,
 420         layout: dict,
 421         all_in_one: bool=False
 422     ) -> list:
 423     """Generates one trending graph per test, each graph includes all selected
 424     metrics.
 425
 426     :param data: Data frame with telemetry data.
 427     :param layout: Layout of plot.ly graph.
 428     :param all_in_one: If True, all telemetry traces are placed in one graph,
 429         otherwise they are split to separate graphs grouped by test ID.
 430     :type data: pandas.DataFrame
 431     :type layout: dict
 432     :type all_in_one: bool
 433     :returns: List of generated graphs together with test names.
 434         list(tuple(plotly.graph_objects.Figure(), str()), tuple(...), ...)
 435     :rtype: list
 436     """
 437
 438     if data.empty:
 439         return list()
 440
 441     def _generate_traces(
 442             data: pd.DataFrame,
 443             test: str,
 444             all_in_one: bool,
 445             color_index: int
 446         ) -> list:
 447         """Generates a trending graph for given test with all metrics.
 448
 449         :param data: Data frame with telemetry data for the given test.
 450         :param test: The name of the test.
 451         :param all_in_one: If True, all telemetry traces are placed in one
 452             graph, otherwise they are split to separate graphs grouped by
 453             test ID.
 454         :param color_index: The index of the test used if all_in_one is True.
 455         :type data: pandas.DataFrame
 456         :type test: str
 457         :type all_in_one: bool
 458         :type color_index: int
 459         :returns: List of traces.
 460         :rtype: list
 461         """
 462         traces = list()
 463         metrics = data.tm_metric.unique().tolist()
 464         for idx, metric in enumerate(metrics):
 465             if "-pdr" in test and "='pdr'" not in metric:
 466                 continue
 467             if "-ndr" in test and "='ndr'" not in metric:
 468                 continue
 469
 470             df = data.loc[(data["tm_metric"] == metric)]
 471             x_axis = df["start_time"].tolist()
 472             y_data = [float(itm) for itm in df["tm_value"].tolist()]
 473             hover = list()
 474             for i, (_, row) in enumerate(df.iterrows()):
 475                 if row["test_type"] == "mrr":
 476                     rate = (
 477                         f"mrr avg [{row[C.UNIT['mrr']]}]: "
 478                         f"{row[C.VALUE['mrr']]:,.0f}<br>"
 479                         f"mrr stdev [{row[C.UNIT['mrr']]}]: "
 480                         f"{row['result_receive_rate_rate_stdev']:,.0f}<br>"
 481                     )
 482                 elif row["test_type"] == "ndrpdr":
 483                     if "-pdr" in test:
 484                         rate = (
 485                             f"pdr [{row[C.UNIT['pdr']]}]: "
 486                             f"{row[C.VALUE['pdr']]:,.0f}<br>"
 487                         )
 488                     elif "-ndr" in test:
 489                         rate = (
 490                             f"ndr [{row[C.UNIT['ndr']]}]: "
 491                             f"{row[C.VALUE['ndr']]:,.0f}<br>"
 492                         )
 493                     else:
 494                         rate = str()
 495                 else:
 496                     rate = str()
 497                 hover.append(
 498                     f"date: "
 499                     f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 500                     f"value: {y_data[i]:,.2f}<br>"
 501                     f"{rate}"
 502                     f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 503                     f"csit-ref: {row['job']}/{row['build']}<br>"
 504                 )
 505             if any(y_data):
 506                 anomalies, trend_avg, trend_stdev = classify_anomalies(
 507                     {k: v for k, v in zip(x_axis, y_data)}
 508                 )
 509                 hover_trend = list()
 510                 for avg, stdev, (_, row) in \
 511                         zip(trend_avg, trend_stdev, df.iterrows()):
 512                     hover_trend.append(
 513                         f"date: "
 514                         f"{row['start_time'].strftime('%Y-%m-%d %H:%M:%S')}<br>"
 515                         f"trend: {avg:,.2f}<br>"
 516                         f"stdev: {stdev:,.2f}<br>"
 517                         f"{row['dut_type']}-ref: {row['dut_version']}<br>"
 518                         f"csit-ref: {row['job']}/{row['build']}"
 519                     )
 520             else:
 521                 anomalies = None
 522             if all_in_one:
 523                 color = get_color(color_index * len(metrics) + idx)
 524                 metric_name = f"{test}<br>{metric}"
 525             else:
 526                 color = get_color(idx)
 527                 metric_name = metric
 528
 529             traces.append(
 530                 go.Scatter(  # Samples
 531                     x=x_axis,
 532                     y=y_data,
 533                     name=metric_name,
 534                     mode="markers",
 535                     marker={
 536                         "size": 5,
 537                         "color": color,
 538                         "symbol": "circle",
 539                     },
 540                     text=hover,
 541                     hoverinfo="text+name",
 542                     showlegend=True,
 543                     legendgroup=metric_name
 544                 )
 545             )
 546             if anomalies:
 547                 traces.append(
 548                     go.Scatter(  # Trend line
 549                         x=x_axis,
 550                         y=trend_avg,
 551                         name=metric_name,
 552                         mode="lines",
 553                         line={
 554                             "shape": "linear",
 555                             "width": 1,
 556                             "color": color,
 557                         },
 558                         text=hover_trend,
 559                         hoverinfo="text+name",
 560                         showlegend=False,
 561                         legendgroup=metric_name
 562                     )
 563                 )
 564
 565                 anomaly_x = list()
 566                 anomaly_y = list()
 567                 anomaly_color = list()
 568                 hover = list()
 569                 for idx, anomaly in enumerate(anomalies):
 570                     if anomaly in ("regression", "progression"):
 571                         anomaly_x.append(x_axis[idx])
 572                         anomaly_y.append(trend_avg[idx])
 573                         anomaly_color.append(C.ANOMALY_COLOR[anomaly])
 574                         hover_itm = (
 575                             f"date: {x_axis[idx].strftime('%Y-%m-%d %H:%M:%S')}"
 576                             f"<br>trend: {trend_avg[idx]:,.2f}"
 577                             f"<br>classification: {anomaly}"
 578                         )
 579                         hover.append(hover_itm)
 580                 anomaly_color.extend([0.0, 0.5, 1.0])
 581                 traces.append(
 582                     go.Scatter(
 583                         x=anomaly_x,
 584                         y=anomaly_y,
 585                         mode="markers",
 586                         text=hover,
 587                         hoverinfo="text+name",
 588                         showlegend=False,
 589                         legendgroup=metric_name,
 590                         name=metric_name,
 591                         marker={
 592                             "size": 15,
 593                             "symbol": "circle-open",
 594                             "color": anomaly_color,
 595                             "colorscale": C.COLORSCALE_TPUT,
 596                             "showscale": True,
 597                             "line": {
 598                                 "width": 2
 599                             },
 600                             "colorbar": {
 601                                 "y": 0.5,
 602                                 "len": 0.8,
 603                                 "title": "Circles Marking Data Classification",
 604                                 "titleside": "right",
 605                                 "tickmode": "array",
 606                                 "tickvals": [0.167, 0.500, 0.833],
 607                                 "ticktext": C.TICK_TEXT_TPUT,
 608                                 "ticks": "",
 609                                 "ticklen": 0,
 610                                 "tickangle": -90,
 611                                 "thickness": 10
 612                             }
 613                         }
 614                     )
 615                 )
 616
 617         unique_metrics = set()
 618         for itm in metrics:
 619             unique_metrics.add(itm.split("{", 1)[0])
 620         return traces, unique_metrics
 621
 622     tm_trending_graphs = list()
 623     graph_layout = layout.get("plot-trending-telemetry", dict())
 624
 625     if all_in_one:
 626         all_traces = list()
 627
 628     all_metrics = set()
 629     all_tests = list()
 630     for idx, test in enumerate(data.test_name.unique()):
 631         df = data.loc[(data["test_name"] == test)]
 632         traces, metrics = _generate_traces(df, test, all_in_one, idx)
 633         if traces:
 634             all_metrics.update(metrics)
 635             if all_in_one:
 636                 all_traces.extend(traces)
 637                 all_tests.append(test)
 638             else:
 639                 graph = go.Figure()
 640                 graph.add_traces(traces)
 641                 graph.update_layout(graph_layout)
 642                 tm_trending_graphs.append((graph, [test, ], ))
 643
 644     if all_in_one:
 645         graph = go.Figure()
 646         graph.add_traces(all_traces)
 647         graph.update_layout(graph_layout)
 648         tm_trending_graphs.append((graph, all_tests, ))
 649
 650     return tm_trending_graphs, list(all_metrics)