resources/tools/presentation/generator_cpta.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import logging
  18 import csv
  19
  20 from collections import OrderedDict
  21 from datetime import datetime
  22 from copy import deepcopy
  23
  24 import prettytable
  25 import plotly.offline as ploff
  26 import plotly.graph_objs as plgo
  27 import plotly.exceptions as plerr
  28
  29 from pal_utils import archive_input_data, execute_command, classify_anomalies
  30
  31
  32 # Command to build the html format of the report
  33 HTML_BUILDER = u'sphinx-build -v -c conf_cpta -a ' \
  34                u'-b html -E ' \
  35                u'-t html ' \
  36                u'-D version="{date}" ' \
  37                u'{working_dir} ' \
  38                u'{build_dir}/'
  39
  40 # .css file for the html format of the report
  41 THEME_OVERRIDES = u"""/* override table width restrictions */
  42 .wy-nav-content {
  43     max-width: 1200px !important;
  44 }
  45 .rst-content blockquote {
  46     margin-left: 0px;
  47     line-height: 18px;
  48     margin-bottom: 0px;
  49 }
  50 .wy-menu-vertical a {
  51     display: inline-block;
  52     line-height: 18px;
  53     padding: 0 2em;
  54     display: block;
  55     position: relative;
  56     font-size: 90%;
  57     color: #d9d9d9
  58 }
  59 .wy-menu-vertical li.current a {
  60     color: gray;
  61     border-right: solid 1px #c9c9c9;
  62     padding: 0 3em;
  63 }
  64 .wy-menu-vertical li.toctree-l2.current > a {
  65     background: #c9c9c9;
  66     padding: 0 3em;
  67 }
  68 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
  69     display: block;
  70     background: #c9c9c9;
  71     padding: 0 4em;
  72 }
  73 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
  74     display: block;
  75     background: #bdbdbd;
  76     padding: 0 5em;
  77 }
  78 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
  79     color: #404040;
  80     padding: 0 2em;
  81     font-weight: bold;
  82     position: relative;
  83     background: #fcfcfc;
  84     border: none;
  85         border-top-width: medium;
  86         border-bottom-width: medium;
  87         border-top-style: none;
  88         border-bottom-style: none;
  89         border-top-color: currentcolor;
  90         border-bottom-color: currentcolor;
  91     padding-left: 2em -4px;
  92 }
  93 """
  94
  95 COLORS = [
  96     u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink",
  97     u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black",
  98     u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson",
  99     u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod",
 100     u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon",
 101     u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey",
 102     u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink",
 103     u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black",
 104     u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson",
 105     u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod",
 106     u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon",
 107     u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey"
 108 ]
 109
 110
 111 def generate_cpta(spec, data):
 112     """Generate all formats and versions of the Continuous Performance Trending
 113     and Analysis.
 114
 115     :param spec: Specification read from the specification file.
 116     :param data: Full data set.
 117     :type spec: Specification
 118     :type data: InputData
 119     """
 120
 121     logging.info(u"Generating the Continuous Performance Trending and Analysis "
 122                  u"...")
 123
 124     ret_code = _generate_all_charts(spec, data)
 125
 126     cmd = HTML_BUILDER.format(
 127         date=datetime.utcnow().strftime(u'%Y-%m-%d %H:%M UTC'),
 128         working_dir=spec.environment[u'paths'][u'DIR[WORKING,SRC]'],
 129         build_dir=spec.environment[u'paths'][u'DIR[BUILD,HTML]'])
 130     execute_command(cmd)
 131
 132     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE]'], u'w') as \
 133             css_file:
 134         css_file.write(THEME_OVERRIDES)
 135
 136     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE2]'], u'w') as \
 137             css_file:
 138         css_file.write(THEME_OVERRIDES)
 139
 140     if spec.configuration.get(u"archive-inputs", True):
 141         archive_input_data(spec)
 142
 143     logging.info(u"Done.")
 144
 145     return ret_code
 146
 147
 148 def _generate_trending_traces(in_data, job_name, build_info,
 149                               show_trend_line=True, name=u"", color=u""):
 150     """Generate the trending traces:
 151      - samples,
 152      - outliers, regress, progress
 153      - average of normal samples (trending line)
 154
 155     :param in_data: Full data set.
 156     :param job_name: The name of job which generated the data.
 157     :param build_info: Information about the builds.
 158     :param show_trend_line: Show moving median (trending plot).
 159     :param name: Name of the plot
 160     :param color: Name of the color for the plot.
 161     :type in_data: OrderedDict
 162     :type job_name: str
 163     :type build_info: dict
 164     :type show_trend_line: bool
 165     :type name: str
 166     :type color: str
 167     :returns: Generated traces (list) and the evaluated result.
 168     :rtype: tuple(traces, result)
 169     """
 170
 171     data_x = list(in_data.keys())
 172     data_y_pps = list(in_data.values())
 173     data_y_mpps = [float(item) / 1e6 for item in data_y_pps]
 174
 175     hover_text = list()
 176     xaxis = list()
 177     for index, key in enumerate(data_x):
 178         str_key = str(key)
 179         date = build_info[job_name][str_key][0]
 180         hover_str = (u"date: {date}<br>"
 181                      u"value [Mpps]: {value:.3f}<br>"
 182                      u"{sut}-ref: {build}<br>"
 183                      u"csit-ref: mrr-{period}-build-{build_nr}<br>"
 184                      u"testbed: {testbed}")
 185         if u"dpdk" in job_name:
 186             hover_text.append(hover_str.format(
 187                 date=date,
 188                 value=data_y_mpps[index],
 189                 sut=u"dpdk",
 190                 build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0],
 191                 period=u"weekly",
 192                 build_nr=str_key,
 193                 testbed=build_info[job_name][str_key][2]))
 194         elif u"vpp" in job_name:
 195             hover_text.append(hover_str.format(
 196                 date=date,
 197                 value=data_y_mpps[index],
 198                 sut=u"vpp",
 199                 build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0],
 200                 period=u"daily",
 201                 build_nr=str_key,
 202                 testbed=build_info[job_name][str_key][2]))
 203
 204         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 205                               int(date[9:11]), int(date[12:])))
 206
 207     data_pd = OrderedDict()
 208     for key, value in zip(xaxis, data_y_pps):
 209         data_pd[key] = value
 210
 211     anomaly_classification, avgs_pps = classify_anomalies(data_pd)
 212     avgs_mpps = [avg_pps / 1e6 for avg_pps in avgs_pps]
 213
 214     anomalies = OrderedDict()
 215     anomalies_colors = list()
 216     anomalies_avgs = list()
 217     anomaly_color = {
 218         u"regression": 0.0,
 219         u"normal": 0.5,
 220         u"progression": 1.0
 221     }
 222     if anomaly_classification:
 223         for index, (key, value) in enumerate(data_pd.items()):
 224             if anomaly_classification[index] in \
 225                     (u"outlier", u"regression", u"progression"):
 226                 anomalies[key] = value / 1e6
 227                 anomalies_colors.append(
 228                     anomaly_color[anomaly_classification[index]])
 229                 anomalies_avgs.append(avgs_mpps[index])
 230         anomalies_colors.extend([0.0, 0.5, 1.0])
 231
 232     # Create traces
 233
 234     trace_samples = plgo.Scatter(
 235         x=xaxis,
 236         y=data_y_mpps,
 237         mode=u"markers",
 238         line={
 239             u"width": 1
 240         },
 241         showlegend=True,
 242         legendgroup=name,
 243         name=f"{name}",
 244         marker={
 245             u"size": 5,
 246             u"color": color,
 247             u"symbol": u"circle",
 248         },
 249         text=hover_text,
 250         hoverinfo=u"text+name"
 251     )
 252     traces = [trace_samples, ]
 253
 254     if show_trend_line:
 255         trace_trend = plgo.Scatter(
 256             x=xaxis,
 257             y=avgs_mpps,
 258             mode=u"lines",
 259             line={
 260                 u"shape": u"linear",
 261                 u"width": 1,
 262                 u"color": color,
 263             },
 264             showlegend=False,
 265             legendgroup=name,
 266             name=f"{name}",
 267             text=[f"trend [Mpps]: {avg:.3f}" for avg in avgs_mpps],
 268             hoverinfo=u"text+name"
 269         )
 270         traces.append(trace_trend)
 271
 272     trace_anomalies = plgo.Scatter(
 273         x=list(anomalies.keys()),
 274         y=anomalies_avgs,
 275         mode=u"markers",
 276         hoverinfo=u"none",
 277         showlegend=False,
 278         legendgroup=name,
 279         name=f"{name}-anomalies",
 280         marker={
 281             u"size": 15,
 282             u"symbol": u"circle-open",
 283             u"color": anomalies_colors,
 284             u"colorscale": [
 285                 [0.00, u"red"],
 286                 [0.33, u"red"],
 287                 [0.33, u"white"],
 288                 [0.66, u"white"],
 289                 [0.66, u"green"],
 290                 [1.00, u"green"]
 291             ],
 292             u"showscale": True,
 293             u"line": {
 294                 u"width": 2
 295             },
 296             u"colorbar": {
 297                 u"y": 0.5,
 298                 u"len": 0.8,
 299                 u"title": u"Circles Marking Data Classification",
 300                 u"titleside": u"right",
 301                 u"titlefont": {
 302                     u"size": 14
 303                 },
 304                 u"tickmode": u"array",
 305                 u"tickvals": [0.167, 0.500, 0.833],
 306                 u"ticktext": [u"Regression", u"Normal", u"Progression"],
 307                 u"ticks": u"",
 308                 u"ticklen": 0,
 309                 u"tickangle": -90,
 310                 u"thickness": 10
 311             }
 312         }
 313     )
 314     traces.append(trace_anomalies)
 315
 316     if anomaly_classification:
 317         return traces, anomaly_classification[-1]
 318
 319     return traces, None
 320
 321
 322 def _generate_all_charts(spec, input_data):
 323     """Generate all charts specified in the specification file.
 324
 325     :param spec: Specification.
 326     :param input_data: Full data set.
 327     :type spec: Specification
 328     :type input_data: InputData
 329     """
 330
 331     def _generate_chart(graph):
 332         """Generates the chart.
 333
 334         :param graph: The graph to be generated
 335         :type graph: dict
 336         :returns: Dictionary with the job name, csv table with results and
 337             list of tests classification results.
 338         :rtype: dict
 339         """
 340
 341         logging.info(f"  Generating the chart {graph.get(u'title', u'')} ...")
 342
 343         job_name = list(graph[u"data"].keys())[0]
 344
 345         csv_tbl = list()
 346         res = dict()
 347
 348         # Transform the data
 349         logging.info(
 350              f"    Creating the data set for the {graph.get(u'type', u'')} "
 351              f"{graph.get(u'title', u'')}."
 352         )
 353
 354         if graph.get(u"include", None):
 355             data = input_data.filter_tests_by_name(
 356                 graph,
 357                 params=[u"type", u"result", u"tags"],
 358                 continue_on_error=True
 359             )
 360         else:
 361             data = input_data.filter_data(
 362                 graph,
 363                 params=[u"type", u"result", u"tags"],
 364                 continue_on_error=True)
 365
 366         if data is None or data.empty:
 367             logging.error(u"No data.")
 368             return dict()
 369
 370         chart_data = dict()
 371         chart_tags = dict()
 372         for job, job_data in data.items():
 373             if job != job_name:
 374                 continue
 375             for index, bld in job_data.items():
 376                 for test_name, test in bld.items():
 377                     if chart_data.get(test_name, None) is None:
 378                         chart_data[test_name] = OrderedDict()
 379                     try:
 380                         chart_data[test_name][int(index)] = \
 381                             test[u"result"][u"receive-rate"]
 382                         chart_tags[test_name] = test.get(u"tags", None)
 383                     except (KeyError, TypeError):
 384                         pass
 385
 386         # Add items to the csv table:
 387         for tst_name, tst_data in chart_data.items():
 388             tst_lst = list()
 389             for bld in builds_dict[job_name]:
 390                 itm = tst_data.get(int(bld), u'')
 391                 # CSIT-1180: Itm will be list, compute stats.
 392                 tst_lst.append(str(itm))
 393             csv_tbl.append(f"{tst_name}," + u",".join(tst_lst) + u'\n')
 394
 395         # Generate traces:
 396         traces = list()
 397         index = 0
 398         groups = graph.get(u"groups", None)
 399         visibility = list()
 400
 401         if groups:
 402             for group in groups:
 403                 visible = list()
 404                 for tag in group:
 405                     for tst_name, test_data in chart_data.items():
 406                         if not test_data:
 407                             logging.warning(f"No data for the test {tst_name}")
 408                             continue
 409                         if tag not in chart_tags[tst_name]:
 410                             continue
 411                         try:
 412                             trace, rslt = _generate_trending_traces(
 413                                 test_data,
 414                                 job_name=job_name,
 415                                 build_info=build_info,
 416                                 name=u'-'.join(tst_name.split(u'.')[-1].
 417                                                split(u'-')[2:-1]),
 418                                 color=COLORS[index])
 419                         except IndexError:
 420                             logging.error(f"Out of colors: index: "
 421                                           f"{index}, test: {tst_name}")
 422                             index += 1
 423                             continue
 424                         traces.extend(trace)
 425                         visible.extend([True for _ in range(len(trace))])
 426                         res[tst_name] = rslt
 427                         index += 1
 428                         break
 429                 visibility.append(visible)
 430         else:
 431             for tst_name, test_data in chart_data.items():
 432                 if not test_data:
 433                     logging.warning(f"No data for the test {tst_name}")
 434                     continue
 435                 try:
 436                     trace, rslt = _generate_trending_traces(
 437                         test_data,
 438                         job_name=job_name,
 439                         build_info=build_info,
 440                         name=u'-'.join(
 441                             tst_name.split(u'.')[-1].split(u'-')[2:-1]),
 442                         color=COLORS[index])
 443                 except IndexError:
 444                     logging.error(
 445                         f"Out of colors: index: {index}, test: {tst_name}"
 446                     )
 447                     index += 1
 448                     continue
 449                 traces.extend(trace)
 450                 res[tst_name] = rslt
 451                 index += 1
 452
 453         if traces:
 454             # Generate the chart:
 455             try:
 456                 layout = deepcopy(graph[u"layout"])
 457             except KeyError as err:
 458                 logging.error(u"Finished with error: No layout defined")
 459                 logging.error(repr(err))
 460                 return dict()
 461             if groups:
 462                 show = list()
 463                 for i in range(len(visibility)):
 464                     visible = list()
 465                     for vis_idx, _ in enumerate(visibility):
 466                         for _ in range(len(visibility[vis_idx])):
 467                             visible.append(i == vis_idx)
 468                     show.append(visible)
 469
 470                 buttons = list()
 471                 buttons.append(dict(
 472                     label=u"All",
 473                     method=u"update",
 474                     args=[{u"visible": [True for _ in range(len(show[0]))]}, ]
 475                 ))
 476                 for i in range(len(groups)):
 477                     try:
 478                         label = graph[u"group-names"][i]
 479                     except (IndexError, KeyError):
 480                         label = f"Group {i + 1}"
 481                     buttons.append(dict(
 482                         label=label,
 483                         method=u"update",
 484                         args=[{u"visible": show[i]}, ]
 485                     ))
 486
 487                 layout[u"updatemenus"] = list([
 488                     dict(
 489                         active=0,
 490                         type=u"dropdown",
 491                         direction=u"down",
 492                         xanchor=u"left",
 493                         yanchor=u"bottom",
 494                         x=-0.12,
 495                         y=1.0,
 496                         buttons=buttons
 497                     )
 498                 ])
 499
 500             name_file = (
 501                 f"{spec.cpta[u'output-file']}/{graph[u'output-file-name']}"
 502                 f"{spec.cpta[u'output-file-type']}")
 503
 504             logging.info(f"    Writing the file {name_file} ...")
 505             plpl = plgo.Figure(data=traces, layout=layout)
 506             try:
 507                 ploff.plot(plpl, show_link=False, auto_open=False,
 508                            filename=name_file)
 509             except plerr.PlotlyEmptyDataError:
 510                 logging.warning(u"No data for the plot. Skipped.")
 511
 512         return {u"job_name": job_name, u"csv_table": csv_tbl, u"results": res}
 513
 514     builds_dict = dict()
 515     for job in spec.input[u"builds"].keys():
 516         if builds_dict.get(job, None) is None:
 517             builds_dict[job] = list()
 518         for build in spec.input[u"builds"][job]:
 519             status = build[u"status"]
 520             if status not in (u"failed", u"not found", u"removed", None):
 521                 builds_dict[job].append(str(build[u"build"]))
 522
 523     # Create "build ID": "date" dict:
 524     build_info = dict()
 525     tb_tbl = spec.environment.get(u"testbeds", None)
 526     for job_name, job_data in builds_dict.items():
 527         if build_info.get(job_name, None) is None:
 528             build_info[job_name] = OrderedDict()
 529         for build in job_data:
 530             testbed = u""
 531             tb_ip = input_data.metadata(job_name, build).get(u"testbed", u"")
 532             if tb_ip and tb_tbl:
 533                 testbed = tb_tbl.get(tb_ip, u"")
 534             build_info[job_name][build] = (
 535                 input_data.metadata(job_name, build).get(u"generated", u""),
 536                 input_data.metadata(job_name, build).get(u"version", u""),
 537                 testbed
 538             )
 539
 540     anomaly_classifications = dict()
 541
 542     # Create the table header:
 543     csv_tables = dict()
 544     for job_name in builds_dict:
 545         if csv_tables.get(job_name, None) is None:
 546             csv_tables[job_name] = list()
 547         header = f"Build Number:,{u','.join(builds_dict[job_name])}\n"
 548         csv_tables[job_name].append(header)
 549         build_dates = [x[0] for x in build_info[job_name].values()]
 550         header = f"Build Date:,{u','.join(build_dates)}\n"
 551         csv_tables[job_name].append(header)
 552         versions = [x[1] for x in build_info[job_name].values()]
 553         header = f"Version:,{u','.join(versions)}\n"
 554         csv_tables[job_name].append(header)
 555
 556     for chart in spec.cpta[u"plots"]:
 557         result = _generate_chart(chart)
 558         if not result:
 559             continue
 560
 561         csv_tables[result[u"job_name"]].extend(result[u"csv_table"])
 562
 563         if anomaly_classifications.get(result[u"job_name"], None) is None:
 564             anomaly_classifications[result[u"job_name"]] = dict()
 565         anomaly_classifications[result[u"job_name"]].update(result[u"results"])
 566
 567     # Write the tables:
 568     for job_name, csv_table in csv_tables.items():
 569         file_name = f"{spec.cpta[u'output-file']}/{job_name}-trending"
 570         with open(f"{file_name}.csv", u"wt") as file_handler:
 571             file_handler.writelines(csv_table)
 572
 573         txt_table = None
 574         with open(f"{file_name}.csv", u"rt") as csv_file:
 575             csv_content = csv.reader(csv_file, delimiter=u',', quotechar=u'"')
 576             line_nr = 0
 577             for row in csv_content:
 578                 if txt_table is None:
 579                     txt_table = prettytable.PrettyTable(row)
 580                 else:
 581                     if line_nr > 1:
 582                         for idx, item in enumerate(row):
 583                             try:
 584                                 row[idx] = str(round(float(item) / 1000000, 2))
 585                             except ValueError:
 586                                 pass
 587                     try:
 588                         txt_table.add_row(row)
 589                     # PrettyTable raises Exception
 590                     except Exception as err:
 591                         logging.warning(
 592                             f"Error occurred while generating TXT table:\n{err}"
 593                         )
 594                 line_nr += 1
 595             txt_table.align[u"Build Number:"] = u"l"
 596         with open(f"{file_name}.txt", u"wt") as txt_file:
 597             txt_file.write(str(txt_table))
 598
 599     # Evaluate result:
 600     if anomaly_classifications:
 601         result = u"PASS"
 602         for job_name, job_data in anomaly_classifications.items():
 603             file_name = \
 604                 f"{spec.cpta[u'output-file']}/regressions-{job_name}.txt"
 605             with open(file_name, u'w') as txt_file:
 606                 for test_name, classification in job_data.items():
 607                     if classification == u"regression":
 608                         txt_file.write(test_name + u'\n')
 609                     if classification in (u"regression", u"outlier"):
 610                         result = u"FAIL"
 611             file_name = \
 612                 f"{spec.cpta[u'output-file']}/progressions-{job_name}.txt"
 613             with open(file_name, u'w') as txt_file:
 614                 for test_name, classification in job_data.items():
 615                     if classification == u"progression":
 616                         txt_file.write(test_name + u'\n')
 617     else:
 618         result = u"FAIL"
 619
 620     logging.info(f"Partial results: {anomaly_classifications}")
 621     logging.info(f"Result: {result}")
 622
 623     return result