resources/tools/presentation/generator_cpta.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import logging
  18 import csv
  19
  20 from collections import OrderedDict
  21 from datetime import datetime
  22 from copy import deepcopy
  23
  24 import prettytable
  25 import plotly.offline as ploff
  26 import plotly.graph_objs as plgo
  27 import plotly.exceptions as plerr
  28
  29 from pal_utils import archive_input_data, execute_command, classify_anomalies
  30
  31
  32 # Command to build the html format of the report
  33 HTML_BUILDER = u'sphinx-build -v -c conf_cpta -a ' \
  34                u'-b html -E ' \
  35                u'-t html ' \
  36                u'-D version="{date}" ' \
  37                u'{working_dir} ' \
  38                u'{build_dir}/'
  39
  40 # .css file for the html format of the report
  41 THEME_OVERRIDES = u"""/* override table width restrictions */
  42 .wy-nav-content {
  43     max-width: 1200px !important;
  44 }
  45 .rst-content blockquote {
  46     margin-left: 0px;
  47     line-height: 18px;
  48     margin-bottom: 0px;
  49 }
  50 .wy-menu-vertical a {
  51     display: inline-block;
  52     line-height: 18px;
  53     padding: 0 2em;
  54     display: block;
  55     position: relative;
  56     font-size: 90%;
  57     color: #d9d9d9
  58 }
  59 .wy-menu-vertical li.current a {
  60     color: gray;
  61     border-right: solid 1px #c9c9c9;
  62     padding: 0 3em;
  63 }
  64 .wy-menu-vertical li.toctree-l2.current > a {
  65     background: #c9c9c9;
  66     padding: 0 3em;
  67 }
  68 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
  69     display: block;
  70     background: #c9c9c9;
  71     padding: 0 4em;
  72 }
  73 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
  74     display: block;
  75     background: #bdbdbd;
  76     padding: 0 5em;
  77 }
  78 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
  79     color: #404040;
  80     padding: 0 2em;
  81     font-weight: bold;
  82     position: relative;
  83     background: #fcfcfc;
  84     border: none;
  85         border-top-width: medium;
  86         border-bottom-width: medium;
  87         border-top-style: none;
  88         border-bottom-style: none;
  89         border-top-color: currentcolor;
  90         border-bottom-color: currentcolor;
  91     padding-left: 2em -4px;
  92 }
  93 """
  94
  95 COLORS = [
  96     u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink",
  97     u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black",
  98     u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson",
  99     u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod",
 100     u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon",
 101     u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey",
 102     u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink",
 103     u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black",
 104     u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson",
 105     u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod",
 106     u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon",
 107     u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey"
 108 ]
 109
 110
 111 def generate_cpta(spec, data):
 112     """Generate all formats and versions of the Continuous Performance Trending
 113     and Analysis.
 114
 115     :param spec: Specification read from the specification file.
 116     :param data: Full data set.
 117     :type spec: Specification
 118     :type data: InputData
 119     """
 120
 121     logging.info(u"Generating the Continuous Performance Trending and Analysis "
 122                  u"...")
 123
 124     ret_code = _generate_all_charts(spec, data)
 125
 126     cmd = HTML_BUILDER.format(
 127         date=datetime.utcnow().strftime(u'%Y-%m-%d %H:%M UTC'),
 128         working_dir=spec.environment[u'paths'][u'DIR[WORKING,SRC]'],
 129         build_dir=spec.environment[u'paths'][u'DIR[BUILD,HTML]'])
 130     execute_command(cmd)
 131
 132     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE]'], u'w') as \
 133             css_file:
 134         css_file.write(THEME_OVERRIDES)
 135
 136     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE2]'], u'w') as \
 137             css_file:
 138         css_file.write(THEME_OVERRIDES)
 139
 140     if spec.configuration.get(u"archive-inputs", True):
 141         archive_input_data(spec)
 142
 143     logging.info(u"Done.")
 144
 145     return ret_code
 146
 147
 148 def _generate_trending_traces(in_data, job_name, build_info,
 149                               show_trend_line=True, name=u"", color=u""):
 150     """Generate the trending traces:
 151      - samples,
 152      - outliers, regress, progress
 153      - average of normal samples (trending line)
 154
 155     :param in_data: Full data set.
 156     :param job_name: The name of job which generated the data.
 157     :param build_info: Information about the builds.
 158     :param show_trend_line: Show moving median (trending plot).
 159     :param name: Name of the plot
 160     :param color: Name of the color for the plot.
 161     :type in_data: OrderedDict
 162     :type job_name: str
 163     :type build_info: dict
 164     :type show_trend_line: bool
 165     :type name: str
 166     :type color: str
 167     :returns: Generated traces (list) and the evaluated result.
 168     :rtype: tuple(traces, result)
 169     """
 170
 171     data_x = list(in_data.keys())
 172     data_y_pps = list(in_data.values())
 173     data_y_mpps = [float(item) / 1e6 for item in data_y_pps]
 174
 175     hover_text = list()
 176     xaxis = list()
 177     for index, key in enumerate(data_x):
 178         str_key = str(key)
 179         date = build_info[job_name][str_key][0]
 180         hover_str = (u"date: {date}<br>"
 181                      u"value [Mpps]: {value:.3f}<br>"
 182                      u"{sut}-ref: {build}<br>"
 183                      u"csit-ref: mrr-{period}-build-{build_nr}<br>"
 184                      u"testbed: {testbed}")
 185         if u"dpdk" in job_name:
 186             hover_text.append(hover_str.format(
 187                 date=date,
 188                 value=data_y_mpps[index],
 189                 sut=u"dpdk",
 190                 build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0],
 191                 period=u"weekly",
 192                 build_nr=str_key,
 193                 testbed=build_info[job_name][str_key][2]))
 194         elif u"vpp" in job_name:
 195             hover_text.append(hover_str.format(
 196                 date=date,
 197                 value=data_y_mpps[index],
 198                 sut=u"vpp",
 199                 build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0],
 200                 period=u"daily",
 201                 build_nr=str_key,
 202                 testbed=build_info[job_name][str_key][2]))
 203
 204         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 205                               int(date[9:11]), int(date[12:])))
 206
 207     data_pd = OrderedDict()
 208     for key, value in zip(xaxis, data_y_pps):
 209         data_pd[key] = value
 210
 211     anomaly_classification, avgs_pps = classify_anomalies(data_pd)
 212     avgs_mpps = [avg_pps / 1e6 for avg_pps in avgs_pps]
 213
 214     anomalies = OrderedDict()
 215     anomalies_colors = list()
 216     anomalies_avgs = list()
 217     anomaly_color = {
 218         u"regression": 0.0,
 219         u"normal": 0.5,
 220         u"progression": 1.0
 221     }
 222     if anomaly_classification:
 223         for index, (key, value) in enumerate(data_pd.items()):
 224             if anomaly_classification[index] in \
 225                     (u"outlier", u"regression", u"progression"):
 226                 anomalies[key] = value / 1e6
 227                 anomalies_colors.append(
 228                     anomaly_color[anomaly_classification[index]])
 229                 anomalies_avgs.append(avgs_mpps[index])
 230         anomalies_colors.extend([0.0, 0.5, 1.0])
 231
 232     # Create traces
 233
 234     trace_samples = plgo.Scatter(
 235         x=xaxis,
 236         y=data_y_mpps,
 237         mode=u"markers",
 238         line={
 239             u"width": 1
 240         },
 241         showlegend=True,
 242         legendgroup=name,
 243         name=f"{name}",
 244         marker={
 245             u"size": 5,
 246             u"color": color,
 247             u"symbol": u"circle",
 248         },
 249         text=hover_text,
 250         hoverinfo=u"text+name"
 251     )
 252     traces = [trace_samples, ]
 253
 254     if show_trend_line:
 255         trace_trend = plgo.Scatter(
 256             x=xaxis,
 257             y=avgs_mpps,
 258             mode=u"lines",
 259             line={
 260                 u"shape": u"linear",
 261                 u"width": 1,
 262                 u"color": color,
 263             },
 264             showlegend=False,
 265             legendgroup=name,
 266             name=f"{name}",
 267             text=[f"trend [Mpps]: {avg:.3f}" for avg in avgs_mpps],
 268             hoverinfo=u"text+name"
 269         )
 270         traces.append(trace_trend)
 271
 272     trace_anomalies = plgo.Scatter(
 273         x=list(anomalies.keys()),
 274         y=anomalies_avgs,
 275         mode=u"markers",
 276         hoverinfo=u"none",
 277         showlegend=False,
 278         legendgroup=name,
 279         name=f"{name}-anomalies",
 280         marker={
 281             u"size": 15,
 282             u"symbol": u"circle-open",
 283             u"color": anomalies_colors,
 284             u"colorscale": [
 285                 [0.00, u"red"],
 286                 [0.33, u"red"],
 287                 [0.33, u"white"],
 288                 [0.66, u"white"],
 289                 [0.66, u"green"],
 290                 [1.00, u"green"]
 291             ],
 292             u"showscale": True,
 293             u"line": {
 294                 u"width": 2
 295             },
 296             u"colorbar": {
 297                 u"y": 0.5,
 298                 u"len": 0.8,
 299                 u"title": u"Circles Marking Data Classification",
 300                 u"titleside": u"right",
 301                 u"titlefont": {
 302                     u"size": 14
 303                 },
 304                 u"tickmode": u"array",
 305                 u"tickvals": [0.167, 0.500, 0.833],
 306                 u"ticktext": [u"Regression", u"Normal", u"Progression"],
 307                 u"ticks": u"",
 308                 u"ticklen": 0,
 309                 u"tickangle": -90,
 310                 u"thickness": 10
 311             }
 312         }
 313     )
 314     traces.append(trace_anomalies)
 315
 316     if anomaly_classification:
 317         return traces, anomaly_classification[-1]
 318
 319     return traces, None
 320
 321
 322 def _generate_all_charts(spec, input_data):
 323     """Generate all charts specified in the specification file.
 324
 325     :param spec: Specification.
 326     :param input_data: Full data set.
 327     :type spec: Specification
 328     :type input_data: InputData
 329     """
 330
 331     def _generate_chart(graph):
 332         """Generates the chart.
 333
 334         :param graph: The graph to be generated
 335         :type graph: dict
 336         :returns: Dictionary with the job name, csv table with results and
 337             list of tests classification results.
 338         :rtype: dict
 339         """
 340
 341         logs = list()
 342
 343         logs.append(
 344             (u"INFO", f"  Generating the chart {graph.get(u'title', u'')} ...")
 345         )
 346
 347         job_name = list(graph[u"data"].keys())[0]
 348
 349         csv_tbl = list()
 350         res = dict()
 351
 352         # Transform the data
 353         logs.append(
 354             (u"INFO",
 355              f"    Creating the data set for the {graph.get(u'type', u'')} "
 356              f"{graph.get(u'title', u'')}."
 357             )
 358         )
 359
 360         if graph.get(u"include", None):
 361             data = input_data.filter_tests_by_name(
 362                 graph,
 363                 params=[u"type", u"result", u"tags"],
 364                 continue_on_error=True
 365             )
 366         else:
 367             data = input_data.filter_data(
 368                 graph,
 369                 params=[u"type", u"result", u"tags"],
 370                 continue_on_error=True)
 371
 372         if data is None or data.empty:
 373             logging.error(u"No data.")
 374             return dict()
 375
 376         chart_data = dict()
 377         chart_tags = dict()
 378         for job, job_data in data.items():
 379             if job != job_name:
 380                 continue
 381             for index, bld in job_data.items():
 382                 for test_name, test in bld.items():
 383                     if chart_data.get(test_name, None) is None:
 384                         chart_data[test_name] = OrderedDict()
 385                     try:
 386                         chart_data[test_name][int(index)] = \
 387                             test[u"result"][u"receive-rate"]
 388                         chart_tags[test_name] = test.get(u"tags", None)
 389                     except (KeyError, TypeError):
 390                         pass
 391
 392         # Add items to the csv table:
 393         for tst_name, tst_data in chart_data.items():
 394             tst_lst = list()
 395             for bld in builds_dict[job_name]:
 396                 itm = tst_data.get(int(bld), u'')
 397                 # CSIT-1180: Itm will be list, compute stats.
 398                 tst_lst.append(str(itm))
 399             csv_tbl.append(f"{tst_name}," + u",".join(tst_lst) + u'\n')
 400
 401         # Generate traces:
 402         traces = list()
 403         index = 0
 404         groups = graph.get(u"groups", None)
 405         visibility = list()
 406
 407         if groups:
 408             for group in groups:
 409                 visible = list()
 410                 for tag in group:
 411                     for tst_name, test_data in chart_data.items():
 412                         if not test_data:
 413                             logs.append(
 414                                 (u"WARNING", f"No data for the test {tst_name}")
 415                             )
 416                             continue
 417                         if tag not in chart_tags[tst_name]:
 418                             continue
 419                         message = f"index: {index}, test: {tst_name}"
 420                         try:
 421                             trace, rslt = _generate_trending_traces(
 422                                 test_data,
 423                                 job_name=job_name,
 424                                 build_info=build_info,
 425                                 name=u'-'.join(tst_name.split(u'.')[-1].
 426                                                split(u'-')[2:-1]),
 427                                 color=COLORS[index])
 428                         except IndexError:
 429                             logs.append(
 430                                 (u"ERROR", f"Out of colors: {message}")
 431                             )
 432                             logging.error(f"Out of colors: {message}")
 433                             index += 1
 434                             continue
 435                         traces.extend(trace)
 436                         visible.extend([True for _ in range(len(trace))])
 437                         res[tst_name] = rslt
 438                         index += 1
 439                         break
 440                 visibility.append(visible)
 441         else:
 442             for tst_name, test_data in chart_data.items():
 443                 if not test_data:
 444                     logs.append(
 445                         (u"WARNING", f"No data for the test {tst_name}")
 446                     )
 447                     continue
 448                 message = f"index: {index}, test: {tst_name}"
 449                 try:
 450                     trace, rslt = _generate_trending_traces(
 451                         test_data,
 452                         job_name=job_name,
 453                         build_info=build_info,
 454                         name=u'-'.join(
 455                             tst_name.split(u'.')[-1].split(u'-')[2:-1]),
 456                         color=COLORS[index])
 457                 except IndexError:
 458                     logs.append((u"ERROR", f"Out of colors: {message}"))
 459                     logging.error(f"Out of colors: {message}")
 460                     index += 1
 461                     continue
 462                 traces.extend(trace)
 463                 res[tst_name] = rslt
 464                 index += 1
 465
 466         if traces:
 467             # Generate the chart:
 468             try:
 469                 layout = deepcopy(graph[u"layout"])
 470             except KeyError as err:
 471                 logging.error(u"Finished with error: No layout defined")
 472                 logging.error(repr(err))
 473                 return dict()
 474             if groups:
 475                 show = list()
 476                 for i in range(len(visibility)):
 477                     visible = list()
 478                     for vis_idx, _ in enumerate(visibility):
 479                         for _ in range(len(visibility[vis_idx])):
 480                             visible.append(i == vis_idx)
 481                     show.append(visible)
 482
 483                 buttons = list()
 484                 buttons.append(dict(
 485                     label=u"All",
 486                     method=u"update",
 487                     args=[{u"visible": [True for _ in range(len(show[0]))]}, ]
 488                 ))
 489                 for i in range(len(groups)):
 490                     try:
 491                         label = graph[u"group-names"][i]
 492                     except (IndexError, KeyError):
 493                         label = f"Group {i + 1}"
 494                     buttons.append(dict(
 495                         label=label,
 496                         method=u"update",
 497                         args=[{u"visible": show[i]}, ]
 498                     ))
 499
 500                 layout[u"updatemenus"] = list([
 501                     dict(
 502                         active=0,
 503                         type=u"dropdown",
 504                         direction=u"down",
 505                         xanchor=u"left",
 506                         yanchor=u"bottom",
 507                         x=-0.12,
 508                         y=1.0,
 509                         buttons=buttons
 510                     )
 511                 ])
 512
 513             name_file = (
 514                 f"{spec.cpta[u'output-file']}/{graph[u'output-file-name']}"
 515                 f"{spec.cpta[u'output-file-type']}")
 516
 517             logs.append((u"INFO", f"    Writing the file {name_file} ..."))
 518             plpl = plgo.Figure(data=traces, layout=layout)
 519             try:
 520                 ploff.plot(plpl, show_link=False, auto_open=False,
 521                            filename=name_file)
 522             except plerr.PlotlyEmptyDataError:
 523                 logs.append((u"WARNING", u"No data for the plot. Skipped."))
 524
 525         for level, line in logs:
 526             if level == u"INFO":
 527                 logging.info(line)
 528             elif level == u"ERROR":
 529                 logging.error(line)
 530             elif level == u"DEBUG":
 531                 logging.debug(line)
 532             elif level == u"CRITICAL":
 533                 logging.critical(line)
 534             elif level == u"WARNING":
 535                 logging.warning(line)
 536
 537         return {u"job_name": job_name, u"csv_table": csv_tbl, u"results": res}
 538
 539     builds_dict = dict()
 540     for job in spec.input[u"builds"].keys():
 541         if builds_dict.get(job, None) is None:
 542             builds_dict[job] = list()
 543         for build in spec.input[u"builds"][job]:
 544             status = build[u"status"]
 545             if status not in (u"failed", u"not found", u"removed"):
 546                 builds_dict[job].append(str(build[u"build"]))
 547
 548     # Create "build ID": "date" dict:
 549     build_info = dict()
 550     tb_tbl = spec.environment.get(u"testbeds", None)
 551     for job_name, job_data in builds_dict.items():
 552         if build_info.get(job_name, None) is None:
 553             build_info[job_name] = OrderedDict()
 554         for build in job_data:
 555             testbed = u""
 556             tb_ip = input_data.metadata(job_name, build).get(u"testbed", u"")
 557             if tb_ip and tb_tbl:
 558                 testbed = tb_tbl.get(tb_ip, u"")
 559             build_info[job_name][build] = (
 560                 input_data.metadata(job_name, build).get(u"generated", u""),
 561                 input_data.metadata(job_name, build).get(u"version", u""),
 562                 testbed
 563             )
 564
 565     anomaly_classifications = dict()
 566
 567     # Create the table header:
 568     csv_tables = dict()
 569     for job_name in builds_dict:
 570         if csv_tables.get(job_name, None) is None:
 571             csv_tables[job_name] = list()
 572         header = f"Build Number:,{u','.join(builds_dict[job_name])}\n"
 573         csv_tables[job_name].append(header)
 574         build_dates = [x[0] for x in build_info[job_name].values()]
 575         header = f"Build Date:,{u','.join(build_dates)}\n"
 576         csv_tables[job_name].append(header)
 577         versions = [x[1] for x in build_info[job_name].values()]
 578         header = f"Version:,{u','.join(versions)}\n"
 579         csv_tables[job_name].append(header)
 580
 581     for chart in spec.cpta[u"plots"]:
 582         result = _generate_chart(chart)
 583         if not result:
 584             continue
 585
 586         csv_tables[result[u"job_name"]].extend(result[u"csv_table"])
 587
 588         if anomaly_classifications.get(result[u"job_name"], None) is None:
 589             anomaly_classifications[result[u"job_name"]] = dict()
 590         anomaly_classifications[result[u"job_name"]].update(result[u"results"])
 591
 592     # Write the tables:
 593     for job_name, csv_table in csv_tables.items():
 594         file_name = f"{spec.cpta[u'output-file']}/{job_name}-trending"
 595         with open(f"{file_name}.csv", u"wt") as file_handler:
 596             file_handler.writelines(csv_table)
 597
 598         txt_table = None
 599         with open(f"{file_name}.csv", u"rt") as csv_file:
 600             csv_content = csv.reader(csv_file, delimiter=u',', quotechar=u'"')
 601             line_nr = 0
 602             for row in csv_content:
 603                 if txt_table is None:
 604                     txt_table = prettytable.PrettyTable(row)
 605                 else:
 606                     if line_nr > 1:
 607                         for idx, item in enumerate(row):
 608                             try:
 609                                 row[idx] = str(round(float(item) / 1000000, 2))
 610                             except ValueError:
 611                                 pass
 612                     try:
 613                         txt_table.add_row(row)
 614                     # PrettyTable raises Exception
 615                     except Exception as err:
 616                         logging.warning(
 617                             f"Error occurred while generating TXT table:\n{err}"
 618                         )
 619                 line_nr += 1
 620             txt_table.align[u"Build Number:"] = u"l"
 621         with open(f"{file_name}.txt", u"wt") as txt_file:
 622             txt_file.write(str(txt_table))
 623
 624     # Evaluate result:
 625     if anomaly_classifications:
 626         result = u"PASS"
 627         for job_name, job_data in anomaly_classifications.items():
 628             file_name = \
 629                 f"{spec.cpta[u'output-file']}/regressions-{job_name}.txt"
 630             with open(file_name, u'w') as txt_file:
 631                 for test_name, classification in job_data.items():
 632                     if classification == u"regression":
 633                         txt_file.write(test_name + u'\n')
 634                     if classification in (u"regression", u"outlier"):
 635                         result = u"FAIL"
 636             file_name = \
 637                 f"{spec.cpta[u'output-file']}/progressions-{job_name}.txt"
 638             with open(file_name, u'w') as txt_file:
 639                 for test_name, classification in job_data.items():
 640                     if classification == u"progression":
 641                         txt_file.write(test_name + u'\n')
 642     else:
 643         result = u"FAIL"
 644
 645     logging.info(f"Partial results: {anomaly_classifications}")
 646     logging.info(f"Result: {result}")
 647
 648     return result