resources/tools/presentation/generator_cpta.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import logging
  18 import csv
  19
  20 from collections import OrderedDict
  21 from datetime import datetime
  22 from copy import deepcopy
  23
  24 import prettytable
  25 import plotly.offline as ploff
  26 import plotly.graph_objs as plgo
  27 import plotly.exceptions as plerr
  28
  29 from pal_utils import archive_input_data, execute_command, classify_anomalies
  30
  31
  32 # Command to build the html format of the report
  33 HTML_BUILDER = u'sphinx-build -v -c conf_cpta -a ' \
  34                u'-b html -E ' \
  35                u'-t html ' \
  36                u'-D version="{date}" ' \
  37                u'{working_dir} ' \
  38                u'{build_dir}/'
  39
  40 # .css file for the html format of the report
  41 THEME_OVERRIDES = u"""/* override table width restrictions */
  42 .wy-nav-content {
  43     max-width: 1200px !important;
  44 }
  45 .rst-content blockquote {
  46     margin-left: 0px;
  47     line-height: 18px;
  48     margin-bottom: 0px;
  49 }
  50 .wy-menu-vertical a {
  51     display: inline-block;
  52     line-height: 18px;
  53     padding: 0 2em;
  54     display: block;
  55     position: relative;
  56     font-size: 90%;
  57     color: #d9d9d9
  58 }
  59 .wy-menu-vertical li.current a {
  60     color: gray;
  61     border-right: solid 1px #c9c9c9;
  62     padding: 0 3em;
  63 }
  64 .wy-menu-vertical li.toctree-l2.current > a {
  65     background: #c9c9c9;
  66     padding: 0 3em;
  67 }
  68 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
  69     display: block;
  70     background: #c9c9c9;
  71     padding: 0 4em;
  72 }
  73 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
  74     display: block;
  75     background: #bdbdbd;
  76     padding: 0 5em;
  77 }
  78 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
  79     color: #404040;
  80     padding: 0 2em;
  81     font-weight: bold;
  82     position: relative;
  83     background: #fcfcfc;
  84     border: none;
  85         border-top-width: medium;
  86         border-bottom-width: medium;
  87         border-top-style: none;
  88         border-bottom-style: none;
  89         border-top-color: currentcolor;
  90         border-bottom-color: currentcolor;
  91     padding-left: 2em -4px;
  92 }
  93 """
  94
  95 COLORS = [
  96     u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink",
  97     u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black",
  98     u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson",
  99     u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod",
 100     u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon",
 101     u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey",
 102     u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink",
 103     u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black",
 104     u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson",
 105     u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod",
 106     u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon",
 107     u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey"
 108 ]
 109
 110
 111 def generate_cpta(spec, data):
 112     """Generate all formats and versions of the Continuous Performance Trending
 113     and Analysis.
 114
 115     :param spec: Specification read from the specification file.
 116     :param data: Full data set.
 117     :type spec: Specification
 118     :type data: InputData
 119     """
 120
 121     logging.info(u"Generating the Continuous Performance Trending and Analysis "
 122                  u"...")
 123
 124     ret_code = _generate_all_charts(spec, data)
 125
 126     cmd = HTML_BUILDER.format(
 127         date=datetime.utcnow().strftime(u'%Y-%m-%d %H:%M UTC'),
 128         working_dir=spec.environment[u'paths'][u'DIR[WORKING,SRC]'],
 129         build_dir=spec.environment[u'paths'][u'DIR[BUILD,HTML]'])
 130     execute_command(cmd)
 131
 132     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE]'], u'w') as \
 133             css_file:
 134         css_file.write(THEME_OVERRIDES)
 135
 136     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE2]'], u'w') as \
 137             css_file:
 138         css_file.write(THEME_OVERRIDES)
 139
 140     if spec.configuration.get(u"archive-inputs", True):
 141         archive_input_data(spec)
 142
 143     logging.info(u"Done.")
 144
 145     return ret_code
 146
 147
 148 def _generate_trending_traces(in_data, job_name, build_info,
 149                               show_trend_line=True, name=u"", color=u""):
 150     """Generate the trending traces:
 151      - samples,
 152      - outliers, regress, progress
 153      - average of normal samples (trending line)
 154
 155     :param in_data: Full data set.
 156     :param job_name: The name of job which generated the data.
 157     :param build_info: Information about the builds.
 158     :param show_trend_line: Show moving median (trending plot).
 159     :param name: Name of the plot
 160     :param color: Name of the color for the plot.
 161     :type in_data: OrderedDict
 162     :type job_name: str
 163     :type build_info: dict
 164     :type show_trend_line: bool
 165     :type name: str
 166     :type color: str
 167     :returns: Generated traces (list) and the evaluated result.
 168     :rtype: tuple(traces, result)
 169     """
 170
 171     data_x = list(in_data.keys())
 172     data_y = list(in_data.values())
 173
 174     hover_text = list()
 175     xaxis = list()
 176     for idx in data_x:
 177         date = build_info[job_name][str(idx)][0]
 178         hover_str = (u"date: {date}<br>"
 179                      u"value: {value:,}<br>"
 180                      u"{sut}-ref: {build}<br>"
 181                      u"csit-ref: mrr-{period}-build-{build_nr}<br>"
 182                      u"testbed: {testbed}")
 183         if u"dpdk" in job_name:
 184             hover_text.append(hover_str.format(
 185                 date=date,
 186                 value=int(in_data[idx]),
 187                 sut=u"dpdk",
 188                 build=build_info[job_name][str(idx)][1].rsplit(u'~', 1)[0],
 189                 period=u"weekly",
 190                 build_nr=idx,
 191                 testbed=build_info[job_name][str(idx)][2]))
 192         elif u"vpp" in job_name:
 193             hover_text.append(hover_str.format(
 194                 date=date,
 195                 value=int(in_data[idx]),
 196                 sut=u"vpp",
 197                 build=build_info[job_name][str(idx)][1].rsplit(u'~', 1)[0],
 198                 period=u"daily",
 199                 build_nr=idx,
 200                 testbed=build_info[job_name][str(idx)][2]))
 201
 202         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 203                               int(date[9:11]), int(date[12:])))
 204
 205     data_pd = OrderedDict()
 206     for key, value in zip(xaxis, data_y):
 207         data_pd[key] = value
 208
 209     anomaly_classification, avgs = classify_anomalies(data_pd)
 210
 211     anomalies = OrderedDict()
 212     anomalies_colors = list()
 213     anomalies_avgs = list()
 214     anomaly_color = {
 215         u"regression": 0.0,
 216         u"normal": 0.5,
 217         u"progression": 1.0
 218     }
 219     if anomaly_classification:
 220         for idx, (key, value) in enumerate(data_pd.items()):
 221             if anomaly_classification[idx] in \
 222                     (u"outlier", u"regression", u"progression"):
 223                 anomalies[key] = value
 224                 anomalies_colors.append(
 225                     anomaly_color[anomaly_classification[idx]])
 226                 anomalies_avgs.append(avgs[idx])
 227         anomalies_colors.extend([0.0, 0.5, 1.0])
 228
 229     # Create traces
 230
 231     trace_samples = plgo.Scatter(
 232         x=xaxis,
 233         y=data_y,
 234         mode=u"markers",
 235         line={
 236             u"width": 1
 237         },
 238         showlegend=True,
 239         legendgroup=name,
 240         name=f"{name}",
 241         marker={
 242             u"size": 5,
 243             u"color": color,
 244             u"symbol": u"circle",
 245         },
 246         text=hover_text,
 247         hoverinfo=u"text"
 248     )
 249     traces = [trace_samples, ]
 250
 251     if show_trend_line:
 252         trace_trend = plgo.Scatter(
 253             x=xaxis,
 254             y=avgs,
 255             mode=u"lines",
 256             line={
 257                 u"shape": u"linear",
 258                 u"width": 1,
 259                 u"color": color,
 260             },
 261             showlegend=False,
 262             legendgroup=name,
 263             name=f"{name}",
 264             text=[f"trend: {int(avg):,}" for avg in avgs],
 265             hoverinfo=u"text+name"
 266         )
 267         traces.append(trace_trend)
 268
 269     trace_anomalies = plgo.Scatter(
 270         x=list(anomalies.keys()),
 271         y=anomalies_avgs,
 272         mode=u"markers",
 273         hoverinfo=u"none",
 274         showlegend=False,
 275         legendgroup=name,
 276         name=f"{name}-anomalies",
 277         marker={
 278             u"size": 15,
 279             u"symbol": u"circle-open",
 280             u"color": anomalies_colors,
 281             u"colorscale": [
 282                 [0.00, u"red"],
 283                 [0.33, u"red"],
 284                 [0.33, u"white"],
 285                 [0.66, u"white"],
 286                 [0.66, u"green"],
 287                 [1.00, u"green"]
 288             ],
 289             u"showscale": True,
 290             u"line": {
 291                 u"width": 2
 292             },
 293             u"colorbar": {
 294                 u"y": 0.5,
 295                 u"len": 0.8,
 296                 u"title": u"Circles Marking Data Classification",
 297                 u"titleside": u"right",
 298                 u"titlefont": {
 299                     u"size": 14
 300                 },
 301                 u"tickmode": u"array",
 302                 u"tickvals": [0.167, 0.500, 0.833],
 303                 u"ticktext": [u"Regression", u"Normal", u"Progression"],
 304                 u"ticks": u"",
 305                 u"ticklen": 0,
 306                 u"tickangle": -90,
 307                 u"thickness": 10
 308             }
 309         }
 310     )
 311     traces.append(trace_anomalies)
 312
 313     if anomaly_classification:
 314         return traces, anomaly_classification[-1]
 315
 316     return traces, None
 317
 318
 319 def _generate_all_charts(spec, input_data):
 320     """Generate all charts specified in the specification file.
 321
 322     :param spec: Specification.
 323     :param input_data: Full data set.
 324     :type spec: Specification
 325     :type input_data: InputData
 326     """
 327
 328     def _generate_chart(graph):
 329         """Generates the chart.
 330
 331         :param graph: The graph to be generated
 332         :type graph: dict
 333         :returns: Dictionary with the job name, csv table with results and
 334             list of tests classification results.
 335         :rtype: dict
 336         """
 337
 338         logs = list()
 339
 340         logs.append(
 341             (u"INFO", f"  Generating the chart {graph.get(u'title', u'')} ...")
 342         )
 343
 344         job_name = list(graph[u"data"].keys())[0]
 345
 346         csv_tbl = list()
 347         res = dict()
 348
 349         # Transform the data
 350         logs.append(
 351             (u"INFO",
 352              f"    Creating the data set for the {graph.get(u'type', u'')} "
 353              f"{graph.get(u'title', u'')}."
 354             )
 355         )
 356
 357         if graph.get(u"include", None):
 358             data = input_data.filter_tests_by_name(
 359                 graph, continue_on_error=True
 360             )
 361         else:
 362             data = input_data.filter_data(graph, continue_on_error=True)
 363
 364         if data is None or data.empty:
 365             logging.error(u"No data.")
 366             return dict()
 367
 368         chart_data = dict()
 369         chart_tags = dict()
 370         for job, job_data in data.items():
 371             if job != job_name:
 372                 continue
 373             for index, bld in job_data.items():
 374                 for test_name, test in bld.items():
 375                     if chart_data.get(test_name, None) is None:
 376                         chart_data[test_name] = OrderedDict()
 377                     try:
 378                         chart_data[test_name][int(index)] = \
 379                             test[u"result"][u"receive-rate"]
 380                         chart_tags[test_name] = test.get(u"tags", None)
 381                     except (KeyError, TypeError):
 382                         pass
 383
 384         # Add items to the csv table:
 385         for tst_name, tst_data in chart_data.items():
 386             tst_lst = list()
 387             for bld in builds_dict[job_name]:
 388                 itm = tst_data.get(int(bld), u'')
 389                 # CSIT-1180: Itm will be list, compute stats.
 390                 tst_lst.append(str(itm))
 391             csv_tbl.append(f"{tst_name}," + u",".join(tst_lst) + u'\n')
 392
 393         # Generate traces:
 394         traces = list()
 395         index = 0
 396         groups = graph.get(u"groups", None)
 397         visibility = list()
 398
 399         if groups:
 400             for group in groups:
 401                 visible = list()
 402                 for tag in group:
 403                     for tst_name, test_data in chart_data.items():
 404                         if not test_data:
 405                             logs.append(
 406                                 (u"WARNING", f"No data for the test {tst_name}")
 407                             )
 408                             continue
 409                         if tag not in chart_tags[tst_name]:
 410                             continue
 411                         message = f"index: {index}, test: {tst_name}"
 412                         try:
 413                             trace, rslt = _generate_trending_traces(
 414                                 test_data,
 415                                 job_name=job_name,
 416                                 build_info=build_info,
 417                                 name=u'-'.join(tst_name.split(u'.')[-1].
 418                                                split(u'-')[2:-1]),
 419                                 color=COLORS[index])
 420                         except IndexError:
 421                             logs.append(
 422                                 (u"ERROR", f"Out of colors: {message}")
 423                             )
 424                             logging.error(f"Out of colors: {message}")
 425                             index += 1
 426                             continue
 427                         traces.extend(trace)
 428                         visible.extend([True for _ in range(len(trace))])
 429                         res[tst_name] = rslt
 430                         index += 1
 431                         break
 432                 visibility.append(visible)
 433         else:
 434             for tst_name, test_data in chart_data.items():
 435                 if not test_data:
 436                     logs.append(
 437                         (u"WARNING", f"No data for the test {tst_name}")
 438                     )
 439                     continue
 440                 message = f"index: {index}, test: {tst_name}"
 441                 try:
 442                     trace, rslt = _generate_trending_traces(
 443                         test_data,
 444                         job_name=job_name,
 445                         build_info=build_info,
 446                         name=u'-'.join(
 447                             tst_name.split(u'.')[-1].split(u'-')[2:-1]),
 448                         color=COLORS[index])
 449                 except IndexError:
 450                     logs.append((u"ERROR", f"Out of colors: {message}"))
 451                     logging.error(f"Out of colors: {message}")
 452                     index += 1
 453                     continue
 454                 traces.extend(trace)
 455                 res[tst_name] = rslt
 456                 index += 1
 457
 458         if traces:
 459             # Generate the chart:
 460             try:
 461                 layout = deepcopy(graph[u"layout"])
 462             except KeyError as err:
 463                 logging.error(u"Finished with error: No layout defined")
 464                 logging.error(repr(err))
 465                 return dict()
 466             if groups:
 467                 show = list()
 468                 for i in range(len(visibility)):
 469                     visible = list()
 470                     for vis_idx, _ in enumerate(visibility):
 471                         for _ in range(len(visibility[vis_idx])):
 472                             visible.append(i == vis_idx)
 473                     show.append(visible)
 474
 475                 buttons = list()
 476                 buttons.append(dict(
 477                     label=u"All",
 478                     method=u"update",
 479                     args=[{u"visible": [True for _ in range(len(show[0]))]}, ]
 480                 ))
 481                 for i in range(len(groups)):
 482                     try:
 483                         label = graph[u"group-names"][i]
 484                     except (IndexError, KeyError):
 485                         label = f"Group {i + 1}"
 486                     buttons.append(dict(
 487                         label=label,
 488                         method=u"update",
 489                         args=[{u"visible": show[i]}, ]
 490                     ))
 491
 492                 layout[u"updatemenus"] = list([
 493                     dict(
 494                         active=0,
 495                         type=u"dropdown",
 496                         direction=u"down",
 497                         xanchor=u"left",
 498                         yanchor=u"bottom",
 499                         x=-0.12,
 500                         y=1.0,
 501                         buttons=buttons
 502                     )
 503                 ])
 504
 505             name_file = (
 506                 f"{spec.cpta[u'output-file']}/{graph[u'output-file-name']}"
 507                 f"{spec.cpta[u'output-file-type']}")
 508
 509             logs.append((u"INFO", f"    Writing the file {name_file} ..."))
 510             plpl = plgo.Figure(data=traces, layout=layout)
 511             try:
 512                 ploff.plot(plpl, show_link=False, auto_open=False,
 513                            filename=name_file)
 514             except plerr.PlotlyEmptyDataError:
 515                 logs.append((u"WARNING", u"No data for the plot. Skipped."))
 516
 517         for level, line in logs:
 518             if level == u"INFO":
 519                 logging.info(line)
 520             elif level == u"ERROR":
 521                 logging.error(line)
 522             elif level == u"DEBUG":
 523                 logging.debug(line)
 524             elif level == u"CRITICAL":
 525                 logging.critical(line)
 526             elif level == u"WARNING":
 527                 logging.warning(line)
 528
 529         return {u"job_name": job_name, u"csv_table": csv_tbl, u"results": res}
 530
 531     builds_dict = dict()
 532     for job in spec.input[u"builds"].keys():
 533         if builds_dict.get(job, None) is None:
 534             builds_dict[job] = list()
 535         for build in spec.input[u"builds"][job]:
 536             status = build[u"status"]
 537             if status not in (u"failed", u"not found", u"removed"):
 538                 builds_dict[job].append(str(build[u"build"]))
 539
 540     # Create "build ID": "date" dict:
 541     build_info = dict()
 542     tb_tbl = spec.environment.get(u"testbeds", None)
 543     for job_name, job_data in builds_dict.items():
 544         if build_info.get(job_name, None) is None:
 545             build_info[job_name] = OrderedDict()
 546         for build in job_data:
 547             testbed = u""
 548             tb_ip = input_data.metadata(job_name, build).get(u"testbed", u"")
 549             if tb_ip and tb_tbl:
 550                 testbed = tb_tbl.get(tb_ip, u"")
 551             build_info[job_name][build] = (
 552                 input_data.metadata(job_name, build).get(u"generated", u""),
 553                 input_data.metadata(job_name, build).get(u"version", u""),
 554                 testbed
 555             )
 556
 557     anomaly_classifications = dict()
 558
 559     # Create the header:
 560     csv_tables = dict()
 561     for job_name in builds_dict:
 562         if csv_tables.get(job_name, None) is None:
 563             csv_tables[job_name] = list()
 564         header = u"Build Number:," + u",".join(builds_dict[job_name]) + u'\n'
 565         csv_tables[job_name].append(header)
 566         build_dates = [x[0] for x in build_info[job_name].values()]
 567         header = u"Build Date:," + u",".join(build_dates) + u'\n'
 568         csv_tables[job_name].append(header)
 569         versions = [x[1] for x in build_info[job_name].values()]
 570         header = u"Version:," + u",".join(versions) + u'\n'
 571         csv_tables[job_name].append(header)
 572
 573     for chart in spec.cpta[u"plots"]:
 574         result = _generate_chart(chart)
 575         if not result:
 576             continue
 577
 578         csv_tables[result[u"job_name"]].extend(result[u"csv_table"])
 579
 580         if anomaly_classifications.get(result[u"job_name"], None) is None:
 581             anomaly_classifications[result[u"job_name"]] = dict()
 582         anomaly_classifications[result[u"job_name"]].update(result[u"results"])
 583
 584     # Write the tables:
 585     for job_name, csv_table in csv_tables.items():
 586         file_name = spec.cpta[u"output-file"] + u"-" + job_name + u"-trending"
 587         with open(f"{file_name}.csv", u"w") as file_handler:
 588             file_handler.writelines(csv_table)
 589
 590         txt_table = None
 591         with open(f"{file_name}.csv", u"rt") as csv_file:
 592             csv_content = csv.reader(csv_file, delimiter=u',', quotechar=u'"')
 593             line_nr = 0
 594             for row in csv_content:
 595                 if txt_table is None:
 596                     txt_table = prettytable.PrettyTable(row)
 597                 else:
 598                     if line_nr > 1:
 599                         for idx, item in enumerate(row):
 600                             try:
 601                                 row[idx] = str(round(float(item) / 1000000, 2))
 602                             except ValueError:
 603                                 pass
 604                     try:
 605                         txt_table.add_row(row)
 606                     # PrettyTable raises Exception
 607                     except Exception as err:
 608                         logging.warning(
 609                             f"Error occurred while generating TXT table:\n{err}"
 610                         )
 611                 line_nr += 1
 612             txt_table.align[u"Build Number:"] = u"l"
 613         with open(f"{file_name}.txt", u"w") as txt_file:
 614             txt_file.write(str(txt_table))
 615
 616     # Evaluate result:
 617     if anomaly_classifications:
 618         result = u"PASS"
 619         for job_name, job_data in anomaly_classifications.items():
 620             file_name = \
 621                 f"{spec.cpta[u'output-file']}/regressions-{job_name}.txt"
 622             with open(file_name, u'w') as txt_file:
 623                 for test_name, classification in job_data.items():
 624                     if classification == u"regression":
 625                         txt_file.write(test_name + u'\n')
 626                     if classification in (u"regression", u"outlier"):
 627                         result = u"FAIL"
 628             file_name = \
 629                 f"{spec.cpta[u'output-file']}/progressions-{job_name}.txt"
 630             with open(file_name, u'w') as txt_file:
 631                 for test_name, classification in job_data.items():
 632                     if classification == u"progression":
 633                         txt_file.write(test_name + u'\n')
 634     else:
 635         result = u"FAIL"
 636
 637     logging.info(f"Partial results: {anomaly_classifications}")
 638     logging.info(f"Result: {result}")
 639
 640     return result