resources/tools/presentation/generator_cpta.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import logging
  18 import csv
  19
  20 from collections import OrderedDict
  21 from datetime import datetime
  22 from copy import deepcopy
  23
  24 import prettytable
  25 import plotly.offline as ploff
  26 import plotly.graph_objs as plgo
  27 import plotly.exceptions as plerr
  28
  29 from pal_utils import archive_input_data, execute_command, classify_anomalies
  30
  31
  32 # Command to build the html format of the report
  33 HTML_BUILDER = u'sphinx-build -v -c conf_cpta -a ' \
  34                u'-b html -E ' \
  35                u'-t html ' \
  36                u'-D version="{date}" ' \
  37                u'{working_dir} ' \
  38                u'{build_dir}/'
  39
  40 # .css file for the html format of the report
  41 THEME_OVERRIDES = u"""/* override table width restrictions */
  42 .wy-nav-content {
  43     max-width: 1200px !important;
  44 }
  45 .rst-content blockquote {
  46     margin-left: 0px;
  47     line-height: 18px;
  48     margin-bottom: 0px;
  49 }
  50 .wy-menu-vertical a {
  51     display: inline-block;
  52     line-height: 18px;
  53     padding: 0 2em;
  54     display: block;
  55     position: relative;
  56     font-size: 90%;
  57     color: #d9d9d9
  58 }
  59 .wy-menu-vertical li.current a {
  60     color: gray;
  61     border-right: solid 1px #c9c9c9;
  62     padding: 0 3em;
  63 }
  64 .wy-menu-vertical li.toctree-l2.current > a {
  65     background: #c9c9c9;
  66     padding: 0 3em;
  67 }
  68 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
  69     display: block;
  70     background: #c9c9c9;
  71     padding: 0 4em;
  72 }
  73 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
  74     display: block;
  75     background: #bdbdbd;
  76     padding: 0 5em;
  77 }
  78 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
  79     color: #404040;
  80     padding: 0 2em;
  81     font-weight: bold;
  82     position: relative;
  83     background: #fcfcfc;
  84     border: none;
  85         border-top-width: medium;
  86         border-bottom-width: medium;
  87         border-top-style: none;
  88         border-bottom-style: none;
  89         border-top-color: currentcolor;
  90         border-bottom-color: currentcolor;
  91     padding-left: 2em -4px;
  92 }
  93 """
  94
  95 COLORS = [
  96     u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink",
  97     u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black",
  98     u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson",
  99     u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod",
 100     u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon",
 101     u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey",
 102     u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink",
 103     u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black",
 104     u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson",
 105     u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod",
 106     u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon",
 107     u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey"
 108 ]
 109
 110
 111 def generate_cpta(spec, data):
 112     """Generate all formats and versions of the Continuous Performance Trending
 113     and Analysis.
 114
 115     :param spec: Specification read from the specification file.
 116     :param data: Full data set.
 117     :type spec: Specification
 118     :type data: InputData
 119     """
 120
 121     logging.info(u"Generating the Continuous Performance Trending and Analysis "
 122                  u"...")
 123
 124     ret_code = _generate_all_charts(spec, data)
 125
 126     cmd = HTML_BUILDER.format(
 127         date=datetime.utcnow().strftime(u'%Y-%m-%d %H:%M UTC'),
 128         working_dir=spec.environment[u'paths'][u'DIR[WORKING,SRC]'],
 129         build_dir=spec.environment[u'paths'][u'DIR[BUILD,HTML]'])
 130     execute_command(cmd)
 131
 132     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE]'], u'w') as \
 133             css_file:
 134         css_file.write(THEME_OVERRIDES)
 135
 136     with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE2]'], u'w') as \
 137             css_file:
 138         css_file.write(THEME_OVERRIDES)
 139
 140     if spec.configuration.get(u"archive-inputs", True):
 141         archive_input_data(spec)
 142
 143     logging.info(u"Done.")
 144
 145     return ret_code
 146
 147
 148 def _generate_trending_traces(in_data, job_name, build_info,
 149                               show_trend_line=True, name=u"", color=u""):
 150     """Generate the trending traces:
 151      - samples,
 152      - outliers, regress, progress
 153      - average of normal samples (trending line)
 154
 155     :param in_data: Full data set.
 156     :param job_name: The name of job which generated the data.
 157     :param build_info: Information about the builds.
 158     :param show_trend_line: Show moving median (trending plot).
 159     :param name: Name of the plot
 160     :param color: Name of the color for the plot.
 161     :type in_data: OrderedDict
 162     :type job_name: str
 163     :type build_info: dict
 164     :type show_trend_line: bool
 165     :type name: str
 166     :type color: str
 167     :returns: Generated traces (list) and the evaluated result.
 168     :rtype: tuple(traces, result)
 169     """
 170
 171     data_x = list(in_data.keys())
 172     data_y = list(in_data.values())
 173
 174     hover_text = list()
 175     xaxis = list()
 176     for idx in data_x:
 177         date = build_info[job_name][str(idx)][0]
 178         hover_str = (u"date: {date}<br>"
 179                      u"value: {value:,}<br>"
 180                      u"{sut}-ref: {build}<br>"
 181                      u"csit-ref: mrr-{period}-build-{build_nr}<br>"
 182                      u"testbed: {testbed}")
 183         if u"dpdk" in job_name:
 184             hover_text.append(hover_str.format(
 185                 date=date,
 186                 value=int(in_data[idx]),
 187                 sut=u"dpdk",
 188                 build=build_info[job_name][str(idx)][1].rsplit(u'~', 1)[0],
 189                 period=u"weekly",
 190                 build_nr=idx,
 191                 testbed=build_info[job_name][str(idx)][2]))
 192         elif u"vpp" in job_name:
 193             hover_text.append(hover_str.format(
 194                 date=date,
 195                 value=int(in_data[idx]),
 196                 sut=u"vpp",
 197                 build=build_info[job_name][str(idx)][1].rsplit(u'~', 1)[0],
 198                 period=u"daily",
 199                 build_nr=idx,
 200                 testbed=build_info[job_name][str(idx)][2]))
 201
 202         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 203                               int(date[9:11]), int(date[12:])))
 204
 205     data_pd = OrderedDict()
 206     for key, value in zip(xaxis, data_y):
 207         data_pd[key] = value
 208
 209     anomaly_classification, avgs = classify_anomalies(data_pd)
 210
 211     anomalies = OrderedDict()
 212     anomalies_colors = list()
 213     anomalies_avgs = list()
 214     anomaly_color = {
 215         u"regression": 0.0,
 216         u"normal": 0.5,
 217         u"progression": 1.0
 218     }
 219     if anomaly_classification:
 220         for idx, (key, value) in enumerate(data_pd.items()):
 221             if anomaly_classification[idx] in \
 222                     (u"outlier", u"regression", u"progression"):
 223                 anomalies[key] = value
 224                 anomalies_colors.append(
 225                     anomaly_color[anomaly_classification[idx]])
 226                 anomalies_avgs.append(avgs[idx])
 227         anomalies_colors.extend([0.0, 0.5, 1.0])
 228
 229     # Create traces
 230
 231     trace_samples = plgo.Scatter(
 232         x=xaxis,
 233         y=data_y,
 234         mode=u"markers",
 235         line={
 236             u"width": 1
 237         },
 238         showlegend=True,
 239         legendgroup=name,
 240         name=f"{name}",
 241         marker={
 242             u"size": 5,
 243             u"color": color,
 244             u"symbol": u"circle",
 245         },
 246         text=hover_text,
 247         hoverinfo=u"text"
 248     )
 249     traces = [trace_samples, ]
 250
 251     if show_trend_line:
 252         trace_trend = plgo.Scatter(
 253             x=xaxis,
 254             y=avgs,
 255             mode=u"lines",
 256             line={
 257                 u"shape": u"linear",
 258                 u"width": 1,
 259                 u"color": color,
 260             },
 261             showlegend=False,
 262             legendgroup=name,
 263             name=f"{name}",
 264             text=[f"trend: {int(avg):,}" for avg in avgs],
 265             hoverinfo=u"text+name"
 266         )
 267         traces.append(trace_trend)
 268
 269     trace_anomalies = plgo.Scatter(
 270         x=list(anomalies.keys()),
 271         y=anomalies_avgs,
 272         mode=u"markers",
 273         hoverinfo=u"none",
 274         showlegend=False,
 275         legendgroup=name,
 276         name=f"{name}-anomalies",
 277         marker={
 278             u"size": 15,
 279             u"symbol": u"circle-open",
 280             u"color": anomalies_colors,
 281             u"colorscale": [
 282                 [0.00, u"red"],
 283                 [0.33, u"red"],
 284                 [0.33, u"white"],
 285                 [0.66, u"white"],
 286                 [0.66, u"green"],
 287                 [1.00, u"green"]
 288             ],
 289             u"showscale": True,
 290             u"line": {
 291                 u"width": 2
 292             },
 293             u"colorbar": {
 294                 u"y": 0.5,
 295                 u"len": 0.8,
 296                 u"title": u"Circles Marking Data Classification",
 297                 u"titleside": u"right",
 298                 u"titlefont": {
 299                     u"size": 14
 300                 },
 301                 u"tickmode": u"array",
 302                 u"tickvals": [0.167, 0.500, 0.833],
 303                 u"ticktext": [u"Regression", u"Normal", u"Progression"],
 304                 u"ticks": u"",
 305                 u"ticklen": 0,
 306                 u"tickangle": -90,
 307                 u"thickness": 10
 308             }
 309         }
 310     )
 311     traces.append(trace_anomalies)
 312
 313     if anomaly_classification:
 314         return traces, anomaly_classification[-1]
 315
 316     return traces, None
 317
 318
 319 def _generate_all_charts(spec, input_data):
 320     """Generate all charts specified in the specification file.
 321
 322     :param spec: Specification.
 323     :param input_data: Full data set.
 324     :type spec: Specification
 325     :type input_data: InputData
 326     """
 327
 328     def _generate_chart(graph):
 329         """Generates the chart.
 330
 331         :param graph: The graph to be generated
 332         :type graph: dict
 333         :returns: Dictionary with the job name, csv table with results and
 334             list of tests classification results.
 335         :rtype: dict
 336         """
 337
 338         logs = list()
 339
 340         logs.append(
 341             (u"INFO", f"  Generating the chart {graph.get(u'title', u'')} ...")
 342         )
 343
 344         job_name = list(graph[u"data"].keys())[0]
 345
 346         csv_tbl = list()
 347         res = dict()
 348
 349         # Transform the data
 350         logs.append(
 351             (u"INFO",
 352              f"    Creating the data set for the {graph.get(u'type', u'')} "
 353              f"{graph.get(u'title', u'')}."
 354             )
 355         )
 356         data = input_data.filter_data(graph, continue_on_error=True)
 357         if data is None:
 358             logging.error(u"No data.")
 359             return dict()
 360
 361         chart_data = dict()
 362         chart_tags = dict()
 363         for job, job_data in data.items():
 364             if job != job_name:
 365                 continue
 366             for index, bld in job_data.items():
 367                 for test_name, test in bld.items():
 368                     if chart_data.get(test_name, None) is None:
 369                         chart_data[test_name] = OrderedDict()
 370                     try:
 371                         chart_data[test_name][int(index)] = \
 372                             test[u"result"][u"receive-rate"]
 373                         chart_tags[test_name] = test.get(u"tags", None)
 374                     except (KeyError, TypeError):
 375                         pass
 376
 377         # Add items to the csv table:
 378         for tst_name, tst_data in chart_data.items():
 379             tst_lst = list()
 380             for bld in builds_dict[job_name]:
 381                 itm = tst_data.get(int(bld), u'')
 382                 # CSIT-1180: Itm will be list, compute stats.
 383                 tst_lst.append(str(itm))
 384             csv_tbl.append(f"{tst_name}," + u",".join(tst_lst) + u'\n')
 385
 386         # Generate traces:
 387         traces = list()
 388         index = 0
 389         groups = graph.get(u"groups", None)
 390         visibility = list()
 391
 392         if groups:
 393             for group in groups:
 394                 visible = list()
 395                 for tag in group:
 396                     for tst_name, test_data in chart_data.items():
 397                         if not test_data:
 398                             logs.append(
 399                                 (u"WARNING", f"No data for the test {tst_name}")
 400                             )
 401                             continue
 402                         if tag not in chart_tags[tst_name]:
 403                             continue
 404                         message = f"index: {index}, test: {tst_name}"
 405                         try:
 406                             trace, rslt = _generate_trending_traces(
 407                                 test_data,
 408                                 job_name=job_name,
 409                                 build_info=build_info,
 410                                 name=u'-'.join(tst_name.split(u'.')[-1].
 411                                                split(u'-')[2:-1]),
 412                                 color=COLORS[index])
 413                         except IndexError:
 414                             logs.append(
 415                                 (u"ERROR", f"Out of colors: {message}")
 416                             )
 417                             logging.error(f"Out of colors: {message}")
 418                             index += 1
 419                             continue
 420                         traces.extend(trace)
 421                         visible.extend([True for _ in range(len(trace))])
 422                         res[tst_name] = rslt
 423                         index += 1
 424                         break
 425                 visibility.append(visible)
 426         else:
 427             for tst_name, test_data in chart_data.items():
 428                 if not test_data:
 429                     logs.append(
 430                         (u"WARNING", f"No data for the test {tst_name}")
 431                     )
 432                     continue
 433                 message = f"index: {index}, test: {tst_name}"
 434                 try:
 435                     trace, rslt = _generate_trending_traces(
 436                         test_data,
 437                         job_name=job_name,
 438                         build_info=build_info,
 439                         name=u'-'.join(
 440                             tst_name.split(u'.')[-1].split(u'-')[2:-1]),
 441                         color=COLORS[index])
 442                 except IndexError:
 443                     logs.append((u"ERROR", f"Out of colors: {message}"))
 444                     logging.error(f"Out of colors: {message}")
 445                     index += 1
 446                     continue
 447                 traces.extend(trace)
 448                 res[tst_name] = rslt
 449                 index += 1
 450
 451         if traces:
 452             # Generate the chart:
 453             try:
 454                 layout = deepcopy(graph[u"layout"])
 455             except KeyError as err:
 456                 logging.error(u"Finished with error: No layout defined")
 457                 logging.error(repr(err))
 458                 return dict()
 459             if groups:
 460                 show = list()
 461                 for i in range(len(visibility)):
 462                     visible = list()
 463                     for vis_idx, _ in enumerate(visibility):
 464                         for _ in range(len(visibility[vis_idx])):
 465                             visible.append(i == vis_idx)
 466                     show.append(visible)
 467
 468                 buttons = list()
 469                 buttons.append(dict(
 470                     label=u"All",
 471                     method=u"update",
 472                     args=[{u"visible": [True for _ in range(len(show[0]))]}, ]
 473                 ))
 474                 for i in range(len(groups)):
 475                     try:
 476                         label = graph[u"group-names"][i]
 477                     except (IndexError, KeyError):
 478                         label = f"Group {i + 1}"
 479                     buttons.append(dict(
 480                         label=label,
 481                         method=u"update",
 482                         args=[{u"visible": show[i]}, ]
 483                     ))
 484
 485                 layout[u"updatemenus"] = list([
 486                     dict(
 487                         active=0,
 488                         type=u"dropdown",
 489                         direction=u"down",
 490                         xanchor=u"left",
 491                         yanchor=u"bottom",
 492                         x=-0.12,
 493                         y=1.0,
 494                         buttons=buttons
 495                     )
 496                 ])
 497
 498             name_file = (
 499                 f"{spec.cpta[u'output-file']}-{graph[u'output-file-name']}"
 500                 f"{spec.cpta[u'output-file-type']}")
 501
 502             logs.append((u"INFO", f"    Writing the file {name_file} ..."))
 503             plpl = plgo.Figure(data=traces, layout=layout)
 504             try:
 505                 ploff.plot(plpl, show_link=False, auto_open=False,
 506                            filename=name_file)
 507             except plerr.PlotlyEmptyDataError:
 508                 logs.append((u"WARNING", u"No data for the plot. Skipped."))
 509
 510         for level, line in logs:
 511             if level == u"INFO":
 512                 logging.info(line)
 513             elif level == u"ERROR":
 514                 logging.error(line)
 515             elif level == u"DEBUG":
 516                 logging.debug(line)
 517             elif level == u"CRITICAL":
 518                 logging.critical(line)
 519             elif level == u"WARNING":
 520                 logging.warning(line)
 521
 522         return {u"job_name": job_name, u"csv_table": csv_tbl, u"results": res}
 523
 524     builds_dict = dict()
 525     for job in spec.input[u"builds"].keys():
 526         if builds_dict.get(job, None) is None:
 527             builds_dict[job] = list()
 528         for build in spec.input[u"builds"][job]:
 529             status = build[u"status"]
 530             if status not in (u"failed", u"not found", u"removed"):
 531                 builds_dict[job].append(str(build[u"build"]))
 532
 533     # Create "build ID": "date" dict:
 534     build_info = dict()
 535     tb_tbl = spec.environment.get(u"testbeds", None)
 536     for job_name, job_data in builds_dict.items():
 537         if build_info.get(job_name, None) is None:
 538             build_info[job_name] = OrderedDict()
 539         for build in job_data:
 540             testbed = u""
 541             tb_ip = input_data.metadata(job_name, build).get(u"testbed", u"")
 542             if tb_ip and tb_tbl:
 543                 testbed = tb_tbl.get(tb_ip, u"")
 544             build_info[job_name][build] = (
 545                 input_data.metadata(job_name, build).get(u"generated", u""),
 546                 input_data.metadata(job_name, build).get(u"version", u""),
 547                 testbed
 548             )
 549
 550     anomaly_classifications = dict()
 551
 552     # Create the header:
 553     csv_tables = dict()
 554     for job_name in builds_dict:
 555         if csv_tables.get(job_name, None) is None:
 556             csv_tables[job_name] = list()
 557         header = u"Build Number:," + u",".join(builds_dict[job_name]) + u'\n'
 558         csv_tables[job_name].append(header)
 559         build_dates = [x[0] for x in build_info[job_name].values()]
 560         header = u"Build Date:," + u",".join(build_dates) + u'\n'
 561         csv_tables[job_name].append(header)
 562         versions = [x[1] for x in build_info[job_name].values()]
 563         header = u"Version:," + u",".join(versions) + u'\n'
 564         csv_tables[job_name].append(header)
 565
 566     for chart in spec.cpta[u"plots"]:
 567         result = _generate_chart(chart)
 568         if not result:
 569             continue
 570
 571         csv_tables[result[u"job_name"]].extend(result[u"csv_table"])
 572
 573         if anomaly_classifications.get(result[u"job_name"], None) is None:
 574             anomaly_classifications[result[u"job_name"]] = dict()
 575         anomaly_classifications[result[u"job_name"]].update(result[u"results"])
 576
 577     # Write the tables:
 578     for job_name, csv_table in csv_tables.items():
 579         file_name = spec.cpta[u"output-file"] + u"-" + job_name + u"-trending"
 580         with open(f"{file_name}.csv", u"w") as file_handler:
 581             file_handler.writelines(csv_table)
 582
 583         txt_table = None
 584         with open(f"{file_name}.csv", u"rt") as csv_file:
 585             csv_content = csv.reader(csv_file, delimiter=u',', quotechar=u'"')
 586             line_nr = 0
 587             for row in csv_content:
 588                 if txt_table is None:
 589                     txt_table = prettytable.PrettyTable(row)
 590                 else:
 591                     if line_nr > 1:
 592                         for idx, item in enumerate(row):
 593                             try:
 594                                 row[idx] = str(round(float(item) / 1000000, 2))
 595                             except ValueError:
 596                                 pass
 597                     try:
 598                         txt_table.add_row(row)
 599                     # PrettyTable raises Exception
 600                     except Exception as err:
 601                         logging.warning(
 602                             f"Error occurred while generating TXT table:\n{err}"
 603                         )
 604                 line_nr += 1
 605             txt_table.align[u"Build Number:"] = u"l"
 606         with open(f"{file_name}.txt", u"w") as txt_file:
 607             txt_file.write(str(txt_table))
 608
 609     # Evaluate result:
 610     if anomaly_classifications:
 611         result = u"PASS"
 612         for job_name, job_data in anomaly_classifications.items():
 613             file_name = \
 614                 f"{spec.cpta[u'output-file']}-regressions-{job_name}.txt"
 615             with open(file_name, u'w') as txt_file:
 616                 for test_name, classification in job_data.items():
 617                     if classification == u"regression":
 618                         txt_file.write(test_name + u'\n')
 619                     if classification in (u"regression", u"outlier"):
 620                         result = u"FAIL"
 621             file_name = \
 622                 f"{spec.cpta[u'output-file']}-progressions-{job_name}.txt"
 623             with open(file_name, u'w') as txt_file:
 624                 for test_name, classification in job_data.items():
 625                     if classification == u"progression":
 626                         txt_file.write(test_name + u'\n')
 627     else:
 628         result = u"FAIL"
 629
 630     logging.info(f"Partial results: {anomaly_classifications}")
 631     logging.info(f"Result: {result}")
 632
 633     return result