resources/tools/presentation/generator_CPTA.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import logging
  18 import csv
  19 import prettytable
  20 import plotly.offline as ploff
  21 import plotly.graph_objs as plgo
  22 import plotly.exceptions as plerr
  23
  24 from collections import OrderedDict
  25 from datetime import datetime
  26 from copy import deepcopy
  27
  28 from utils import archive_input_data, execute_command, classify_anomalies
  29
  30
  31 # Command to build the html format of the report
  32 HTML_BUILDER = 'sphinx-build -v -c conf_cpta -a ' \
  33                '-b html -E ' \
  34                '-t html ' \
  35                '-D version="{date}" ' \
  36                '{working_dir} ' \
  37                '{build_dir}/'
  38
  39 # .css file for the html format of the report
  40 THEME_OVERRIDES = """/* override table width restrictions */
  41 .wy-nav-content {
  42     max-width: 1200px !important;
  43 }
  44 .rst-content blockquote {
  45     margin-left: 0px;
  46     line-height: 18px;
  47     margin-bottom: 0px;
  48 }
  49 .wy-menu-vertical a {
  50     display: inline-block;
  51     line-height: 18px;
  52     padding: 0 2em;
  53     display: block;
  54     position: relative;
  55     font-size: 90%;
  56     color: #d9d9d9
  57 }
  58 .wy-menu-vertical li.current a {
  59     color: gray;
  60     border-right: solid 1px #c9c9c9;
  61     padding: 0 3em;
  62 }
  63 .wy-menu-vertical li.toctree-l2.current > a {
  64     background: #c9c9c9;
  65     padding: 0 3em;
  66 }
  67 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
  68     display: block;
  69     background: #c9c9c9;
  70     padding: 0 4em;
  71 }
  72 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
  73     display: block;
  74     background: #bdbdbd;
  75     padding: 0 5em;
  76 }
  77 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
  78     color: #404040;
  79     padding: 0 2em;
  80     font-weight: bold;
  81     position: relative;
  82     background: #fcfcfc;
  83     border: none;
  84         border-top-width: medium;
  85         border-bottom-width: medium;
  86         border-top-style: none;
  87         border-bottom-style: none;
  88         border-top-color: currentcolor;
  89         border-bottom-color: currentcolor;
  90     padding-left: 2em -4px;
  91 }
  92 """
  93
  94 COLORS = ["SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
  95           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
  96           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
  97           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
  98           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
  99           "MediumSeaGreen", "SeaGreen", "LightSlateGrey",
 100           "SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
 101           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
 102           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
 103           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
 104           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
 105           "MediumSeaGreen", "SeaGreen", "LightSlateGrey"
 106           ]
 107
 108
 109 def generate_cpta(spec, data):
 110     """Generate all formats and versions of the Continuous Performance Trending
 111     and Analysis.
 112
 113     :param spec: Specification read from the specification file.
 114     :param data: Full data set.
 115     :type spec: Specification
 116     :type data: InputData
 117     """
 118
 119     logging.info("Generating the Continuous Performance Trending and Analysis "
 120                  "...")
 121
 122     ret_code = _generate_all_charts(spec, data)
 123
 124     cmd = HTML_BUILDER.format(
 125         date=datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC'),
 126         working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
 127         build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"])
 128     execute_command(cmd)
 129
 130     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE]"], "w") as \
 131             css_file:
 132         css_file.write(THEME_OVERRIDES)
 133
 134     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE2]"], "w") as \
 135             css_file:
 136         css_file.write(THEME_OVERRIDES)
 137
 138     if spec.configuration.get("archive-inputs", True):
 139         archive_input_data(spec)
 140
 141     logging.info("Done.")
 142
 143     return ret_code
 144
 145
 146 def _generate_trending_traces(in_data, job_name, build_info,
 147                               show_trend_line=True, name="", color=""):
 148     """Generate the trending traces:
 149      - samples,
 150      - outliers, regress, progress
 151      - average of normal samples (trending line)
 152
 153     :param in_data: Full data set.
 154     :param job_name: The name of job which generated the data.
 155     :param build_info: Information about the builds.
 156     :param show_trend_line: Show moving median (trending plot).
 157     :param name: Name of the plot
 158     :param color: Name of the color for the plot.
 159     :type in_data: OrderedDict
 160     :type job_name: str
 161     :type build_info: dict
 162     :type show_trend_line: bool
 163     :type name: str
 164     :type color: str
 165     :returns: Generated traces (list) and the evaluated result.
 166     :rtype: tuple(traces, result)
 167     """
 168
 169     data_x = list(in_data.keys())
 170     data_y = list(in_data.values())
 171
 172     hover_text = list()
 173     xaxis = list()
 174     for idx in data_x:
 175         date = build_info[job_name][str(idx)][0]
 176         hover_str = ("date: {date}<br>"
 177                      "value: {value:,}<br>"
 178                      "{sut}-ref: {build}<br>"
 179                      "csit-ref: mrr-{period}-build-{build_nr}<br>"
 180                      "testbed: {testbed}")
 181         if "dpdk" in job_name:
 182             hover_text.append(hover_str.format(
 183                 date=date,
 184                 value=int(in_data[idx]),
 185                 sut="dpdk",
 186                 build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
 187                 period="weekly",
 188                 build_nr=idx,
 189                 testbed=build_info[job_name][str(idx)][2]))
 190         elif "vpp" in job_name:
 191             hover_text.append(hover_str.format(
 192                 date=date,
 193                 value=int(in_data[idx]),
 194                 sut="vpp",
 195                 build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
 196                 period="daily",
 197                 build_nr=idx,
 198                 testbed=build_info[job_name][str(idx)][2]))
 199
 200         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 201                               int(date[9:11]), int(date[12:])))
 202
 203     data_pd = OrderedDict()
 204     for key, value in zip(xaxis, data_y):
 205         data_pd[key] = value
 206
 207     anomaly_classification, avgs = classify_anomalies(data_pd)
 208
 209     anomalies = OrderedDict()
 210     anomalies_colors = list()
 211     anomalies_avgs = list()
 212     anomaly_color = {
 213         "regression": 0.0,
 214         "normal": 0.5,
 215         "progression": 1.0
 216     }
 217     if anomaly_classification:
 218         for idx, (key, value) in enumerate(data_pd.iteritems()):
 219             if anomaly_classification[idx] in \
 220                     ("outlier", "regression", "progression"):
 221                 anomalies[key] = value
 222                 anomalies_colors.append(
 223                     anomaly_color[anomaly_classification[idx]])
 224                 anomalies_avgs.append(avgs[idx])
 225         anomalies_colors.extend([0.0, 0.5, 1.0])
 226
 227     # Create traces
 228
 229     trace_samples = plgo.Scatter(
 230         x=xaxis,
 231         y=[y for y in data_y],  # Was: y.avg
 232         mode='markers',
 233         line={
 234             "width": 1
 235         },
 236         showlegend=True,
 237         legendgroup=name,
 238         name="{name}".format(name=name),
 239         marker={
 240             "size": 5,
 241             "color": color,
 242             "symbol": "circle",
 243         },
 244         text=hover_text,
 245         hoverinfo="text"
 246     )
 247     traces = [trace_samples, ]
 248
 249     if show_trend_line:
 250         trace_trend = plgo.Scatter(
 251             x=xaxis,
 252             y=avgs,
 253             mode='lines',
 254             line={
 255                 "shape": "linear",
 256                 "width": 1,
 257                 "color": color,
 258             },
 259             showlegend=False,
 260             legendgroup=name,
 261             name='{name}'.format(name=name),
 262             text=["trend: {0:,}".format(int(avg)) for avg in avgs],
 263             hoverinfo="text+name"
 264         )
 265         traces.append(trace_trend)
 266
 267     trace_anomalies = plgo.Scatter(
 268         x=anomalies.keys(),
 269         y=anomalies_avgs,
 270         mode='markers',
 271         hoverinfo="none",
 272         showlegend=False,
 273         legendgroup=name,
 274         name="{name}-anomalies".format(name=name),
 275         marker={
 276             "size": 15,
 277             "symbol": "circle-open",
 278             "color": anomalies_colors,
 279             "colorscale": [[0.00, "red"],
 280                            [0.33, "red"],
 281                            [0.33, "white"],
 282                            [0.66, "white"],
 283                            [0.66, "green"],
 284                            [1.00, "green"]],
 285             "showscale": True,
 286             "line": {
 287                 "width": 2
 288             },
 289             "colorbar": {
 290                 "y": 0.5,
 291                 "len": 0.8,
 292                 "title": "Circles Marking Data Classification",
 293                 "titleside": 'right',
 294                 "titlefont": {
 295                     "size": 14
 296                 },
 297                 "tickmode": 'array',
 298                 "tickvals": [0.167, 0.500, 0.833],
 299                 "ticktext": ["Regression", "Normal", "Progression"],
 300                 "ticks": "",
 301                 "ticklen": 0,
 302                 "tickangle": -90,
 303                 "thickness": 10
 304             }
 305         }
 306     )
 307     traces.append(trace_anomalies)
 308
 309     if anomaly_classification:
 310         return traces, anomaly_classification[-1]
 311     else:
 312         return traces, None
 313
 314
 315 def _generate_all_charts(spec, input_data):
 316     """Generate all charts specified in the specification file.
 317
 318     :param spec: Specification.
 319     :param input_data: Full data set.
 320     :type spec: Specification
 321     :type input_data: InputData
 322     """
 323
 324     def _generate_chart(graph):
 325         """Generates the chart.
 326         """
 327
 328         logs = list()
 329
 330         logs.append(("INFO", "  Generating the chart '{0}' ...".
 331                      format(graph.get("title", ""))))
 332
 333         job_name = graph["data"].keys()[0]
 334
 335         csv_tbl = list()
 336         res = dict()
 337
 338         # Transform the data
 339         logs.append(("INFO", "    Creating the data set for the {0} '{1}'.".
 340                      format(graph.get("type", ""), graph.get("title", ""))))
 341         data = input_data.filter_data(graph, continue_on_error=True)
 342         if data is None:
 343             logging.error("No data.")
 344             return
 345
 346         chart_data = dict()
 347         chart_tags = dict()
 348         for job, job_data in data.iteritems():
 349             if job != job_name:
 350                 continue
 351             for index, bld in job_data.items():
 352                 for test_name, test in bld.items():
 353                     if chart_data.get(test_name, None) is None:
 354                         chart_data[test_name] = OrderedDict()
 355                     try:
 356                         chart_data[test_name][int(index)] = \
 357                             test["result"]["receive-rate"]
 358                         chart_tags[test_name] = test.get("tags", None)
 359                     except (KeyError, TypeError):
 360                         pass
 361
 362         # Add items to the csv table:
 363         for tst_name, tst_data in chart_data.items():
 364             tst_lst = list()
 365             for bld in builds_dict[job_name]:
 366                 itm = tst_data.get(int(bld), '')
 367                 # CSIT-1180: Itm will be list, compute stats.
 368                 tst_lst.append(str(itm))
 369             csv_tbl.append("{0},".format(tst_name) + ",".join(tst_lst) + '\n')
 370
 371         # Generate traces:
 372         traces = list()
 373         index = 0
 374         groups = graph.get("groups", None)
 375         visibility = list()
 376
 377         if groups:
 378             for group in groups:
 379                 visible = list()
 380                 for tag in group:
 381                     for tst_name, test_data in chart_data.items():
 382                         if not test_data:
 383                             logs.append(("WARNING",
 384                                          "No data for the test '{0}'".
 385                                          format(tst_name)))
 386                             continue
 387                         if tag in chart_tags[tst_name]:
 388                             message = "index: {index}, test: {test}".format(
 389                                 index=index, test=tst_name)
 390                             try:
 391                                 trace, rslt = _generate_trending_traces(
 392                                     test_data,
 393                                     job_name=job_name,
 394                                     build_info=build_info,
 395                                     name='-'.join(tst_name.split('.')[-1].
 396                                                   split('-')[2:-1]),
 397                                     color=COLORS[index])
 398                             except IndexError:
 399                                 message = "Out of colors: {}".format(message)
 400                                 logs.append(("ERROR", message))
 401                                 logging.error(message)
 402                                 index += 1
 403                                 continue
 404                             traces.extend(trace)
 405                             visible.extend([True for _ in range(len(trace))])
 406                             res[tst_name] = rslt
 407                             index += 1
 408                             break
 409                 visibility.append(visible)
 410         else:
 411             for tst_name, test_data in chart_data.items():
 412                 if not test_data:
 413                     logs.append(("WARNING", "No data for the test '{0}'".
 414                                  format(tst_name)))
 415                     continue
 416                 message = "index: {index}, test: {test}".format(
 417                     index=index, test=tst_name)
 418                 try:
 419                     trace, rslt = _generate_trending_traces(
 420                         test_data,
 421                         job_name=job_name,
 422                         build_info=build_info,
 423                         name='-'.join(tst_name.split('.')[-1].split('-')[2:-1]),
 424                         color=COLORS[index])
 425                 except IndexError:
 426                     message = "Out of colors: {}".format(message)
 427                     logs.append(("ERROR", message))
 428                     logging.error(message)
 429                     index += 1
 430                     continue
 431                 traces.extend(trace)
 432                 res[tst_name] = rslt
 433                 index += 1
 434
 435         if traces:
 436             # Generate the chart:
 437             try:
 438                 layout = deepcopy(graph["layout"])
 439             except KeyError as err:
 440                 logging.error("Finished with error: No layout defined")
 441                 logging.error(repr(err))
 442                 return
 443             if groups:
 444                 show = list()
 445                 for i in range(len(visibility)):
 446                     visible = list()
 447                     for r in range(len(visibility)):
 448                         for _ in range(len(visibility[r])):
 449                             visible.append(i == r)
 450                     show.append(visible)
 451
 452                 buttons = list()
 453                 buttons.append(dict(
 454                     label="All",
 455                     method="update",
 456                     args=[{"visible": [True for _ in range(len(show[0]))]}, ]
 457                 ))
 458                 for i in range(len(groups)):
 459                     try:
 460                         label = graph["group-names"][i]
 461                     except (IndexError, KeyError):
 462                         label = "Group {num}".format(num=i + 1)
 463                     buttons.append(dict(
 464                         label=label,
 465                         method="update",
 466                         args=[{"visible": show[i]}, ]
 467                     ))
 468
 469                 layout['updatemenus'] = list([
 470                     dict(
 471                         active=0,
 472                         type="dropdown",
 473                         direction="down",
 474                         xanchor="left",
 475                         yanchor="bottom",
 476                         x=-0.12,
 477                         y=1.0,
 478                         buttons=buttons
 479                     )
 480                 ])
 481
 482             name_file = "{0}-{1}{2}".format(spec.cpta["output-file"],
 483                                             graph["output-file-name"],
 484                                             spec.cpta["output-file-type"])
 485
 486             logs.append(("INFO", "    Writing the file '{0}' ...".
 487                          format(name_file)))
 488             plpl = plgo.Figure(data=traces, layout=layout)
 489             try:
 490                 ploff.plot(plpl, show_link=False, auto_open=False,
 491                            filename=name_file)
 492             except plerr.PlotlyEmptyDataError:
 493                 logs.append(("WARNING", "No data for the plot. Skipped."))
 494
 495         for level, line in logs:
 496             if level == "INFO":
 497                 logging.info(line)
 498             elif level == "ERROR":
 499                 logging.error(line)
 500             elif level == "DEBUG":
 501                 logging.debug(line)
 502             elif level == "CRITICAL":
 503                 logging.critical(line)
 504             elif level == "WARNING":
 505                 logging.warning(line)
 506
 507         return {"job_name": job_name, "csv_table": csv_tbl, "results": res}
 508
 509     builds_dict = dict()
 510     for job in spec.input["builds"].keys():
 511         if builds_dict.get(job, None) is None:
 512             builds_dict[job] = list()
 513         for build in spec.input["builds"][job]:
 514             status = build["status"]
 515             if status != "failed" and status != "not found" and \
 516                 status != "removed":
 517                 builds_dict[job].append(str(build["build"]))
 518
 519     # Create "build ID": "date" dict:
 520     build_info = dict()
 521     tb_tbl = spec.environment.get("testbeds", None)
 522     for job_name, job_data in builds_dict.items():
 523         if build_info.get(job_name, None) is None:
 524             build_info[job_name] = OrderedDict()
 525         for build in job_data:
 526             testbed = ""
 527             tb_ip = input_data.metadata(job_name, build).get("testbed", "")
 528             if tb_ip and tb_tbl:
 529                 testbed = tb_tbl.get(tb_ip, "")
 530             build_info[job_name][build] = (
 531                 input_data.metadata(job_name, build).get("generated", ""),
 532                 input_data.metadata(job_name, build).get("version", ""),
 533                 testbed
 534             )
 535
 536     anomaly_classifications = dict()
 537
 538     # Create the header:
 539     csv_tables = dict()
 540     for job_name in builds_dict.keys():
 541         if csv_tables.get(job_name, None) is None:
 542             csv_tables[job_name] = list()
 543         header = "Build Number:," + ",".join(builds_dict[job_name]) + '\n'
 544         csv_tables[job_name].append(header)
 545         build_dates = [x[0] for x in build_info[job_name].values()]
 546         header = "Build Date:," + ",".join(build_dates) + '\n'
 547         csv_tables[job_name].append(header)
 548         versions = [x[1] for x in build_info[job_name].values()]
 549         header = "Version:," + ",".join(versions) + '\n'
 550         csv_tables[job_name].append(header)
 551
 552     for chart in spec.cpta["plots"]:
 553         result = _generate_chart(chart)
 554
 555         csv_tables[result["job_name"]].extend(result["csv_table"])
 556
 557         if anomaly_classifications.get(result["job_name"], None) is None:
 558             anomaly_classifications[result["job_name"]] = dict()
 559         anomaly_classifications[result["job_name"]].update(result["results"])
 560
 561     # Write the tables:
 562     for job_name, csv_table in csv_tables.items():
 563         file_name = spec.cpta["output-file"] + "-" + job_name + "-trending"
 564         with open("{0}.csv".format(file_name), 'w') as file_handler:
 565             file_handler.writelines(csv_table)
 566
 567         txt_table = None
 568         with open("{0}.csv".format(file_name), 'rb') as csv_file:
 569             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 570             line_nr = 0
 571             for row in csv_content:
 572                 if txt_table is None:
 573                     txt_table = prettytable.PrettyTable(row)
 574                 else:
 575                     if line_nr > 1:
 576                         for idx, item in enumerate(row):
 577                             try:
 578                                 row[idx] = str(round(float(item) / 1000000, 2))
 579                             except ValueError:
 580                                 pass
 581                     try:
 582                         txt_table.add_row(row)
 583                     except Exception as err:
 584                         logging.warning("Error occurred while generating TXT "
 585                                         "table:\n{0}".format(err))
 586                 line_nr += 1
 587             txt_table.align["Build Number:"] = "l"
 588         with open("{0}.txt".format(file_name), "w") as txt_file:
 589             txt_file.write(str(txt_table))
 590
 591     # Evaluate result:
 592     if anomaly_classifications:
 593         result = "PASS"
 594         for job_name, job_data in anomaly_classifications.iteritems():
 595             file_name = "{0}-regressions-{1}.txt".\
 596                 format(spec.cpta["output-file"], job_name)
 597             with open(file_name, 'w') as txt_file:
 598                 for test_name, classification in job_data.iteritems():
 599                     if classification == "regression":
 600                         txt_file.write(test_name + '\n')
 601                     if classification == "regression" or \
 602                             classification == "outlier":
 603                         result = "FAIL"
 604             file_name = "{0}-progressions-{1}.txt".\
 605                 format(spec.cpta["output-file"], job_name)
 606             with open(file_name, 'w') as txt_file:
 607                 for test_name, classification in job_data.iteritems():
 608                     if classification == "progression":
 609                         txt_file.write(test_name + '\n')
 610     else:
 611         result = "FAIL"
 612
 613     logging.info("Partial results: {0}".format(anomaly_classifications))
 614     logging.info("Result: {0}".format(result))
 615
 616     return result