resources/tools/presentation/generator_CPTA.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generation of Continuous Performance Trending and Analysis.
  15 """
  16
  17 import logging
  18 import csv
  19 import prettytable
  20 import plotly.offline as ploff
  21 import plotly.graph_objs as plgo
  22 import plotly.exceptions as plerr
  23
  24 from collections import OrderedDict
  25 from datetime import datetime
  26 from copy import deepcopy
  27
  28 from utils import archive_input_data, execute_command, classify_anomalies
  29
  30
  31 # Command to build the html format of the report
  32 HTML_BUILDER = 'sphinx-build -v -c conf_cpta -a ' \
  33                '-b html -E ' \
  34                '-t html ' \
  35                '-D version="{date}" ' \
  36                '{working_dir} ' \
  37                '{build_dir}/'
  38
  39 # .css file for the html format of the report
  40 THEME_OVERRIDES = """/* override table width restrictions */
  41 .wy-nav-content {
  42     max-width: 1200px !important;
  43 }
  44 .rst-content blockquote {
  45     margin-left: 0px;
  46     line-height: 18px;
  47     margin-bottom: 0px;
  48 }
  49 .wy-menu-vertical a {
  50     display: inline-block;
  51     line-height: 18px;
  52     padding: 0 2em;
  53     display: block;
  54     position: relative;
  55     font-size: 90%;
  56     color: #d9d9d9
  57 }
  58 .wy-menu-vertical li.current a {
  59     color: gray;
  60     border-right: solid 1px #c9c9c9;
  61     padding: 0 3em;
  62 }
  63 .wy-menu-vertical li.toctree-l2.current > a {
  64     background: #c9c9c9;
  65     padding: 0 3em;
  66 }
  67 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
  68     display: block;
  69     background: #c9c9c9;
  70     padding: 0 4em;
  71 }
  72 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
  73     display: block;
  74     background: #bdbdbd;
  75     padding: 0 5em;
  76 }
  77 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
  78     color: #404040;
  79     padding: 0 2em;
  80     font-weight: bold;
  81     position: relative;
  82     background: #fcfcfc;
  83     border: none;
  84         border-top-width: medium;
  85         border-bottom-width: medium;
  86         border-top-style: none;
  87         border-bottom-style: none;
  88         border-top-color: currentcolor;
  89         border-bottom-color: currentcolor;
  90     padding-left: 2em -4px;
  91 }
  92 """
  93
  94 COLORS = ["SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
  95           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
  96           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
  97           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
  98           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
  99           "MediumSeaGreen", "SeaGreen", "LightSlateGrey",
 100           "SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
 101           "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
 102           "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
 103           "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
 104           "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
 105           "MediumSeaGreen", "SeaGreen", "LightSlateGrey"
 106           ]
 107
 108
 109 def generate_cpta(spec, data):
 110     """Generate all formats and versions of the Continuous Performance Trending
 111     and Analysis.
 112
 113     :param spec: Specification read from the specification file.
 114     :param data: Full data set.
 115     :type spec: Specification
 116     :type data: InputData
 117     """
 118
 119     logging.info("Generating the Continuous Performance Trending and Analysis "
 120                  "...")
 121
 122     ret_code = _generate_all_charts(spec, data)
 123
 124     cmd = HTML_BUILDER.format(
 125         date=datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC'),
 126         working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
 127         build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"])
 128     execute_command(cmd)
 129
 130     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE]"], "w") as \
 131             css_file:
 132         css_file.write(THEME_OVERRIDES)
 133
 134     with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE2]"], "w") as \
 135             css_file:
 136         css_file.write(THEME_OVERRIDES)
 137
 138     if spec.configuration.get("archive-inputs", True):
 139         archive_input_data(spec)
 140
 141     logging.info("Done.")
 142
 143     return ret_code
 144
 145
 146 def _generate_trending_traces(in_data, job_name, build_info,
 147                               show_trend_line=True, name="", color=""):
 148     """Generate the trending traces:
 149      - samples,
 150      - outliers, regress, progress
 151      - average of normal samples (trending line)
 152
 153     :param in_data: Full data set.
 154     :param job_name: The name of job which generated the data.
 155     :param build_info: Information about the builds.
 156     :param show_trend_line: Show moving median (trending plot).
 157     :param name: Name of the plot
 158     :param color: Name of the color for the plot.
 159     :type in_data: OrderedDict
 160     :type job_name: str
 161     :type build_info: dict
 162     :type show_trend_line: bool
 163     :type name: str
 164     :type color: str
 165     :returns: Generated traces (list) and the evaluated result.
 166     :rtype: tuple(traces, result)
 167     """
 168
 169     data_x = list(in_data.keys())
 170     data_y = list(in_data.values())
 171
 172     hover_text = list()
 173     xaxis = list()
 174     for idx in data_x:
 175         date = build_info[job_name][str(idx)][0]
 176         hover_str = ("date: {date}<br>"
 177                      "value: {value:,}<br>"
 178                      "{sut}-ref: {build}<br>"
 179                      "csit-ref: mrr-{period}-build-{build_nr}<br>"
 180                      "testbed: {testbed}")
 181         if "dpdk" in job_name:
 182             hover_text.append(hover_str.format(
 183                 date=date,
 184                 value=int(in_data[idx].avg),
 185                 sut="dpdk",
 186                 build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
 187                 period="weekly",
 188                 build_nr=idx,
 189                 testbed=build_info[job_name][str(idx)][2]))
 190         elif "vpp" in job_name:
 191             hover_text.append(hover_str.format(
 192                 date=date,
 193                 value=int(in_data[idx].avg),
 194                 sut="vpp",
 195                 build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
 196                 period="daily",
 197                 build_nr=idx,
 198                 testbed=build_info[job_name][str(idx)][2]))
 199
 200         xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
 201                               int(date[9:11]), int(date[12:])))
 202
 203     data_pd = OrderedDict()
 204     for key, value in zip(xaxis, data_y):
 205         data_pd[key] = value
 206
 207     anomaly_classification, avgs = classify_anomalies(data_pd)
 208
 209     anomalies = OrderedDict()
 210     anomalies_colors = list()
 211     anomalies_avgs = list()
 212     anomaly_color = {
 213         "regression": 0.0,
 214         "normal": 0.5,
 215         "progression": 1.0
 216     }
 217     if anomaly_classification:
 218         for idx, (key, value) in enumerate(data_pd.iteritems()):
 219             if anomaly_classification[idx] in \
 220                     ("outlier", "regression", "progression"):
 221                 anomalies[key] = value
 222                 anomalies_colors.append(
 223                     anomaly_color[anomaly_classification[idx]])
 224                 anomalies_avgs.append(avgs[idx])
 225         anomalies_colors.extend([0.0, 0.5, 1.0])
 226
 227     # Create traces
 228
 229     trace_samples = plgo.Scatter(
 230         x=xaxis,
 231         y=[y.avg for y in data_y],
 232         mode='markers',
 233         line={
 234             "width": 1
 235         },
 236         showlegend=True,
 237         legendgroup=name,
 238         name="{name}".format(name=name),
 239         marker={
 240             "size": 5,
 241             "color": color,
 242             "symbol": "circle",
 243         },
 244         text=hover_text,
 245         hoverinfo="text"
 246     )
 247     traces = [trace_samples, ]
 248
 249     if show_trend_line:
 250         trace_trend = plgo.Scatter(
 251             x=xaxis,
 252             y=avgs,
 253             mode='lines',
 254             line={
 255                 "shape": "linear",
 256                 "width": 1,
 257                 "color": color,
 258             },
 259             showlegend=False,
 260             legendgroup=name,
 261             name='{name}'.format(name=name),
 262             text=["trend: {0:,}".format(int(avg)) for avg in avgs],
 263             hoverinfo="text+name"
 264         )
 265         traces.append(trace_trend)
 266
 267     trace_anomalies = plgo.Scatter(
 268         x=anomalies.keys(),
 269         y=anomalies_avgs,
 270         mode='markers',
 271         hoverinfo="none",
 272         showlegend=False,
 273         legendgroup=name,
 274         name="{name}-anomalies".format(name=name),
 275         marker={
 276             "size": 15,
 277             "symbol": "circle-open",
 278             "color": anomalies_colors,
 279             "colorscale": [[0.00, "red"],
 280                            [0.33, "red"],
 281                            [0.33, "white"],
 282                            [0.66, "white"],
 283                            [0.66, "green"],
 284                            [1.00, "green"]],
 285             "showscale": True,
 286             "line": {
 287                 "width": 2
 288             },
 289             "colorbar": {
 290                 "y": 0.5,
 291                 "len": 0.8,
 292                 "title": "Circles Marking Data Classification",
 293                 "titleside": 'right',
 294                 "titlefont": {
 295                     "size": 14
 296                 },
 297                 "tickmode": 'array',
 298                 "tickvals": [0.167, 0.500, 0.833],
 299                 "ticktext": ["Regression", "Normal", "Progression"],
 300                 "ticks": "",
 301                 "ticklen": 0,
 302                 "tickangle": -90,
 303                 "thickness": 10
 304             }
 305         }
 306     )
 307     traces.append(trace_anomalies)
 308
 309     if anomaly_classification:
 310         return traces, anomaly_classification[-1]
 311     else:
 312         return traces, None
 313
 314
 315 def _generate_all_charts(spec, input_data):
 316     """Generate all charts specified in the specification file.
 317
 318     :param spec: Specification.
 319     :param input_data: Full data set.
 320     :type spec: Specification
 321     :type input_data: InputData
 322     """
 323
 324     def _generate_chart(graph):
 325         """Generates the chart.
 326         """
 327
 328         logs = list()
 329
 330         logging.info("  Generating the chart '{0}' ...".
 331                      format(graph.get("title", "")))
 332         logs.append(("INFO", "  Generating the chart '{0}' ...".
 333                      format(graph.get("title", ""))))
 334
 335         job_name = graph["data"].keys()[0]
 336
 337         csv_tbl = list()
 338         res = list()
 339
 340         # Transform the data
 341         logs.append(("INFO", "    Creating the data set for the {0} '{1}'.".
 342                      format(graph.get("type", ""), graph.get("title", ""))))
 343         data = input_data.filter_data(graph, continue_on_error=True)
 344         if data is None:
 345             logging.error("No data.")
 346             return
 347
 348         chart_data = dict()
 349         chart_tags = dict()
 350         for job, job_data in data.iteritems():
 351             if job != job_name:
 352                 continue
 353             for index, bld in job_data.items():
 354                 for test_name, test in bld.items():
 355                     if chart_data.get(test_name, None) is None:
 356                         chart_data[test_name] = OrderedDict()
 357                     try:
 358                         chart_data[test_name][int(index)] = \
 359                             test["result"]["receive-rate"]
 360                         chart_tags[test_name] = test.get("tags", None)
 361                     except (KeyError, TypeError):
 362                         pass
 363
 364         # Add items to the csv table:
 365         for tst_name, tst_data in chart_data.items():
 366             tst_lst = list()
 367             for bld in builds_dict[job_name]:
 368                 itm = tst_data.get(int(bld), '')
 369                 if not isinstance(itm, str):
 370                     itm = itm.avg
 371                 tst_lst.append(str(itm))
 372             csv_tbl.append("{0},".format(tst_name) + ",".join(tst_lst) + '\n')
 373
 374         # Generate traces:
 375         traces = list()
 376         index = 0
 377         groups = graph.get("groups", None)
 378         visibility = list()
 379
 380         if groups:
 381             for group in groups:
 382                 visible = list()
 383                 for tag in group:
 384                     for test_name, test_data in chart_data.items():
 385                         if not test_data:
 386                             logs.append(("WARNING",
 387                                          "No data for the test '{0}'".
 388                                          format(test_name)))
 389                             continue
 390                         if tag in chart_tags[test_name]:
 391                             message = "index: {index}, test: {test}".format(
 392                                 index=index, test=test_name)
 393                             test_name = test_name.split('.')[-1]
 394                             try:
 395                                 trace, rslt = _generate_trending_traces(
 396                                     test_data,
 397                                     job_name=job_name,
 398                                     build_info=build_info,
 399                                     name='-'.join(test_name.split('-')[2:-1]),
 400                                     color=COLORS[index])
 401                             except IndexError:
 402                                 message = "Out of colors: {}".format(message)
 403                                 logs.append(("ERROR", message))
 404                                 logging.error(message)
 405                                 index += 1
 406                                 continue
 407                             traces.extend(trace)
 408                             visible.extend([True for _ in range(len(trace))])
 409                             res.append(rslt)
 410                             index += 1
 411                             break
 412                 visibility.append(visible)
 413         else:
 414             for test_name, test_data in chart_data.items():
 415                 if not test_data:
 416                     logs.append(("WARNING", "No data for the test '{0}'".
 417                                  format(test_name)))
 418                     continue
 419                 message = "index: {index}, test: {test}".format(
 420                     index=index, test=test_name)
 421                 test_name = test_name.split('.')[-1]
 422                 try:
 423                     trace, rslt = _generate_trending_traces(
 424                         test_data,
 425                         job_name=job_name,
 426                         build_info=build_info,
 427                         name='-'.join(test_name.split('-')[2:-1]),
 428                         color=COLORS[index])
 429                 except IndexError:
 430                     message = "Out of colors: {}".format(message)
 431                     logs.append(("ERROR", message))
 432                     logging.error(message)
 433                     index += 1
 434                     continue
 435                 traces.extend(trace)
 436                 res.append(rslt)
 437                 index += 1
 438
 439         if traces:
 440             # Generate the chart:
 441             try:
 442                 layout = deepcopy(graph["layout"])
 443             except KeyError as err:
 444                 logging.error("Finished with error: No layout defined")
 445                 logging.error(repr(err))
 446                 return
 447             if groups:
 448                 show = list()
 449                 for i in range(len(visibility)):
 450                     visible = list()
 451                     for r in range(len(visibility)):
 452                         for _ in range(len(visibility[r])):
 453                             visible.append(i == r)
 454                     show.append(visible)
 455
 456                 buttons = list()
 457                 buttons.append(dict(
 458                     label="All",
 459                     method="update",
 460                     args=[{"visible": [True for _ in range(len(show[0]))]}, ]
 461                 ))
 462                 for i in range(len(groups)):
 463                     try:
 464                         label = graph["group-names"][i]
 465                     except (IndexError, KeyError):
 466                         label = "Group {num}".format(num=i + 1)
 467                     buttons.append(dict(
 468                         label=label,
 469                         method="update",
 470                         args=[{"visible": show[i]}, ]
 471                     ))
 472
 473                 layout['updatemenus'] = list([
 474                     dict(
 475                         active=0,
 476                         type="dropdown",
 477                         direction="down",
 478                         xanchor="left",
 479                         yanchor="bottom",
 480                         x=-0.12,
 481                         y=1.0,
 482                         buttons=buttons
 483                     )
 484                 ])
 485
 486             name_file = "{0}-{1}{2}".format(spec.cpta["output-file"],
 487                                             graph["output-file-name"],
 488                                             spec.cpta["output-file-type"])
 489
 490             logs.append(("INFO", "    Writing the file '{0}' ...".
 491                          format(name_file)))
 492             plpl = plgo.Figure(data=traces, layout=layout)
 493             try:
 494                 ploff.plot(plpl, show_link=False, auto_open=False,
 495                            filename=name_file)
 496             except plerr.PlotlyEmptyDataError:
 497                 logs.append(("WARNING", "No data for the plot. Skipped."))
 498
 499         for level, line in logs:
 500             if level == "INFO":
 501                 logging.info(line)
 502             elif level == "ERROR":
 503                 logging.error(line)
 504             elif level == "DEBUG":
 505                 logging.debug(line)
 506             elif level == "CRITICAL":
 507                 logging.critical(line)
 508             elif level == "WARNING":
 509                 logging.warning(line)
 510
 511         return {"job_name": job_name, "csv_table": csv_tbl, "results": res}
 512
 513     builds_dict = dict()
 514     for job in spec.input["builds"].keys():
 515         if builds_dict.get(job, None) is None:
 516             builds_dict[job] = list()
 517         for build in spec.input["builds"][job]:
 518             status = build["status"]
 519             if status != "failed" and status != "not found" and \
 520                 status != "removed":
 521                 builds_dict[job].append(str(build["build"]))
 522
 523     # Create "build ID": "date" dict:
 524     build_info = dict()
 525     tb_tbl = spec.environment.get("testbeds", None)
 526     for job_name, job_data in builds_dict.items():
 527         if build_info.get(job_name, None) is None:
 528             build_info[job_name] = OrderedDict()
 529         for build in job_data:
 530             testbed = ""
 531             tb_ip = input_data.metadata(job_name, build).get("testbed", "")
 532             if tb_ip and tb_tbl:
 533                 testbed = tb_tbl.get(tb_ip, "")
 534             build_info[job_name][build] = (
 535                 input_data.metadata(job_name, build).get("generated", ""),
 536                 input_data.metadata(job_name, build).get("version", ""),
 537                 testbed
 538             )
 539
 540     anomaly_classifications = list()
 541
 542     # Create the header:
 543     csv_tables = dict()
 544     for job_name in builds_dict.keys():
 545         if csv_tables.get(job_name, None) is None:
 546             csv_tables[job_name] = list()
 547         header = "Build Number:," + ",".join(builds_dict[job_name]) + '\n'
 548         csv_tables[job_name].append(header)
 549         build_dates = [x[0] for x in build_info[job_name].values()]
 550         header = "Build Date:," + ",".join(build_dates) + '\n'
 551         csv_tables[job_name].append(header)
 552         versions = [x[1] for x in build_info[job_name].values()]
 553         header = "Version:," + ",".join(versions) + '\n'
 554         csv_tables[job_name].append(header)
 555
 556     for chart in spec.cpta["plots"]:
 557         result = _generate_chart(chart)
 558
 559         anomaly_classifications.extend(result["results"])
 560         csv_tables[result["job_name"]].extend(result["csv_table"])
 561
 562     # Write the tables:
 563     for job_name, csv_table in csv_tables.items():
 564         file_name = spec.cpta["output-file"] + "-" + job_name + "-trending"
 565         with open("{0}.csv".format(file_name), 'w') as file_handler:
 566             file_handler.writelines(csv_table)
 567
 568         txt_table = None
 569         with open("{0}.csv".format(file_name), 'rb') as csv_file:
 570             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
 571             line_nr = 0
 572             for row in csv_content:
 573                 if txt_table is None:
 574                     txt_table = prettytable.PrettyTable(row)
 575                 else:
 576                     if line_nr > 1:
 577                         for idx, item in enumerate(row):
 578                             try:
 579                                 row[idx] = str(round(float(item) / 1000000, 2))
 580                             except ValueError:
 581                                 pass
 582                     try:
 583                         txt_table.add_row(row)
 584                     except Exception as err:
 585                         logging.warning("Error occurred while generating TXT "
 586                                         "table:\n{0}".format(err))
 587                 line_nr += 1
 588             txt_table.align["Build Number:"] = "l"
 589         with open("{0}.txt".format(file_name), "w") as txt_file:
 590             txt_file.write(str(txt_table))
 591
 592     # Evaluate result:
 593     if anomaly_classifications:
 594         result = "PASS"
 595         for classification in anomaly_classifications:
 596             if classification == "regression" or classification == "outlier":
 597                 result = "FAIL"
 598                 break
 599     else:
 600         result = "FAIL"
 601
 602     logging.info("Partial results: {0}".format(anomaly_classifications))
 603     logging.info("Result: {0}".format(result))
 604
 605     return result